Example #1
0
   def destroy( self, allowed_exit = [ 0 ] ):
      """
      Destroy credential

      Argument other than self:
         allowed_exit - List of exit codes accepted without error
                        when issuing system command for destroying credential

      Return value: False if command for destroying credential is undefined,
                    or True otherwise
      """

      if not self.command.destroy:
         logger.warning( "Dummy CommandSet used - no credential created" )
         return False

      destroyList = [ self.command.destroy ]
      for optName, optVal in self.command.destroyOpts.iteritems():
         destroyList.append( "%s %s" % ( optName, optVal ) )
         
      Coordinator.notifyInvalidCredential(self)
      
      status, output, message = \
         self.shell.cmd1( " ".join( destroyList ), allowed_exit )
      proxyPath = self.location()
      if proxyPath:
         os.remove( proxyPath )
      return True
Example #2
0
    def destroy(self, allowed_exit=[0]):
        """
        Destroy credential

        Argument other than self:
           allowed_exit - List of exit codes accepted without error
                          when issuing system command for destroying credential

        Return value: False if command for destroying credential is undefined,
                      or True otherwise
        """

        if not self.command.destroy:
            logger.warning("Dummy CommandSet used - no credential created")
            return False

        destroyList = [self.command.destroy]
        for optName, optVal in self.command.destroyOpts.iteritems():
            destroyList.append("%s %s" % (optName, optVal))

        Coordinator.notifyInvalidCredential(self)

        status, output, message = \
            self.shell.cmd1(" ".join(destroyList), allowed_exit)
        proxyPath = self.location()
        if proxyPath:
            os.remove(proxyPath)
        return True
Example #3
0
def bootstrap(reg_slice, interactive_session, my_interface=None):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # start the internal services coordinator
    from Ganga.Core.InternalServices import Coordinator
    Coordinator.bootstrap()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    # for j in reg_slice:
    #    if hasattr(j,'status') and j.status in ['submitted','running']:
    #        if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
    #            if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
    #                j.backend.setup()

    start_jobregistry_monitor(reg_slice)

    # Set the shutdown policy depending on whether we're interactive or not
    if config['forced_shutdown_policy'] in ['interactive', 'batch']:
        GangaThreadPool.shutdown_policy = config['forced_shutdown_policy']
    else:
        if interactive_session:
            GangaThreadPool.shutdown_policy = 'interactive'
        else:
            GangaThreadPool.shutdown_policy = 'batch'

    # export to GPI moved to Runtime bootstrap

    autostart_default = interactive_session
    config.overrideDefaultValue('autostart', bool(autostart_default))

    if config['autostart'] is not autostart_default:
        msg = 'monitoring loop %s (the default setting for %s session is %s)'
        val = {
            True: ('enabled', 'batch', 'disabled'),
            False: ('disabled', 'interactive', 'enabled')
        }
        logger.warning(msg % val[config['autostart']])

    if config['autostart']:
        monitoring_component.enableMonitoring()

    if not my_interface:
        import Ganga.GPI
        my_interface = Ganga.GPI
    from Ganga.Runtime.GPIexport import exportToInterface
    exportToInterface(my_interface, 'runMonitoring',
                      monitoring_component.runMonitoring, 'Functions')
Example #4
0
def bootstrap(reg_slice, interactive_session, my_interface=None):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # start the internal services coordinator
    from Ganga.Core.InternalServices import Coordinator

    Coordinator.bootstrap()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    # for j in reg_slice:
    #    if hasattr(j,'status') and j.status in ['submitted','running']:
    #        if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
    #            if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
    #                j.backend.setup()

    start_jobregistry_monitor(reg_slice)

    # Set the shutdown policy depending on whether we're interactive or not
    if config["forced_shutdown_policy"] in ["interactive", "batch"]:
        GangaThreadPool.shutdown_policy = config["forced_shutdown_policy"]
    else:
        if interactive_session:
            GangaThreadPool.shutdown_policy = "interactive"
        else:
            GangaThreadPool.shutdown_policy = "batch"

    # export to GPI moved to Runtime bootstrap

    autostart_default = interactive_session
    config.overrideDefaultValue("autostart", bool(autostart_default))

    if config["autostart"] is not autostart_default:
        msg = "monitoring loop %s (the default setting for %s session is %s)"
        val = {True: ("enabled", "batch", "disabled"), False: ("disabled", "interactive", "enabled")}
        logger.warning(msg % val[config["autostart"]])

    if config["autostart"]:
        monitoring_component.enableMonitoring()

    if not my_interface:
        import Ganga.GPI

        my_interface = Ganga.GPI
    from Ganga.Runtime.GPIexport import exportToInterface

    exportToInterface(my_interface, "runMonitoring", monitoring_component.runMonitoring, "Functions")
Example #5
0
def bootstrap(reg, interactive_session):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # start the internal services coordinator
    from Ganga.Core.InternalServices import Coordinator
    Coordinator.bootstrap()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    # for j in reg:
    #    if hasattr(j,'status') and j.status in ['submitted','running']:
    #        if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
    #            if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
    #                j.backend.setup()

    start_jobregistry_monitor(reg)

    # register the MC shutdown hook

    change_atexitPolicy(interactive_session)

    # export to GPI
    from Ganga.Runtime.GPIexport import exportToGPI
    exportToGPI(
        'runMonitoring', monitoring_component.runMonitoring, 'Functions')

    autostart_default = interactive_session
    config.overrideDefaultValue('autostart', bool(autostart_default))

    if config['autostart'] is not autostart_default:
        msg = 'monitoring loop %s (the default setting for %s session is %s)'
        val = {True: ('enabled', 'batch', 'disabled'),
               False: ('disabled', 'interactive', 'enabled')}
        logger.warning(msg % val[config['autostart']])

    if config['autostart']:
        monitoring_component.enableMonitoring()
Example #6
0
def start_ganga(gangadir_for_test, extra_opts=[]):
    """
    Startup Ganga by calling the same set of 'safe' functions each time
    Args:
        gangadir_for_test (str): This is the directory which the GangaUnitTest is to be run, a new gangadir has been created per test to avoid collisions
        extra_opts (list): A list of tuples which are used to pass command line style options to Ganga
    """

    import Ganga.PACKAGE
    Ganga.PACKAGE.standardSetup()

    # End taken from the ganga binary

    import Ganga.Runtime
    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Start ganga by passing some options for unittesting

    logger.info("Starting ganga")

    logger.info("Parsing Command Line options")
    this_argv = [
        'ganga',  # `argv[0]` is usually the name of the program so fake that here
    ]

    # These are the default options for all test instances
    # They can be overridden by extra_opts
    default_opts = [
        ('Configuration', 'RUNTIME_PATH', 'GangaTest'),
        ('Configuration', 'gangadir', gangadir_for_test),
        ('Configuration', 'user', 'testframework'),
        ('Configuration', 'repositorytype', 'LocalXML'),
        ('Configuration', 'UsageMonitoringMSG', False),  # Turn off spyware
        ('TestingFramework', 'ReleaseTesting', True),
        ('Queues', 'NumWorkerThreads', 2),
    ]

    # FIXME Should we need to add the ability to load from a custom .ini file
    # to configure tests without editting this?

    # Actually parse the options
    Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
    Ganga.Runtime._prog.default_config_file = ganga_config_file
    Ganga.Runtime._prog.parseOptions()

    # For all the default and extra options, we set the session value
    from Ganga.Utility.Config import setConfigOption
    for opt in default_opts + extra_opts:
        setConfigOption(*opt)

    # The configuration is currently created at module import and hence can't be
    # regenerated.
    # The values read in from any .ini file or from command line will change this
    # but the configuration can't be obliterated and re-created. (yet, 16.06.16)

    # Perform the configuration and bootstrap steps in ganga
    logger.info("Parsing Configuration Options")
    Ganga.Runtime._prog.configure()

    logger.info("Initializing")
    Ganga.Runtime._prog.initEnvironment()

    logger.info("Bootstrapping")
    Ganga.Runtime._prog.bootstrap(interactive=False)

    # We need to test if the internal services need to be reinitialized
    from Ganga.Core.InternalServices import Coordinator
    if not Coordinator.servicesEnabled:
        # Start internal services
        logger.info("InternalServices restarting")

        from Ganga.Core.InternalServices.Coordinator import enableInternalServices
        enableInternalServices()
    else:
        logger.info("InternalServices still running")

    # Adapted from the Coordinator class, check for the required credentials and stop if not found
    # Hopefully stops us falling over due to no AFS access of something similar
    from Ganga.Core.InternalServices import Coordinator
    missing_cred = Coordinator.getMissingCredentials()

    logger.info("Checking Credentials")

    if missing_cred:
        raise Exception("Failed due to missing credentials %s" %
                        str(missing_cred))

    # Make sure that all the config options are really set.
    # Some from plugins may not have taken during startup
    for opt in default_opts + extra_opts:
        setConfigOption(*opt)

    logger.info("Passing to Unittest")
    def runMonitoring(self, jobs=None, steps=1, timeout=60, _loadCredentials=False):
        """
        Enable/Run the monitoring loop and wait for the monitoring steps completion.
        Parameters:
          steps:   number of monitoring steps to run
          timeout: how long to wait for monitor steps termination (seconds)
          jobs: a registry slice to be monitored (None -> all jobs), it may be passed by the user so ._impl is stripped if needed
        Return:
          False, if the loop cannot be started or the timeout occured while waiting for monitoring termination
          True, if the monitoring steps were successfully executed  
        Note:         
          This method is meant to be used in Ganga scripts to request monitoring on demand. 
        """

        log.debug("runMonitoring")

        if not isType(steps, int) and steps < 0:
            log.warning("The number of monitor steps should be a positive (non-zero) integer")
            return False

        if not self.alive:
            log.error("Cannot run the monitoring loop. It has already been stopped")
            return False

        # we don not allow the user's request the monitoring loop while the
        # internal services are stopped
        if not Coordinator.servicesEnabled:
            log.error("Cannot run the monitoring loop."
                      "The internal services are disabled (check your credentials or available disk space)")
            return False

        # if the monitoring is disabled (e.g. scripts)
        if not self.enabled:
            # and there are some required cred which are missing
            # (the monitoring loop does not monitor the credentials so we need to check 'by hand' here)
            if _loadCredentials is True:
                _missingCreds = Coordinator.getMissingCredentials()
            else:
                _missingCreds = False
            if _missingCreds:
                log.error("Cannot run the monitoring loop. The following credentials are required: %s" % _missingCreds)
                return False

        #log.debug("jobs: %s" % str(jobs))
        #log.debug("self.__mainLoopCond: %s" % str(self.__mainLoopCond))

        with self.__mainLoopCond:
            log.debug('Monitoring loop lock acquired. Enabling mon loop')
            if self.enabled or self.__isInProgress():
                log.error("The monitoring loop is already running.")
                return False

            if jobs is not None:
                m_jobs = jobs

                # additional check if m_jobs is really a registry slice
                # the underlying code is not prepared to handle correctly the
                # situation if it is not
                from Ganga.GPIDev.Lib.Registry.RegistrySlice import RegistrySlice
                if not isType(m_jobs, RegistrySlice):
                    log.warning('runMonitoring: jobs argument must be a registry slice such as a result of jobs.select() or jobs[i1:i2]')
                    return False

                self.registry = m_jobs
                #log.debug("m_jobs: %s" % str(m_jobs))
                self.makeUpdateJobStatusFunction()

            log.debug("Enable Loop, Clear Iterators and setCallbackHook")
            # enable mon loop
            self.enabled = True
            # set how many steps to run
            self.steps = steps
            # enable job list iterators
            self.stopIter.clear()
            # Start backend update timeout checking.
            self.setCallbackHook(self.updateDict_ts.timeoutCheck, {}, True)

            log.debug("Waking up Main Loop")
            # wake up the mon loop
            self.__mainLoopCond.notifyAll()

        log.debug("Waiting to execute steps")
        # wait to execute the steps
        self.__monStepsTerminatedEvent.wait()
        self.__monStepsTerminatedEvent.clear()

        log.debug("Test for timeout")
        # wait the steps to be executed or timeout to occur
        if not self.__awaitTermination(timeout):
            log.warning("Monitoring loop started but did not complete in the given timeout.")
            # force loops termination
            self.stopIter.set()
            return False
        return True
Example #8
0
   def create( self, validity = "", maxTry = 0, minValidity = "", \
      check = False ):
      """
      Create credential.

      Arguments other than self:
         validity    - Validity with which credential should be created,
                       specified as string of format "hh:mm"
                       [ Defaults to value of self.validityAtCreation ]
         maxTry      - Number of password attempts allowed
                       [ Defaults to value of self.maxTry ]
         minValidity - Minimum validity in case checking of
                       pre-existing credential is performed,
                       specified as strong of format "hh:mm"
                       [ Defaults to value of self.minValidity ]
         check       - Flag to request checking of pre-existing
                       credential; if flag is set to true, then new
                       credential is created only if the validity of
                       any pre-existing credential is less than the
                       value of minValidity
                       [ Default: False ] 

      Note: create is the same as renew, except for the default value of check

      Return value: True if credential is created successfully, and False
      otherwise.
      """
      global logTimeStamp

      dummy = False
      if not self.command.init:
          dummy = True
      if self.command.init_parameters.has_key( "valid" ):
         if not self.command.init_parameters[ "valid" ]:
             dummy = True

      if dummy:
         logger.warning( "Dummy CommandSet used - no credential created" )
         return False

      if not maxTry:
         maxTry = self.maxTry

      if not minValidity:
         minValidity = self.minValidity

      if not validity:
         validity = self.validityAtCreation

      validityInSeconds = self.timeInSeconds( validity )

      if not validityInSeconds:
         logger.warning( "Problems with requested validity: %s" \
            % str( validity ) )
         return False
      if check and self.isValid( minValidity ):
         return True

      ntry = 0

      while ntry < maxTry:

         ntry = ntry + 1
         # Test if GUI widget is to be used.
         if self.inputPW_Widget:
            # Since self.inputPW_Widget is called, current arguments are
            # ignored since renew() and create() in GUI mode will not be
            # called with any arguments.
            if self.inputPW_Widget.ask( self._proxyObject ):
               logger.debug( "Proceeding to retrieve password from inputPW_Widget." )
               __pw = self.inputPW_Widget.getPassword( self._proxyObject )
               if not __pw:
                  logger.warning( "Password/passphrase expected!" )
                  return False
               try:
                  tFile = tempfile.NamedTemporaryFile()
                  tFile.write( __pw )
                  tFile.flush()
               except:
                  del __pw
                  logger.warning( "Could not create secure temporary file for password!" )
                  return False
               del __pw
            else:
               # Current credential modification denied for various reasons.
               # see GangaGUI.customDialogs.ask() method for more details.
               return False
            # self.inputPW_Widget.ask() may have modified parameters.
            # Calling buildOpts() to take them into account.
            self.buildOpts( self.command.init, False )
            # Create initialisation list with the 'pipe' parameter
            initList = [ self.command.init, self.command.init_parameters[ "pipe" ] ]
            # Append option value pairs
            for optName, optVal in self.command.currentOpts.iteritems():
               initList.append( "%s %s" % ( optName, optVal ) )
            status = self.shell.system( "cat %s|%s" % ( tFile.name, " ".join( initList ) ) )
            tFile.close()
            # self.inputPW_Widget dialog postprocessing. 
            # E.g. disable autorenew mechanism if status != 0.
            self.inputPW_Widget.renewalStatus( self._proxyObject, status )
            if status == 0:
               logger.info( "%s creation/renewal successful." % self._name )
               return True
            else:
               logger.warning( "%s creation/renewal failed [%s]." % ( self._name, status ) )
               return False
         else: # Non-GUI credential renewal/creation
            # Check if renewal is from main process (i.e. by bootstrap or user)
            if threading.currentThread().getName() == 'MainThread':
               if self.command.init_parameters.has_key( "valid" ):
                  self.command.currentOpts\
                     [ self.command.init_parameters[ 'valid' ] ] = validity
               initList = [ self.command.init ]
               # Append option value pairs
               for optName, optVal in self.command.currentOpts.iteritems():
                  initList.append( "%s %s" % ( optName, optVal ) )
               status = self.shell.system( " ".join( initList ) )
               if status == 0:
                  logger.info( "%s creation/renewal successful." % self._name )
                  return True
               else:
                  logger.warning( "%s creation/renewal failed [%s]." % ( self._name, status ) )
            else: # create initiated from worker thread from monitoring component.
               currTime = time.time()
               if currTime - logTimeStamp >= logRepeatDuration:
                  logTimeStamp = currTime

                  # Check validity but print logging messages this time
                  self.isValid( "", True )
                  _credentialObject = self._name[ 0 ].lower() + self._name[ 1: ]
                  logger.warning( \
                     "Renew by typing '%s.renew()' at the prompt." % \
                     ( _credentialObject ) )


                  #notify the Core that the credential is not valid                  
                  _validity = self.timeInSeconds(self.timeleft())
                  _minValidity = self.timeInSeconds(minValidity)/2
                  if _validity <= max(120,_minValidity):
                     Coordinator.notifyInvalidCredential(self)

               return True

      logger.warning( "%s creation/renewal attempts exceeded %s tries!" % ( self._name, maxTry ) )
      return False
Example #9
0
def start_ganga(gangadir_for_test, extra_opts=[]):
    """
    Startup Ganga by calling the same set of 'safe' functions each time
    Args:
        gangadir_for_test (str): This is the directory which the GangaUnitTest is to be run, a new gangadir has been created per test to avoid collisions
        extra_opts (list): A list of tuples which are used to pass command line style options to Ganga
    """

    import Ganga.PACKAGE
    Ganga.PACKAGE.standardSetup()

    # End taken from the ganga binary

    import Ganga.Runtime
    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Start ganga by passing some options for unittesting

    logger.info("Starting ganga")

    logger.info("Parsing Command Line options")
    this_argv = [
        'ganga',  # `argv[0]` is usually the name of the program so fake that here
    ]

    # These are the default options for all test instances
    # They can be overridden by extra_opts
    default_opts = [
        ('Configuration', 'RUNTIME_PATH', 'GangaTest'),
        ('Configuration', 'gangadir', gangadir_for_test),
        ('Configuration', 'user', 'testframework'),
        ('Configuration', 'repositorytype', 'LocalXML'),
        ('Configuration', 'UsageMonitoringMSG', False),  # Turn off spyware
        ('TestingFramework', 'ReleaseTesting', True),
        ('Queues', 'NumWorkerThreads', 2),
    ]

    # FIXME Should we need to add the ability to load from a custom .ini file
    # to configure tests without editting this?

    # Actually parse the options
    Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
    Ganga.Runtime._prog.parseOptions()

    # For all the default and extra options, we set the session value
    from Ganga.Utility.Config import setConfigOption
    for opt in default_opts + extra_opts:
        setConfigOption(*opt)

    # The configuration is currently created at module import and hence can't be
    # regenerated.
    # The values read in from any .ini file or from command line will change this
    # but the configuration can't be obliterated and re-created. (yet, 16.06.16)

    # Perform the configuration and bootstrap steps in ganga
    logger.info("Parsing Configuration Options")
    Ganga.Runtime._prog.configure()

    logger.info("Initializing")
    Ganga.Runtime._prog.initEnvironment()

    logger.info("Bootstrapping")
    Ganga.Runtime._prog.bootstrap(interactive=False)

    # We need to test if the internal services need to be reinitialized
    from Ganga.Core.InternalServices import Coordinator
    if not Coordinator.servicesEnabled:
        # Start internal services
        logger.info("InternalServices restarting")

        from Ganga.Core.InternalServices.Coordinator import enableInternalServices
        enableInternalServices()
    else:
        logger.info("InternalServices still running")

    # Adapted from the Coordinator class, check for the required credentials and stop if not found
    # Hopefully stops us falling over due to no AFS access of something similar
    from Ganga.Core.InternalServices import Coordinator
    missing_cred = Coordinator.getMissingCredentials()

    logger.info("Checking Credentials")

    if missing_cred:
        raise Exception("Failed due to missing credentials %s" % str(missing_cred))

    # Make sure that all the config options are really set.
    # Some from plugins may not have taken during startup
    for opt in default_opts + extra_opts:
        setConfigOption(*opt)

    logger.info("Passing to Unittest")
Example #10
0
    def runMonitoring(self,
                      jobs=None,
                      steps=1,
                      timeout=60,
                      _loadCredentials=False):
        """
        Enable/Run the monitoring loop and wait for the monitoring steps completion.
        Parameters:
          steps:   number of monitoring steps to run
          timeout: how long to wait for monitor steps termination (seconds)
          jobs: a registry slice to be monitored (None -> all jobs), it may be passed by the user so ._impl is stripped if needed
        Return:
          False, if the loop cannot be started or the timeout occured while waiting for monitoring termination
          True, if the monitoring steps were successfully executed  
        Note:         
          This method is meant to be used in Ganga scripts to request monitoring on demand. 
        """

        log.debug("runMonitoring")

        if not isType(steps, int) and steps < 0:
            log.warning(
                "The number of monitor steps should be a positive (non-zero) integer"
            )
            return False

        if not self.alive:
            log.error(
                "Cannot run the monitoring loop. It has already been stopped")
            return False

        # we don not allow the user's request the monitoring loop while the
        # internal services are stopped
        if not Coordinator.servicesEnabled:
            log.error(
                "Cannot run the monitoring loop."
                "The internal services are disabled (check your credentials or available disk space)"
            )
            return False

        # if the monitoring is disabled (e.g. scripts)
        if not self.enabled:
            # and there are some required cred which are missing
            # (the monitoring loop does not monitor the credentials so we need to check 'by hand' here)
            if _loadCredentials is True:
                _missingCreds = Coordinator.getMissingCredentials()
            else:
                _missingCreds = False
            if _missingCreds:
                log.error(
                    "Cannot run the monitoring loop. The following credentials are required: %s"
                    % _missingCreds)
                return False

        #log.debug("jobs: %s" % str(jobs))
        #log.debug("self.__mainLoopCond: %s" % str(self.__mainLoopCond))

        with self.__mainLoopCond:
            log.debug('Monitoring loop lock acquired. Enabling mon loop')
            if self.enabled or self.__isInProgress():
                log.error("The monitoring loop is already running.")
                return False

            if jobs is not None:
                m_jobs = jobs

                # additional check if m_jobs is really a registry slice
                # the underlying code is not prepared to handle correctly the
                # situation if it is not
                from Ganga.GPIDev.Lib.Registry.RegistrySlice import RegistrySlice
                if not isType(m_jobs, RegistrySlice):
                    log.warning(
                        'runMonitoring: jobs argument must be a registry slice such as a result of jobs.select() or jobs[i1:i2]'
                    )
                    return False

                self.registry = m_jobs
                #log.debug("m_jobs: %s" % str(m_jobs))
                self.makeUpdateJobStatusFunction()

            log.debug("Enable Loop, Clear Iterators and setCallbackHook")
            # enable mon loop
            self.enabled = True
            # set how many steps to run
            self.steps = steps
            # enable job list iterators
            self.stopIter.clear()
            # Start backend update timeout checking.
            self.setCallbackHook(updateDict_ts.timeoutCheck, {}, True)

            log.debug("Waking up Main Loop")
            # wake up the mon loop
            self.__mainLoopCond.notifyAll()

        log.debug("Waiting to execute steps")
        # wait to execute the steps
        self.__monStepsTerminatedEvent.wait()
        self.__monStepsTerminatedEvent.clear()

        log.debug("Test for timeout")
        # wait the steps to be executed or timeout to occur
        if not self.__awaitTermination(timeout):
            log.warning(
                "Monitoring loop started but did not complete in the given timeout."
            )
            # force loops termination
            self.stopIter.set()
            return False
        return True
Example #11
0
def start_ganga(gangadir_for_test='$HOME/gangadir_testing', extra_opts=[]):

    import sys
    import os.path


    file_path = os.path.dirname(os.path.realpath(__file__))
    ganga_python_dir = os.path.join(file_path, '..', '..', '..')
    ganga_python_dir = os.path.realpath(ganga_python_dir)
    if len(sys.path) >= 1 and ganga_python_dir != sys.path[0]:
        sys.path.insert(0, ganga_python_dir)

        print("Adding: %s to Python Path\n" % ganga_python_dir)

    import Ganga.PACKAGE
    Ganga.PACKAGE.standardSetup()

    # End taken from the ganga binary

    import Ganga.Runtime
    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Start ganga by passing some options for unittesting

    logger.info("Starting ganga")

    logger.info("Parsing Command Line options")
    import Ganga.Runtime
    this_argv = [
        'ganga',  # `argv[0]` is usually the name of the program so fake that here
    ]

    # These are the default options for all test instances
    # They can be overridden by extra_opts
    default_opts = [
        ('Configuration', 'RUNTIME_PATH', 'GangaTest'),
        ('Configuration', 'gangadir', gangadir_for_test),
        ('Configuration', 'user', 'testframework'),
        ('Configuration', 'repositorytype', 'LocalXML'),
        ('TestingFramework', 'ReleaseTesting', True),
        ('Queues', 'NumWorkerThreads', 2),
    ]

    # FIXME Should we need to add the ability to load from a custom .ini file
    # to configure tests without editting this?

    # Actually parse the options
    Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
    Ganga.Runtime._prog.parseOptions()

    # Determine if ganga has actually finished initializing...
    # This is here to protect against the startGanga being called on an
    # initialized ganga environment
    try:
        do_config = not Ganga.Utility.Config.Config._after_bootstrap
    except:
        do_config = True

    # For all the default and extra options, we set the session value
    from Ganga.Utility.Config import setConfigOption
    for opt in default_opts + extra_opts:
        setConfigOption(*opt)

    if do_config:
        # Perform the configuration and bootstrap steps in ganga
        logger.info("Parsing Configuration Options")
        Ganga.Runtime._prog.configure()
        logger.info("Initializing")
        Ganga.Runtime._prog.initEnvironment(opt_rexec=False)
    else:
        # We need to test if the internal services need to be reinitialized
        from Ganga.Core.InternalServices import Coordinator
        if not Coordinator.servicesEnabled:
            # Start internal services
            logger.info("InternalServices restarting")

            from Ganga.GPI import reactivate
            reactivate()
        else:
            logger.info("InternalServices still running")

        # The queues are shut down by the atexit handlers so we need to start them here
        from Ganga.Core.GangaThread.WorkerThreads import startUpQueues
        startUpQueues()

    logger.info("Bootstrapping")
    Ganga.Runtime._prog.bootstrap(interactive=False)

    # Adapted from the Coordinator class, check for the required credentials and stop if not found
    # Hopefully stops us falling over due to no AFS access of something similar
    from Ganga.Core.InternalServices import Coordinator
    missing_cred = Coordinator.getMissingCredentials()

    logger.info("Checking Credentials")

    if missing_cred:
        raise Exception("Failed due to missing credentials %s" % str(missing_cred))

    logger.info("Passing to Unittest")
Example #12
0
def bootstrap(reg, interactive_session):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config

    config.addOption('forced_shutdown_policy', 'session_type',
                     'If there are remaining background activities at exit such as monitoring, output download Ganga will attempt to wait for the activities to complete. You may select if a user is prompted to answer if he wants to force shutdown ("interactive") or if the system waits on a timeout without questions ("timeout"). The default is "session_type" which will do interactive shutdown for CLI and timeout for scripts.')

    config.addOption('forced_shutdown_timeout', 60,
                     "Timeout in seconds for forced Ganga shutdown in batch mode.")
    config.addOption('forced_shutdown_prompt_time', 10,
                     "User will get the prompt every N seconds, as specified by this parameter.")
    config.addOption('forced_shutdown_first_prompt_time', 5,
                     "User will get the FIRST prompt after N seconds, as specified by this parameter. This parameter also defines the time that Ganga will wait before shutting down, if there are only non-critical threads alive, in both interactive and batch mode.")

    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # start the internal services coordinator
    from Ganga.Core.InternalServices import Coordinator
    Coordinator.bootstrap()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    # for j in reg:
    #    if hasattr(j,'status') and j.status in ['submitted','running']:
    #        if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
    #            if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
    #                j.backend.setup()

    # start the monitoring loop
    global monitoring_component
    monitoring_component = JobRegistry_Monitor(reg)
    monitoring_component.start()

    # register the MC shutdown hook

    change_atexitPolicy(interactive_session)

    # export to GPI
    from Ganga.Runtime.GPIexport import exportToGPI
    exportToGPI(
        'runMonitoring', monitoring_component.runMonitoring, 'Functions')

    autostart_default = interactive_session
    config.overrideDefaultValue('autostart', bool(autostart_default))

    if config['autostart'] is not autostart_default:
        msg = 'monitoring loop %s (the default setting for %s session is %s)'
        val = {True: ('enabled', 'batch', 'disabled'),
               False: ('disabled', 'interactive', 'enabled')}
        logger.warning(msg % val[config['autostart']])

    if config['autostart']:
        monitoring_component.enableMonitoring()
Example #13
0
def start_ganga():

    import sys
    import os.path


    file_path = os.path.dirname(os.path.realpath(__file__))
    ganga_python_dir = os.path.join(file_path, '..', '..', '..')
    ganga_python_dir = os.path.realpath(ganga_python_dir)
    sys.path.insert(0, ganga_python_dir)

    print("Adding: %s to Python Path\n" % ganga_python_dir)
    sys.path.insert(0, ganga_python_dir)

    import Ganga.PACKAGE
    Ganga.PACKAGE.standardSetup()

    # End taken from the ganga binary

    import Ganga.Runtime
    import Ganga.Utility.logging
    logger = Ganga.Utility.logging.getLogger()

    # Start ganga by passing some options for unittesting

    logger.info("Starting ganga")

    logger.info("Parsing Command Line options")
    import Ganga.Runtime
    this_argv = [
        'ganga',  # `argv[0]` is usually the name of the program so fake that here
        '-o[Configuration]RUNTIME_PATH=GangaTest',
        '-o[Configuration]user=testframework',
        '-o[Configuration]gangadir=$HOME/gangadir_testing',
        '-o[Configuration]repositorytype=LocalXML',
        '-o[PollThread]autostart_monThreads=False',
        '-o[TestingFramework]ReleaseTesting=True',
    ]

    # FIXME Should we need to add the ability to load from a custom .ini file
    # to configure tests without editting this?

    # Actually parse the options
    Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
    Ganga.Runtime._prog.parseOptions()

    # Determine if ganga has actually finished initializing...
    # This is here to protect against the startGanga being called on an
    # initialized ganga environment
    try:
        do_config = not Ganga.Utility.Config.Config._after_bootstrap
    except:
        do_config = True

    if do_config:
        # Perform the configuration and bootstrap steps in ganga
        logger.info("Parsing Configuration Options")
        Ganga.Runtime._prog.configure()
        logger.info("Initializing")
        Ganga.Runtime._prog.initEnvironment(opt_rexec=False)
    else:
        # We need to test if the internal services need to be reinitialized
        from Ganga.Core.InternalServices import Coordinator
        if not Coordinator.servicesEnabled:
            # Start internal services
            logger.info("InternalServices restarting")

            from Ganga.GPI import reactivate
            reactivate()
        else:
            logger.info("InternalServices still running")

    logger.info("Bootstrapping")
    Ganga.Runtime._prog.bootstrap(interactive=False)

    # [PollThread]autostart_monThreads=False has turned this off being done automatically.
    # The thread pool is emptied by _ganga_run_exitfuncs
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _makeThreadPool
    _makeThreadPool()

    # Adapted from the Coordinator class, check for the required credentials and stop if not found
    # Hopefully stops us falling over due to no AFS access of something similar
    from Ganga.Core.InternalServices import Coordinator
    missing_cred = Coordinator.getMissingCredentials()

    logger.info("Checking Credentials")

    if missing_cred:
        raise Exception("Failed due to missing credentials %s" %
                        str(missing_cred))

    logger.info("Passing to Unittest")
Example #14
0
            logger.info("InternalServices restarting")

            def testing_cb(t_total, critical_thread_ids, non_critical_thread_ids):
                return True
            from Ganga.Core.GangaThread import GangaThreadPool
            thread_pool = GangaThreadPool.getInstance()
            thread_pool.shutdown(should_wait_cb=testing_cb)
            from Ganga.GPI import reactivate
            reactivate()
        else:
            logger.info("InternalServices still running")

    # Adapted from the Coordinator class, check for the required credentials and stop if not found
    # Hopefully stops us falling over due to no AFS access of something similar
    from Ganga.Core.InternalServices import Coordinator
    missing_cred = Coordinator.getMissingCredentials()

    logger.info("Checking Credentials")

    if missing_cred:
        raise Exception("Failed due to missing credentials %s" %
                        str(missing_cred))

    logger.info("Passing to Unittest")


def stopGanga():

    import Ganga.Utility.logging
    logger = Ganga.Utility.logging.getLogger()
Example #15
0
    def create(self, validity="", maxTry=0, minValidity="", check=False):
        """
        Create credential.

        Arguments other than self:
           validity    - Validity with which credential should be created,
                         specified as string of format "hh:mm"
                         [ Defaults to value of self.validityAtCreation ]
           maxTry      - Number of password attempts allowed
                         [ Defaults to value of self.maxTry ]
           minValidity - Minimum validity in case checking of
                         pre-existing credential is performed,
                         specified as strong of format "hh:mm"
                         [ Defaults to value of self.minValidity ]
           check       - Flag to request checking of pre-existing
                         credential; if flag is set to true, then new
                         credential is created only if the validity of
                         any pre-existing credential is less than the
                         value of minValidity
                         [ Default: False ] 

        Note: create is the same as renew, except for the default value of check

        Return value: True if credential is created successfully, and False
        otherwise.
        """
        global logTimeStamp

        dummy = False
        if not self.command.init:
            dummy = True
        if "valid" in self.command.init_parameters:
            if not self.command.init_parameters["valid"]:
                dummy = True

        if dummy:
            logger.warning("Dummy CommandSet used - no credential created")
            return False

        if not maxTry:
            maxTry = self.maxTry

        if not minValidity:
            minValidity = self.minValidity

        if not validity:
            validity = self.validityAtCreation

        validityInSeconds = self.timeInSeconds(validity)

        if not validityInSeconds:
            logger.warning("Problems with requested validity: %s" %
                           str(validity))
            return False
        if check and self.isValid(minValidity):
            return True

        ntry = 0

        while ntry < maxTry:

            ntry = ntry + 1
            # Test if GUI widget is to be used.
            if self.inputPW_Widget:
                # Since self.inputPW_Widget is called, current arguments are
                # ignored since renew() and create() in GUI mode will not be
                # called with any arguments.
                #proxy_obj = self._proxyObject ## This is removed to get rid of ref to _proxyObject
                proxy_obj = self
                if self.inputPW_Widget.ask(proxy_obj):
                    logger.dg(
                        "Proceeding to retrieve password from inputPW_Widget.")
                    __pw = self.inputPW_Widget.getPassword(proxy_obj)
                    if not __pw:
                        logger.warning("Password/passphrase expected!")
                        return False
                    try:
                        tFile = tempfile.NamedTemporaryFile()
                        tFile.write(__pw)
                        tFile.flush()
                    except:
                        del __pw
                        logger.warning(
                            "Could not create secure temporary file for password!"
                        )
                        return False
                    del __pw
                else:
                    # Current credential modification denied for various reasons.
                    # see GangaGUI.customDialogs.ask() method for more details.
                    return False
                # self.inputPW_Widget.ask() may have modified parameters.
                # Calling buildOpts() to take them into account.
                self.buildOpts(self.command.init, False)
                # Create initialisation list with the 'pipe' parameter
                initList = [
                    self.command.init, self.command.init_parameters["pipe"]
                ]
                # Append option value pairs
                for optName, optVal in self.command.currentOpts.iteritems():
                    initList.append("%s %s" % (optName, optVal))
                status = self.shell.system("cat %s|%s" %
                                           (tFile.name, " ".join(initList)))
                tFile.close()
                # self.inputPW_Widget dialog postprocessing.
                # E.g. disable autorenew mechanism if status != 0.
                self.inputPW_Widget.renewalStatus(proxy_obj, status)
                if status == 0:
                    logger.info("%s creation/renewal successful." % self._name)
                    return True
                else:
                    logger.warning("%s creation/renewal failed [%s]." %
                                   (self._name, status))
                    return False
            else:  # Non-GUI credential renewal/creation
                # Check if renewal is from main process (i.e. by bootstrap or
                # user)
                if threading.currentThread().getName() == 'MainThread' or\
                        threading.currentThread().getName().startswith('GANGA_Update_Thread_Ganga_Worker_'):
                    if "valid" in self.command.init_parameters:
                        self.command.currentOpts[
                            self.command.init_parameters['valid']] = validity
                    initList = [self.command.init]
                    # Append option value pairs
                    for optName, optVal in self.command.currentOpts.iteritems(
                    ):
                        initList.append("%s %s" % (optName, optVal))
                    status = self.shell.system(" ".join(initList))
                    if status == 0:
                        logger.info("%s creation/renewal successful." %
                                    self._name)
                        return True
                    else:
                        logger.warning("%s creation/renewal failed [%s]." %
                                       (self._name, status))
                # create initiated from worker thread from monitoring
                # component.
                else:
                    currTime = time.time()
                    if currTime - logTimeStamp >= logRepeatDuration:
                        logTimeStamp = currTime

                        # Check validity but print logging messages this time
                        self.isValid("", True)
                        _credentialObject = self._name[0].lower(
                        ) + self._name[1:]
                        logger.warning(
                            "Renew by typing '%s.renew()' at the prompt." %
                            (_credentialObject))

                        # notify the Core that the credential is not valid
                        _validity = self.timeInSeconds(self.timeleft())
                        _minValidity = self.timeInSeconds(minValidity) / 2.
                        if _validity <= max(120, _minValidity):
                            Coordinator.notifyInvalidCredential(self)

                    return True

        logger.warning("%s creation/renewal attempts exceeded %s tries!" %
                       (self._name, maxTry))
        return False
Example #16
0
def bootstrap(reg, interactive_session):
    """
    Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
    FIXME: this procedure should be moved to the Runtime package.

    This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
    The autostart value may be overriden in the config file, so warn if it differs from the default.
    """
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config

    config.addOption('forced_shutdown_policy','session_type','If there are remaining background activities at exit such as monitoring, output download Ganga will attempt to wait for the activities to complete. You may select if a user is prompted to answer if he wants to force shutdown ("interactive") or if the system waits on a timeout without questions ("timeout"). The default is "session_type" which will do interactive shutdown for CLI and timeout for scripts.')

    config.addOption('forced_shutdown_timeout',60,"Timeout in seconds for forced Ganga shutdown in batch mode.")
    config.addOption('forced_shutdown_prompt_time',10,"User will get the prompt every N seconds, as specified by this parameter.")
    config.addOption('forced_shutdown_first_prompt_time',5,"User will get the FIRST prompt after N seconds, as specified by this parameter. This parameter also defines the time that Ganga will wait before shutting down, if there are only non-critical threads alive, in both interactive and batch mode.")

    from Ganga.Utility.logging import getLogger

    logger = getLogger()

    from Ganga.Core.GangaThread import GangaThreadPool

    # create generic Ganga thread pool
    thread_pool = GangaThreadPool.getInstance()

    #start the internal services coordinator    
    from Ganga.Core.InternalServices import Coordinator,ShutdownManager
    Coordinator.bootstrap()
    #load the shutdown manager    
    #ShutdownManager.install()

    # backend-specific setup (e.g. Remote: setup any remote ssh pipes)
    for j in reg:
        if hasattr(j,'status') and j.status in ['submitted','running']:
            if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
                if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
                    j.backend.setup()
            
    #start the monitoring loop
    global monitoring_component
    monitoring_component = JobRegistry_Monitor( reg )
    monitoring_component.start()
    
    #register the MC shutdown hook
    import atexit

    def should_wait_interactive_cb(t_total, critical_thread_ids, non_critical_thread_ids):
        global t_last
        if t_last is None:
            t_last = -time.time()
        # if there are critical threads then prompt user or wait depending on configuration 
        if critical_thread_ids:
            if ((t_last<0 and time.time()+t_last > config['forced_shutdown_first_prompt_time']) or 
               (t_last>0 and time.time()-t_last > config['forced_shutdown_prompt_time'])):
                msg = """Job status update or output download still in progress (shutdown not completed after %d seconds).
%d background thread(s) still running: %s.
Do you want to force the exit (y/[n])? """ % (t_total, len(critical_thread_ids), critical_thread_ids) 
                resp = raw_input(msg) 
                t_last = time.time()
                return resp.lower() != 'y'
            else:
                return True
        # if there are non-critical threads then wait or shutdown depending on configuration
        elif non_critical_thread_ids:
            if t_total < config['forced_shutdown_first_prompt_time']:
                return True
            else:
                return False
        # if there are no threads then shutdown
        else:
            return False

    def should_wait_batch_cb(t_total, critical_thread_ids, non_critical_thread_ids):
        # if there are critical threads then wait or shutdown depending on configuration
        if critical_thread_ids:
            if t_total < config['forced_shutdown_timeout']:
                return True
            else:
                logger.warning('Shutdown was forced after waiting for %d seconds for background activities to finish (monitoring, output download, etc). This may result in some jobs being corrupted.',t_total)
                return False
        # if there are non-critical threads then wait or shutdown depending on configuration
        elif non_critical_thread_ids:
            if t_total < config['forced_shutdown_first_prompt_time']:
                return True
            else:
                return False
        # if there are no threads then shutdown
        else:
            return False

    #register the exit function with the highest priority (==0)    
    #atexit.register((0,monitoring_component.stop), fail_cb=mc_fail_cb,max_retries=config['max_shutdown_retries'])


    #select the shutdown method based on configuration and/or session type
    forced_shutdown_policy = config['forced_shutdown_policy']

    if forced_shutdown_policy == 'interactive':
        should_wait_cb = should_wait_interactive_cb
    else:
        if forced_shutdown_policy == 'batch':
            should_wait_cb = should_wait_batch_cb
        else:
            if interactive_session:
                should_wait_cb = should_wait_interactive_cb
            else:
                should_wait_cb = should_wait_batch_cb

    atexit.register((0,thread_pool.shutdown), should_wait_cb=should_wait_cb)

    #export to GPI
    from Ganga.Runtime.GPIexport import exportToGPI
    exportToGPI('runMonitoring',monitoring_component.runMonitoring,'Functions')     

    autostart_default = interactive_session
    config.overrideDefaultValue('autostart',bool(autostart_default))

    if config['autostart'] is not autostart_default:
        msg = 'monitoring loop %s (the default setting for %s session is %s)'
        val = { True : ('enabled', 'batch', 'disabled'), 
                False: ('disabled', 'interactive', 'enabled')}
        logger.warning(msg%val[config['autostart']])

    if config['autostart']:        
        monitoring_component.enableMonitoring()


    # THIS IS FOR DEBUGGING ONLY
    import time
    class Stuck(GangaThread.GangaThread):
        def __init__(self):
            GangaThread.GangaThread.__init__(self,name='Stuck')
        def run(self):
            i = 0
            while i < 10:
                time.sleep(3)
                #print '*'*30,i
                i += 1
        def stop(self):
            print "I was asked to stop..."