Beispiel #1
0
def parse_service_config():
    parser = optparse.OptionParser()
    options.add_options(parser)
    opts, args = parser.parse_args([])
    feat_ini = os.path.join(configure.confdir, 'feat.ini')
    local_ini = os.path.join(configure.confdir, 'local.ini')
    with open(feat_ini, 'r') as f:
        configfile.parse_file(parser, f)
    if os.path.exists(local_ini):
        with open(local_ini, 'r') as f:
            configfile.parse_file(parser, f)

    c = Config()
    c.load(os.environ, opts)
    return c
 def testDefaultConfig(self):
     parser = optparse.OptionParser()
     options_module.add_options(parser)
     options = parser.get_default_values()
     self.assertTrue(hasattr(options, 'msg_host'))
     self.assertTrue(hasattr(options, 'msg_port'))
     self.assertTrue(hasattr(options, 'msg_user'))
     self.assertTrue(hasattr(options, 'msg_password'))
     self.assertTrue(hasattr(options, 'db_host'))
     self.assertTrue(hasattr(options, 'db_port'))
     self.assertTrue(hasattr(options, 'db_name'))
     self.assertTrue(hasattr(options, 'manhole_public_key'))
     self.assertTrue(hasattr(options, 'manhole_private_key'))
     self.assertTrue(hasattr(options, 'manhole_authorized_keys'))
     self.assertTrue(hasattr(options, 'manhole_port'))
     a = agency.Agency.from_config(dict())
     self.assertEqual(a.config['msg']['host'],
                      options_module.DEFAULT_MSG_HOST)
     self.assertEqual(a.config['msg']['port'],
                      options_module.DEFAULT_MSG_PORT)
     self.assertEqual(a.config['msg']['user'],
                      options_module.DEFAULT_MSG_USER)
     self.assertEqual(a.config['msg']['password'],
                      options_module.DEFAULT_MSG_PASSWORD)
     self.assertEqual(a.config['db']['host'],
                      options_module.DEFAULT_DB_HOST)
     self.assertEqual(a.config['db']['port'],
                      options_module.DEFAULT_DB_PORT)
     self.assertEqual(a.config['db']['name'],
                      options_module.DEFAULT_DB_NAME)
     self.assertEqual(a.config['manhole']['public_key'],
                      options_module.DEFAULT_MH_PUBKEY)
     self.assertEqual(a.config['manhole']['private_key'],
                      options_module.DEFAULT_MH_PRIVKEY)
     self.assertEqual(a.config['manhole']['authorized_keys'],
                      options_module.DEFAULT_MH_AUTH)
     self.assertEqual(a.config['manhole']['port'],
                      options_module.DEFAULT_MH_PORT)
Beispiel #3
0
 def testDefaultConfig(self):
     parser = optparse.OptionParser()
     options.add_options(parser)
     defaults = parser.get_default_values()
     self.assertTrue(hasattr(defaults, 'msg_host'))
     self.assertTrue(hasattr(defaults, 'msg_port'))
     self.assertTrue(hasattr(defaults, 'msg_user'))
     self.assertTrue(hasattr(defaults, 'msg_password'))
     self.assertTrue(hasattr(defaults, 'db_host'))
     self.assertTrue(hasattr(defaults, 'db_port'))
     self.assertTrue(hasattr(defaults, 'db_name'))
     self.assertTrue(hasattr(defaults, 'manhole_public_key'))
     self.assertTrue(hasattr(defaults, 'manhole_private_key'))
     self.assertTrue(hasattr(defaults, 'manhole_authorized_keys'))
     self.assertTrue(hasattr(defaults, 'manhole_port'))
     self.assertEqual(self.config.msg.host,
                      options.DEFAULT_MSG_HOST)
     self.assertEqual(self.config.msg.port,
                      options.DEFAULT_MSG_PORT)
     self.assertEqual(self.config.msg.user,
                      options.DEFAULT_MSG_USER)
     self.assertEqual(self.config.msg.password,
                      options.DEFAULT_MSG_PASSWORD)
     self.assertEqual(self.config.db.host,
                      options.DEFAULT_DB_HOST)
     self.assertEqual(self.config.db.port,
                      options.DEFAULT_DB_PORT)
     self.assertEqual(self.config.db.name,
                      options.DEFAULT_DB_NAME)
     self.assertEqual(self.config.manhole.public_key,
                      options.DEFAULT_MH_PUBKEY)
     self.assertEqual(self.config.manhole.private_key,
                      options.DEFAULT_MH_PRIVKEY)
     self.assertEqual(self.config.manhole.authorized_keys,
                      options.DEFAULT_MH_AUTH)
     self.assertEqual(self.config.manhole.port,
                      options.DEFAULT_MH_PORT)
 def setUp(self):
     self.parser = optparse.OptionParser()
     options.add_options(self.parser)
     self.parser.parse_args(args=[])
Beispiel #5
0
def bootstrap(parser=None, args=None, descriptors=None):
    """Bootstrap a feat process, handling command line arguments.
    @param parser: the option parser to use; more options will be
        added to the parser; if not specified or None
        a new one will be created
    @type  parser: optparse.OptionParser or None
    @param args: the command line arguments to parse; if not specified
        or None, sys.argv[1:] will be used
    @type  args: [str()] or None
    @param descriptors: the descriptors of the agent to starts in addition
        of the host agent; if not specified or None
        no additional agents will be started
    @type  descriptors: [Descriptor()] or None

    @return: the deferred of the bootstrap chain
    @rtype:  defer.Deferred()"""

    tee = log.init()
    # The purpose of having log buffer here, is to be able to dump the
    # log lines to a journal after establishing connection with it.
    # This is done in stage_configure() of net agency Startup procedure.
    tee.add_keeper('buffer', log.LogBuffer(limit=10000))

    # use the resolver from twisted.names instead of the default
    # the reason for this is that ThreadedResolver behaves strangely
    # after the reconnection - raises the DNSLookupError for names
    # which have been resolved while there was no connection
    resolver.installResolver(reactor)

    if parser is None:
        parser = optparse.OptionParser()
        options.add_options(parser)
    try:
        opts, args = check_options(*parser.parse_args(args))
    except Exception as e:
        error.handle_exception('bootstrap', e, "Failed parsing config")
        sys.exit(1)

    if opts.standalone:
        cls = standalone.Agency
    else:
        cls = net_agency.Agency
    config = config_module.Config()
    config.load(os.environ, opts)
    agency = cls(config)

    applications.load('feat.agents.application', 'feat')
    applications.load('feat.gateway.application', 'featmodels')

    d = defer.Deferred()
    reactor.callWhenRunning(d.callback, None)

    if not opts.standalone:
        # specific to running normal agency

        hostdef = opts.hostdef

        if opts.hostres or opts.hostcat or opts.hostports:
            from feat.agents.common import host
            hostdef = host.HostDef()
            for resdef in opts.hostres:
                parts = resdef.split(":", 1)
                name = parts[0]
                value = 1
                if len(parts) > 1:
                    try:
                        value = int(parts[1])
                    except ValueError:
                        raise OptionError(
                            "Invalid host resource: %s" % resdef), \
                            None, sys.exc_info()[2]
                hostdef.resources[name] = value

            for catdef in opts.hostcat:
                name, value = check_category(catdef)
                hostdef.categories[name] = value

            if opts.hostports:
                hostdef.ports_ranges = dict()
            for ports in opts.hostports:
                group, start, stop = tuple(ports.split(":"))
                hostdef.ports_ranges[group] = (int(start), int(stop))

        agency.set_host_def(hostdef)

        d.addCallback(defer.drop_param, agency.initiate)
        for desc, kwargs, name in opts.agents:
            d.addCallback(defer.drop_param, agency.add_static_agent,
                          desc, kwargs, name)
    else:
        # standalone specific
        kwargs = opts.standalone_kwargs or dict()
        to_spawn = opts.agent_id or opts.agents[0][0]
        d.addCallback(defer.drop_param, agency.initiate)
        d.addCallback(defer.drop_param, agency.spawn_agent,
                      to_spawn, **kwargs)
    queue = None
    if opts.agency_daemonize:
        import multiprocessing
        queue = multiprocessing.Queue()

    d.addCallbacks(_bootstrap_success, _bootstrap_failure,
                   callbackArgs=(queue, ), errbackArgs=(agency, queue))

    if not opts.agency_daemonize:
        reactor.run()
    else:
        logname = "%s.%s.log" % ('feat', agency.agency_id)
        logfile = os.path.join(config.agency.logdir, logname)
        log.info("bootstrap", "Daemon processs will be logging to %s",
                 logfile)

        try:
            pid = os.fork()
        except OSError, e:
            sys.stderr.write("Failed to fork: (%d) %s\n" %
                             (e.errno, e.strerror))
            os._exit(1)

        if pid > 0:
            # original process waits for information about what status code
            # to use on exit
            log.info('bootstrap',
                     "Waiting for deamon process to intialize the agency")
            try:
                exit_code, reason = queue.get(timeout=20)
            except multiprocessing.queues.Empty:
                log.error('bootstrap',
                          "20 seconds timeout expires waiting for agency"
                          " in child process to initiate.")
                os._exit(1)
            else:
                log.info('bootstrap', "Process exiting with %d status",
                         exit_code)
                if exit_code:
                    log.info('bootstrap', 'Reason for failure: %s', reason)
                sys.exit(exit_code)
        else:
            # child process performs second fork
            try:
                pid = os.fork()
            except OSError, e:
                sys.stderr.write("Failed to fork: (%d) %s\n" %
                                 (e.errno, e.strerror))
                os._exit(1)
            if pid > 0:
                # child process just exits
                sys.exit(0)
            else:
                # grandchild runs the reactor and logs to an external log file
                log.FluLogKeeper.redirect_to(logfile, logfile)
                reactor.run()
Beispiel #6
0
def bootstrap(parser=None, args=None, descriptors=None, init_callback=None):
    """Bootstrap a feat process, handling command line arguments.
    @param parser: the option parser to use; more options will be
        added to the parser; if not specified or None
        a new one will be created
    @type  parser: optparse.OptionParser or None
    @param args: the command line arguments to parse; if not specified
        or None, sys.argv[1:] will be used
    @type  args: [str()] or None
    @param descriptors: the descriptors of the agent to starts in addition
        of the host agent; if not specified or None
        no additional agents will be started
    @type  descriptors: [Descriptor()] or None

    @return: the deferred of the bootstrap chain
    @rtype:  defer.Deferred()"""

    if parser is None:
        parser = optparse.OptionParser()
        options.add_options(parser)

    with _Bootstrap(parser=parser, args=args) as bootstrap:
        agency = bootstrap.agency
        opts = bootstrap.opts
        args = bootstrap.args
        opts, args = check_options(opts, args)

        if callable(init_callback):
            init_callback(agency, opts, args)

        applications.load('feat.agents.application', 'feat')
        applications.load('feat.gateway.application', 'featmodels')

        d = defer.Deferred()
        reactor.callWhenRunning(d.callback, None)

        if not opts.standalone:
            # specific to running normal agency
            if opts.force_host_restart:
                # lazy import not to load descriptor before feat is loaded
                from feat.utils import host_restart

                dbc = agency.config.db
                db = driver.Database(dbc.host, int(dbc.port), dbc.name)
                connection = db.get_connection()
                d.addCallback(defer.drop_param, host_restart.do_cleanup,
                              connection, agency._get_host_agent_id())

            hostdef = opts.hostdef

            if opts.hostres or opts.hostcat or opts.hostports:
                from feat.agents.common import host
                hostdef = host.HostDef()
                for resdef in opts.hostres:
                    parts = resdef.split(":", 1)
                    name = parts[0]
                    value = 1
                    if len(parts) > 1:
                        try:
                            value = int(parts[1])
                        except ValueError:
                            raise OptionError(
                                "Invalid host resource: %s" % resdef), \
                                None, sys.exc_info()[2]
                    hostdef.resources[name] = value

                for catdef in opts.hostcat:
                    name, value = check_category(catdef)
                    hostdef.categories[name] = value

                if opts.hostports:
                    hostdef.ports_ranges = dict()
                for ports in opts.hostports:
                    group, start, stop = tuple(ports.split(":"))
                    hostdef.ports_ranges[group] = (int(start), int(stop))

            agency.set_host_def(hostdef)

            d.addCallback(defer.drop_param, agency.initiate)
            for desc, kwargs, name in opts.agents:
                d.addCallback(defer.drop_param, agency.add_static_agent,
                              desc, kwargs, name)
        else:
            # standalone specific
            kwargs = opts.standalone_kwargs or dict()
            to_spawn = opts.agent_id or opts.agents[0][0]
            d.addCallback(defer.drop_param, agency.initiate)
            d.addCallback(defer.drop_param, agency.spawn_agent,
                          to_spawn, **kwargs)
        return d
Beispiel #7
0
def add_options(parser):
    options.add_options(parser)
Beispiel #8
0
def bootstrap(parser=None, args=None, descriptors=None):
    """Bootstrap a feat process, handling command line arguments.
    @param parser: the option parser to use; more options will be
        added to the parser; if not specified or None
        a new one will be created
    @type  parser: optparse.OptionParser or None
    @param args: the command line arguments to parse; if not specified
        or None, sys.argv[1:] will be used
    @type  args: [str()] or None
    @param descriptors: the descriptors of the agent to starts in addition
        of the host agent; if not specified or None
        no additional agents will be started
    @type  descriptors: [Descriptor()] or None
    @return: the deferred of the bootstrap chain
    @rtype:  defer.Deferred()"""

    parser = parser or optparse.OptionParser()
    add_options(parser)

    with _Bootstrap(parser=parser, args=args) as bootstrap:
        agency = bootstrap.agency
        opts = bootstrap.opts
        args = bootstrap.args
        opts, args = check_options(opts, args)

        descriptors = descriptors or []

        if not opts.standalone:
            # specific to running normal agency
            for name in opts.agents:
                factory = descriptor.lookup(name)
                if factory is None:
                    msg = "No descriptor factory found for agent %s" % name
                    raise OptionError(msg)
                descriptors.append(factory())

            hostdef = opts.hostdef

            if opts.hostres or opts.hostcat or opts.hostports:
                hostdef = host.HostDef()
                for resdef in opts.hostres:
                    parts = resdef.split(":", 1)
                    name = parts[0]
                    value = 1
                    if len(parts) > 1:
                        try:
                            value = int(parts[1])
                        except ValueError:
                            raise OptionError(
                                "Invalid host resource: %s" % resdef)
                    hostdef.resources[name] = value

                for catdef in opts.hostcat:
                    name, value = check_category(catdef)
                    hostdef.categories[name] = value

                if opts.hostports:
                    hostdef.ports_ranges = dict()
                for ports in opts.hostports:
                    group, start, stop = tuple(ports.split(":"))
                    hostdef.ports_ranges[group] = (int(start), int(stop))

            agency.set_host_def(hostdef)

            d = agency.initiate()
            for desc in descriptors:
                log.debug("feat", "Starting agent with descriptor %r", desc)
                d.addCallback(defer.drop_param, agency.spawn_agent, desc)
        else:
            # standalone specific
            kwargs = opts.standalone_kwargs or dict()
            d = agency.initiate()
            d.addCallback(defer.drop_param, agency.spawn_agent,
                          opts.agents[0], **kwargs)
        return d