def parse_service_config():
parser = optparse.OptionParser()
options.add_options(parser)
opts, args = parser.parse_args([])
feat_ini = os.path.join(configure.confdir, 'feat.ini')
local_ini = os.path.join(configure.confdir, 'local.ini')
with open(feat_ini, 'r') as f:
configfile.parse_file(parser, f)
if os.path.exists(local_ini):
with open(local_ini, 'r') as f:
configfile.parse_file(parser, f)
c = Config()
c.load(os.environ, opts)
return c
def testDefaultConfig(self):
parser = optparse.OptionParser()
options_module.add_options(parser)
options = parser.get_default_values()
self.assertTrue(hasattr(options, 'msg_host'))
self.assertTrue(hasattr(options, 'msg_port'))
self.assertTrue(hasattr(options, 'msg_user'))
self.assertTrue(hasattr(options, 'msg_password'))
self.assertTrue(hasattr(options, 'db_host'))
self.assertTrue(hasattr(options, 'db_port'))
self.assertTrue(hasattr(options, 'db_name'))
self.assertTrue(hasattr(options, 'manhole_public_key'))
self.assertTrue(hasattr(options, 'manhole_private_key'))
self.assertTrue(hasattr(options, 'manhole_authorized_keys'))
self.assertTrue(hasattr(options, 'manhole_port'))
a = agency.Agency.from_config(dict())
self.assertEqual(a.config['msg']['host'],
options_module.DEFAULT_MSG_HOST)
self.assertEqual(a.config['msg']['port'],
options_module.DEFAULT_MSG_PORT)
self.assertEqual(a.config['msg']['user'],
options_module.DEFAULT_MSG_USER)
self.assertEqual(a.config['msg']['password'],
options_module.DEFAULT_MSG_PASSWORD)
self.assertEqual(a.config['db']['host'],
options_module.DEFAULT_DB_HOST)
self.assertEqual(a.config['db']['port'],
options_module.DEFAULT_DB_PORT)
self.assertEqual(a.config['db']['name'],
options_module.DEFAULT_DB_NAME)
self.assertEqual(a.config['manhole']['public_key'],
options_module.DEFAULT_MH_PUBKEY)
self.assertEqual(a.config['manhole']['private_key'],
options_module.DEFAULT_MH_PRIVKEY)
self.assertEqual(a.config['manhole']['authorized_keys'],
options_module.DEFAULT_MH_AUTH)
self.assertEqual(a.config['manhole']['port'],
options_module.DEFAULT_MH_PORT)
def testDefaultConfig(self):
parser = optparse.OptionParser()
options.add_options(parser)
defaults = parser.get_default_values()
self.assertTrue(hasattr(defaults, 'msg_host'))
self.assertTrue(hasattr(defaults, 'msg_port'))
self.assertTrue(hasattr(defaults, 'msg_user'))
self.assertTrue(hasattr(defaults, 'msg_password'))
self.assertTrue(hasattr(defaults, 'db_host'))
self.assertTrue(hasattr(defaults, 'db_port'))
self.assertTrue(hasattr(defaults, 'db_name'))
self.assertTrue(hasattr(defaults, 'manhole_public_key'))
self.assertTrue(hasattr(defaults, 'manhole_private_key'))
self.assertTrue(hasattr(defaults, 'manhole_authorized_keys'))
self.assertTrue(hasattr(defaults, 'manhole_port'))
self.assertEqual(self.config.msg.host,
options.DEFAULT_MSG_HOST)
self.assertEqual(self.config.msg.port,
options.DEFAULT_MSG_PORT)
self.assertEqual(self.config.msg.user,
options.DEFAULT_MSG_USER)
self.assertEqual(self.config.msg.password,
options.DEFAULT_MSG_PASSWORD)
self.assertEqual(self.config.db.host,
options.DEFAULT_DB_HOST)
self.assertEqual(self.config.db.port,
options.DEFAULT_DB_PORT)
self.assertEqual(self.config.db.name,
options.DEFAULT_DB_NAME)
self.assertEqual(self.config.manhole.public_key,
options.DEFAULT_MH_PUBKEY)
self.assertEqual(self.config.manhole.private_key,
options.DEFAULT_MH_PRIVKEY)
self.assertEqual(self.config.manhole.authorized_keys,
options.DEFAULT_MH_AUTH)
self.assertEqual(self.config.manhole.port,
options.DEFAULT_MH_PORT)
def setUp(self):
self.parser = optparse.OptionParser()
options.add_options(self.parser)
self.parser.parse_args(args=[])
def bootstrap(parser=None, args=None, descriptors=None):
"""Bootstrap a feat process, handling command line arguments.
@param parser: the option parser to use; more options will be
added to the parser; if not specified or None
a new one will be created
@type parser: optparse.OptionParser or None
@param args: the command line arguments to parse; if not specified
or None, sys.argv[1:] will be used
@type args: [str()] or None
@param descriptors: the descriptors of the agent to starts in addition
of the host agent; if not specified or None
no additional agents will be started
@type descriptors: [Descriptor()] or None
@return: the deferred of the bootstrap chain
@rtype: defer.Deferred()"""
tee = log.init()
# The purpose of having log buffer here, is to be able to dump the
# log lines to a journal after establishing connection with it.
# This is done in stage_configure() of net agency Startup procedure.
tee.add_keeper('buffer', log.LogBuffer(limit=10000))
# use the resolver from twisted.names instead of the default
# the reason for this is that ThreadedResolver behaves strangely
# after the reconnection - raises the DNSLookupError for names
# which have been resolved while there was no connection
resolver.installResolver(reactor)
if parser is None:
parser = optparse.OptionParser()
options.add_options(parser)
try:
opts, args = check_options(*parser.parse_args(args))
except Exception as e:
error.handle_exception('bootstrap', e, "Failed parsing config")
sys.exit(1)
if opts.standalone:
cls = standalone.Agency
else:
cls = net_agency.Agency
config = config_module.Config()
config.load(os.environ, opts)
agency = cls(config)
applications.load('feat.agents.application', 'feat')
applications.load('feat.gateway.application', 'featmodels')
d = defer.Deferred()
reactor.callWhenRunning(d.callback, None)
if not opts.standalone:
# specific to running normal agency
hostdef = opts.hostdef
if opts.hostres or opts.hostcat or opts.hostports:
from feat.agents.common import host
hostdef = host.HostDef()
for resdef in opts.hostres:
parts = resdef.split(":", 1)
name = parts[0]
value = 1
if len(parts) > 1:
try:
value = int(parts[1])
except ValueError:
raise OptionError(
"Invalid host resource: %s" % resdef), \
None, sys.exc_info()[2]
hostdef.resources[name] = value
for catdef in opts.hostcat:
name, value = check_category(catdef)
hostdef.categories[name] = value
if opts.hostports:
hostdef.ports_ranges = dict()
for ports in opts.hostports:
group, start, stop = tuple(ports.split(":"))
hostdef.ports_ranges[group] = (int(start), int(stop))
agency.set_host_def(hostdef)
d.addCallback(defer.drop_param, agency.initiate)
for desc, kwargs, name in opts.agents:
d.addCallback(defer.drop_param, agency.add_static_agent,
desc, kwargs, name)
else:
# standalone specific
kwargs = opts.standalone_kwargs or dict()
to_spawn = opts.agent_id or opts.agents[0][0]
d.addCallback(defer.drop_param, agency.initiate)
d.addCallback(defer.drop_param, agency.spawn_agent,
to_spawn, **kwargs)
queue = None
if opts.agency_daemonize:
import multiprocessing
queue = multiprocessing.Queue()
d.addCallbacks(_bootstrap_success, _bootstrap_failure,
callbackArgs=(queue, ), errbackArgs=(agency, queue))
if not opts.agency_daemonize:
reactor.run()
else:
logname = "%s.%s.log" % ('feat', agency.agency_id)
logfile = os.path.join(config.agency.logdir, logname)
log.info("bootstrap", "Daemon processs will be logging to %s",
logfile)
try:
pid = os.fork()
except OSError, e:
sys.stderr.write("Failed to fork: (%d) %s\n" %
(e.errno, e.strerror))
os._exit(1)
if pid > 0:
# original process waits for information about what status code
# to use on exit
log.info('bootstrap',
"Waiting for deamon process to intialize the agency")
try:
exit_code, reason = queue.get(timeout=20)
except multiprocessing.queues.Empty:
log.error('bootstrap',
"20 seconds timeout expires waiting for agency"
" in child process to initiate.")
os._exit(1)
else:
log.info('bootstrap', "Process exiting with %d status",
exit_code)
if exit_code:
log.info('bootstrap', 'Reason for failure: %s', reason)
sys.exit(exit_code)
else:
# child process performs second fork
try:
pid = os.fork()
except OSError, e:
sys.stderr.write("Failed to fork: (%d) %s\n" %
(e.errno, e.strerror))
os._exit(1)
if pid > 0:
# child process just exits
sys.exit(0)
else:
# grandchild runs the reactor and logs to an external log file
log.FluLogKeeper.redirect_to(logfile, logfile)
reactor.run()
def bootstrap(parser=None, args=None, descriptors=None, init_callback=None):
"""Bootstrap a feat process, handling command line arguments.
@param parser: the option parser to use; more options will be
added to the parser; if not specified or None
a new one will be created
@type parser: optparse.OptionParser or None
@param args: the command line arguments to parse; if not specified
or None, sys.argv[1:] will be used
@type args: [str()] or None
@param descriptors: the descriptors of the agent to starts in addition
of the host agent; if not specified or None
no additional agents will be started
@type descriptors: [Descriptor()] or None
@return: the deferred of the bootstrap chain
@rtype: defer.Deferred()"""
if parser is None:
parser = optparse.OptionParser()
options.add_options(parser)
with _Bootstrap(parser=parser, args=args) as bootstrap:
agency = bootstrap.agency
opts = bootstrap.opts
args = bootstrap.args
opts, args = check_options(opts, args)
if callable(init_callback):
init_callback(agency, opts, args)
applications.load('feat.agents.application', 'feat')
applications.load('feat.gateway.application', 'featmodels')
d = defer.Deferred()
reactor.callWhenRunning(d.callback, None)
if not opts.standalone:
# specific to running normal agency
if opts.force_host_restart:
# lazy import not to load descriptor before feat is loaded
from feat.utils import host_restart
dbc = agency.config.db
db = driver.Database(dbc.host, int(dbc.port), dbc.name)
connection = db.get_connection()
d.addCallback(defer.drop_param, host_restart.do_cleanup,
connection, agency._get_host_agent_id())
hostdef = opts.hostdef
if opts.hostres or opts.hostcat or opts.hostports:
from feat.agents.common import host
hostdef = host.HostDef()
for resdef in opts.hostres:
parts = resdef.split(":", 1)
name = parts[0]
value = 1
if len(parts) > 1:
try:
value = int(parts[1])
except ValueError:
raise OptionError(
"Invalid host resource: %s" % resdef), \
None, sys.exc_info()[2]
hostdef.resources[name] = value
for catdef in opts.hostcat:
name, value = check_category(catdef)
hostdef.categories[name] = value
if opts.hostports:
hostdef.ports_ranges = dict()
for ports in opts.hostports:
group, start, stop = tuple(ports.split(":"))
hostdef.ports_ranges[group] = (int(start), int(stop))
agency.set_host_def(hostdef)
d.addCallback(defer.drop_param, agency.initiate)
for desc, kwargs, name in opts.agents:
d.addCallback(defer.drop_param, agency.add_static_agent,
desc, kwargs, name)
else:
# standalone specific
kwargs = opts.standalone_kwargs or dict()
to_spawn = opts.agent_id or opts.agents[0][0]
d.addCallback(defer.drop_param, agency.initiate)
d.addCallback(defer.drop_param, agency.spawn_agent,
to_spawn, **kwargs)
return d
def add_options(parser):
options.add_options(parser)
def bootstrap(parser=None, args=None, descriptors=None):
"""Bootstrap a feat process, handling command line arguments.
@param parser: the option parser to use; more options will be
added to the parser; if not specified or None
a new one will be created
@type parser: optparse.OptionParser or None
@param args: the command line arguments to parse; if not specified
or None, sys.argv[1:] will be used
@type args: [str()] or None
@param descriptors: the descriptors of the agent to starts in addition
of the host agent; if not specified or None
no additional agents will be started
@type descriptors: [Descriptor()] or None
@return: the deferred of the bootstrap chain
@rtype: defer.Deferred()"""
parser = parser or optparse.OptionParser()
add_options(parser)
with _Bootstrap(parser=parser, args=args) as bootstrap:
agency = bootstrap.agency
opts = bootstrap.opts
args = bootstrap.args
opts, args = check_options(opts, args)
descriptors = descriptors or []
if not opts.standalone:
# specific to running normal agency
for name in opts.agents:
factory = descriptor.lookup(name)
if factory is None:
msg = "No descriptor factory found for agent %s" % name
raise OptionError(msg)
descriptors.append(factory())
hostdef = opts.hostdef
if opts.hostres or opts.hostcat or opts.hostports:
hostdef = host.HostDef()
for resdef in opts.hostres:
parts = resdef.split(":", 1)
name = parts[0]
value = 1
if len(parts) > 1:
try:
value = int(parts[1])
except ValueError:
raise OptionError(
"Invalid host resource: %s" % resdef)
hostdef.resources[name] = value
for catdef in opts.hostcat:
name, value = check_category(catdef)
hostdef.categories[name] = value
if opts.hostports:
hostdef.ports_ranges = dict()
for ports in opts.hostports:
group, start, stop = tuple(ports.split(":"))
hostdef.ports_ranges[group] = (int(start), int(stop))
agency.set_host_def(hostdef)
d = agency.initiate()
for desc in descriptors:
log.debug("feat", "Starting agent with descriptor %r", desc)
d.addCallback(defer.drop_param, agency.spawn_agent, desc)
else:
# standalone specific
kwargs = opts.standalone_kwargs or dict()
d = agency.initiate()
d.addCallback(defer.drop_param, agency.spawn_agent,
opts.agents[0], **kwargs)
return d