def _load_application(option, opt, value, parser):
splitted = value.split('.')
if len(splitted) < 2:
raise OptionError("Invalid application name to load: %r" % (value, ))
module = '.'.join(splitted[:-1])
name = splitted[-1]
try:
applications.load(module, name)
except ImportError:
raise (
OptionError(
"Loading application %s.%s failed" % (module, name)),
None, sys.exc_info()[2])
def _parse_application_section(cfg, parser, section):
'''
Example of application section:
[application:flt]
import: flt.application
name: flt
'''
#TODO: Create an Application object here which will manage loading
#the modules and (in future) reloading them.
module = cfg.get(section, 'import')
name = cfg.get(section, 'name')
try:
applications.load(module, name)
except ImportError:
raise (ConfigParser.Error(
"Loading application %s.%s failed, requested from section %s" %
(module, name, section, )), None, sys.exc_info()[2])
def _parse_application_section(cfg, parser, section):
'''
Example of application section:
[application:flt]
import: flt.application
name: flt
# pythonpath is optional
pythonpath: /etc/flt/python
'''
try:
sys.path.append(cfg.get(section, 'pythonpath'))
except ConfigParser.NoOptionError:
pass
module = cfg.get(section, 'import')
name = cfg.get(section, 'name')
try:
applications.load(module, name)
except ImportError as e:
raise error.FeatError(
"Loading application %s.%s failed, requested from section %s" %
(module, name, section, ), cause=e)
def bootstrap(parser=None, args=None, descriptors=None):
"""Bootstrap a feat process, handling command line arguments.
@param parser: the option parser to use; more options will be
added to the parser; if not specified or None
a new one will be created
@type parser: optparse.OptionParser or None
@param args: the command line arguments to parse; if not specified
or None, sys.argv[1:] will be used
@type args: [str()] or None
@param descriptors: the descriptors of the agent to starts in addition
of the host agent; if not specified or None
no additional agents will be started
@type descriptors: [Descriptor()] or None
@return: the deferred of the bootstrap chain
@rtype: defer.Deferred()"""
tee = log.init()
# The purpose of having log buffer here, is to be able to dump the
# log lines to a journal after establishing connection with it.
# This is done in stage_configure() of net agency Startup procedure.
tee.add_keeper('buffer', log.LogBuffer(limit=10000))
# use the resolver from twisted.names instead of the default
# the reason for this is that ThreadedResolver behaves strangely
# after the reconnection - raises the DNSLookupError for names
# which have been resolved while there was no connection
resolver.installResolver(reactor)
if parser is None:
parser = optparse.OptionParser()
options.add_options(parser)
try:
opts, args = check_options(*parser.parse_args(args))
except Exception as e:
error.handle_exception('bootstrap', e, "Failed parsing config")
sys.exit(1)
if opts.standalone:
cls = standalone.Agency
else:
cls = net_agency.Agency
config = config_module.Config()
config.load(os.environ, opts)
agency = cls(config)
applications.load('feat.agents.application', 'feat')
applications.load('feat.gateway.application', 'featmodels')
d = defer.Deferred()
reactor.callWhenRunning(d.callback, None)
if not opts.standalone:
# specific to running normal agency
hostdef = opts.hostdef
if opts.hostres or opts.hostcat or opts.hostports:
from feat.agents.common import host
hostdef = host.HostDef()
for resdef in opts.hostres:
parts = resdef.split(":", 1)
name = parts[0]
value = 1
if len(parts) > 1:
try:
value = int(parts[1])
except ValueError:
raise OptionError(
"Invalid host resource: %s" % resdef), \
None, sys.exc_info()[2]
hostdef.resources[name] = value
for catdef in opts.hostcat:
name, value = check_category(catdef)
hostdef.categories[name] = value
if opts.hostports:
hostdef.ports_ranges = dict()
for ports in opts.hostports:
group, start, stop = tuple(ports.split(":"))
hostdef.ports_ranges[group] = (int(start), int(stop))
agency.set_host_def(hostdef)
d.addCallback(defer.drop_param, agency.initiate)
for desc, kwargs, name in opts.agents:
d.addCallback(defer.drop_param, agency.add_static_agent,
desc, kwargs, name)
else:
# standalone specific
kwargs = opts.standalone_kwargs or dict()
to_spawn = opts.agent_id or opts.agents[0][0]
d.addCallback(defer.drop_param, agency.initiate)
d.addCallback(defer.drop_param, agency.spawn_agent,
to_spawn, **kwargs)
queue = None
if opts.agency_daemonize:
import multiprocessing
queue = multiprocessing.Queue()
d.addCallbacks(_bootstrap_success, _bootstrap_failure,
callbackArgs=(queue, ), errbackArgs=(agency, queue))
if not opts.agency_daemonize:
reactor.run()
else:
logname = "%s.%s.log" % ('feat', agency.agency_id)
logfile = os.path.join(config.agency.logdir, logname)
log.info("bootstrap", "Daemon processs will be logging to %s",
logfile)
try:
pid = os.fork()
except OSError, e:
sys.stderr.write("Failed to fork: (%d) %s\n" %
(e.errno, e.strerror))
os._exit(1)
if pid > 0:
# original process waits for information about what status code
# to use on exit
log.info('bootstrap',
"Waiting for deamon process to intialize the agency")
try:
exit_code, reason = queue.get(timeout=20)
except multiprocessing.queues.Empty:
log.error('bootstrap',
"20 seconds timeout expires waiting for agency"
" in child process to initiate.")
os._exit(1)
else:
log.info('bootstrap', "Process exiting with %d status",
exit_code)
if exit_code:
log.info('bootstrap', 'Reason for failure: %s', reason)
sys.exit(exit_code)
else:
# child process performs second fork
try:
pid = os.fork()
except OSError, e:
sys.stderr.write("Failed to fork: (%d) %s\n" %
(e.errno, e.strerror))
os._exit(1)
if pid > 0:
# child process just exits
sys.exit(0)
else:
# grandchild runs the reactor and logs to an external log file
log.FluLogKeeper.redirect_to(logfile, logfile)
reactor.run()
def do_reload(self, value):
applications.unload(self.model.source.name)
applications.load(self.model.source.module, self.model.source.name)
return value
def bootstrap(parser=None, args=None, descriptors=None, init_callback=None):
"""Bootstrap a feat process, handling command line arguments.
@param parser: the option parser to use; more options will be
added to the parser; if not specified or None
a new one will be created
@type parser: optparse.OptionParser or None
@param args: the command line arguments to parse; if not specified
or None, sys.argv[1:] will be used
@type args: [str()] or None
@param descriptors: the descriptors of the agent to starts in addition
of the host agent; if not specified or None
no additional agents will be started
@type descriptors: [Descriptor()] or None
@return: the deferred of the bootstrap chain
@rtype: defer.Deferred()"""
if parser is None:
parser = optparse.OptionParser()
options.add_options(parser)
with _Bootstrap(parser=parser, args=args) as bootstrap:
agency = bootstrap.agency
opts = bootstrap.opts
args = bootstrap.args
opts, args = check_options(opts, args)
if callable(init_callback):
init_callback(agency, opts, args)
applications.load('feat.agents.application', 'feat')
applications.load('feat.gateway.application', 'featmodels')
d = defer.Deferred()
reactor.callWhenRunning(d.callback, None)
if not opts.standalone:
# specific to running normal agency
if opts.force_host_restart:
# lazy import not to load descriptor before feat is loaded
from feat.utils import host_restart
dbc = agency.config.db
db = driver.Database(dbc.host, int(dbc.port), dbc.name)
connection = db.get_connection()
d.addCallback(defer.drop_param, host_restart.do_cleanup,
connection, agency._get_host_agent_id())
hostdef = opts.hostdef
if opts.hostres or opts.hostcat or opts.hostports:
from feat.agents.common import host
hostdef = host.HostDef()
for resdef in opts.hostres:
parts = resdef.split(":", 1)
name = parts[0]
value = 1
if len(parts) > 1:
try:
value = int(parts[1])
except ValueError:
raise OptionError(
"Invalid host resource: %s" % resdef), \
None, sys.exc_info()[2]
hostdef.resources[name] = value
for catdef in opts.hostcat:
name, value = check_category(catdef)
hostdef.categories[name] = value
if opts.hostports:
hostdef.ports_ranges = dict()
for ports in opts.hostports:
group, start, stop = tuple(ports.split(":"))
hostdef.ports_ranges[group] = (int(start), int(stop))
agency.set_host_def(hostdef)
d.addCallback(defer.drop_param, agency.initiate)
for desc, kwargs, name in opts.agents:
d.addCallback(defer.drop_param, agency.add_static_agent,
desc, kwargs, name)
else:
# standalone specific
kwargs = opts.standalone_kwargs or dict()
to_spawn = opts.agent_id or opts.agents[0][0]
d.addCallback(defer.drop_param, agency.initiate)
d.addCallback(defer.drop_param, agency.spawn_agent,
to_spawn, **kwargs)
return d
def load_application(option, opt_str, value, parser):
splited = value.split('.')
module, name = '.'.join(splited[:-1]), splited[-1]
applications.load(module, name)
