def test_value_inheritance(self):
option_list = [
options.Option('-a', dest='a'),
options.Option('-b', dest='b')
]
values, leftovers = options.parser().options(option_list).parse([])
assert not hasattr(values, 'a')
assert not hasattr(values, 'b')
# w/ option
values, leftovers = options.parser().options(option_list).parse(
['-a', 'value_a'])
assert hasattr(values, 'a')
assert values.a == 'value_a'
assert not hasattr(values, 'b')
# w/ inherited option
values, leftovers = options.parser().values(values).options(
option_list).parse(['-b', 'value_b'])
assert values.a == 'value_a'
assert values.b == 'value_b'
# w/ inherits w/o parsing any new args
values, leftovers = options.parser().values(values).options(
option_list).parse([])
assert values.a == 'value_a'
assert values.b == 'value_b'
# w/ overwrites despite inheriting
values, leftovers = options.parser().values(values).options(
option_list).parse(['-a', 'new_value_a'])
assert values.a == 'new_value_a'
assert values.b == 'value_b'
def test_multiple_option_inheritance(self):
option_a = options.Option('-a', dest='a')
option_b = options.Option('-b', dest='b')
values, leftovers = (options.parser().options([option_a]).options(
[option_b])).parse(['-a', 'value_a', '-b', 'value_b'])
assert values.a == 'value_a'
assert values.b == 'value_b'
class VarsSubsystem(app.Module):
"""
Exports a /vars endpoint on the root http server bound to twitter.common.metrics.RootMetrics.
"""
OPTIONS = {
'sampling_delay':
options.Option(
'--vars-sampling-delay-ms',
default=1000,
type='int',
metavar='MILLISECONDS',
dest='twitter_common_metrics_vars_sampling_delay_ms',
help='How long between taking samples of the vars subsystem.'),
'trace_endpoints':
options.Option(
'--vars-trace-endpoints',
'--no-vars-trace-endpoints',
default=True,
action='callback',
callback=set_bool,
dest='twitter_common_app_modules_varz_trace_endpoints',
help='Trace all registered http endpoints in this application.'),
'trace_namespace':
options.Option('--trace-namespace',
default='http',
dest='twitter_common_app_modules_varz_trace_namespace',
help='The prefix for http request metrics.')
}
def __init__(self):
app.Module.__init__(self,
__name__,
description='Vars subsystem',
dependencies='twitter.common.app.modules.http')
def setup_function(self):
options = app.get_options()
rs = RootServer()
if rs:
varz = VarsEndpoint(period=Amount(
options.twitter_common_metrics_vars_sampling_delay_ms,
Time.MILLISECONDS))
rs.mount_routes(varz)
register_diagnostics()
register_build_properties()
if options.twitter_common_app_modules_varz_trace_endpoints:
plugin = EndpointTracePlugin()
rs.install(plugin)
RootMetrics().register_observable(
options.twitter_common_app_modules_varz_trace_namespace,
plugin)
def test_app_add_options_with_Option(self):
# options.Option
opt = options.Option('--option1', dest='option1')
app.add_option(opt)
app.init(force_args=['--option1', 'option1value', 'extraargs'])
assert app.get_options().option1 == 'option1value'
assert app.argv() == ['extraargs']
class VarsSubsystem(app.Module):
"""
Exports a /vars endpoint on the root http server bound to twitter.common.metrics.RootMetrics.
"""
OPTIONS = {
'sampling_delay':
options.Option(
'--vars_sampling_delay_ms',
default=1000,
type='int',
metavar='MILLISECONDS',
dest='twitter_common_metrics_vars_sampling_delay_ms',
help='How long between taking samples of the vars subsystem.')
}
def __init__(self):
app.Module.__init__(self,
__name__,
description="Vars subsystem",
dependencies='twitter.common.app.modules.http')
def setup_function(self):
options = app.get_options()
rs = RootServer()
if rs:
varz = VarsEndpoint(period=Amount(
options.twitter_common_metrics_vars_sampling_delay_ms,
Time.MILLISECONDS))
rs.mount_routes(varz)
register_diagnostics()
register_build_properties()
def test_basic_parsing(self):
option = options.Option('-m', '--my_option', dest='my_option')
# w/o option
values, leftovers = options.parser().options([option]).parse([])
assert not hasattr(values, 'my_option')
assert leftovers == []
# w/ option
values, leftovers = options.parser().options([option
]).parse(['-m', 'poop'])
assert values.my_option == 'poop'
assert leftovers == []
# w/ long option
values, leftovers = options.parser().options([option]).parse(
['--my_option', 'plork'])
assert values.my_option == 'plork'
assert leftovers == []
# w/ option and leftover
values, leftovers = options.parser().options([option]).parse(
['--my_option', 'plork', 'hork'])
assert values.my_option == 'plork'
assert leftovers == ['hork']
def test_default_parsing(self):
option = options.Option('-m',
'--my_option',
default="specified",
dest='my_option')
values, leftovers = options.parser().options([option]).parse([])
assert hasattr(values, 'my_option')
assert leftovers == []
assert values.my_option == 'specified'
def test_multiple_value_inheritance(self):
option_list = [
options.Option('-a', dest='a'),
options.Option('-b', dest='b')
]
values_with_a, _ = options.parser().options(option_list).parse(
['-a', 'value_a'])
values_with_b, _ = options.parser().options(option_list).parse(
['-b', 'value_b'])
values, leftovers = (options.parser().options(option_list).values(
values_with_a).values(values_with_b)).parse([])
assert values.a == 'value_a'
assert values.b == 'value_b'
# and parsed values overwrite
values, leftovers = (options.parser().options(option_list).values(
values_with_a).values(values_with_b)).parse(['-a', 'new_value_a'])
assert values.a == 'new_value_a'
assert values.b == 'value_b'
def test_groups(self):
option_a = options.Option('-a', dest='a')
option_b = options.Option('-b', dest='b')
option_group_a = options.group('a')
option_group_b = options.group('b')
option_group_a.add_option(options.Option('--a1', dest='a1'),
options.Option('--a2', dest='a2'))
option_group_b.add_option(options.Option('--b1', dest='b1'),
options.Option('--b2', dest='b2'))
partial_parser = (options.parser().interspersed_arguments(True).groups(
[option_group_a, option_group_b]))
full_parser = partial_parser.options([option_a, option_b])
parameters = [
'--a1', 'value_a1', '--a2', 'value_a2', '--b1', 'value_b1', '--b2',
'value_b2'
]
full_parameters = parameters + ['-a', 'value_a', '-b', 'value_b']
values, leftovers = partial_parser.parse(parameters)
assert values.a1 == 'value_a1'
assert values.a2 == 'value_a2'
assert values.b1 == 'value_b1'
assert values.b2 == 'value_b2'
assert leftovers == []
values, leftovers = full_parser.parse(full_parameters)
assert values.a1 == 'value_a1'
assert values.a2 == 'value_a2'
assert values.b1 == 'value_b1'
assert values.b2 == 'value_b2'
assert values.a == 'value_a'
assert values.b == 'value_b'
assert leftovers == []
class Application(object):
class Error(Exception):
pass
# enforce a quasi-singleton interface (for resettable applications in test)
_Global = None
@staticmethod
def reset():
"""Reset the global application. Only useful for testing."""
Application._Global = Application()
@staticmethod
def active():
"""Return the current resident application object."""
return Application._Global
HELP_OPTIONS = [
options.Option("-h",
"--help",
"--short-help",
action="callback",
callback=lambda *args, **kwargs: Application.active().
_short_help(*args, **kwargs),
help="show this help message and exit."),
options.Option(
"--long-help",
action="callback",
callback=lambda *args, **kwargs: Application.active()._long_help(
*args, **kwargs),
help=
"show options from all registered modules, not just the __main__ module."
)
]
IGNORE_RC_FLAG = '--app_ignore_rc_file'
APP_OPTIONS = {
'daemonize':
options.Option('--app_daemonize',
action='store_true',
default=False,
dest='twitter_common_app_daemonize',
help="Daemonize this application."),
'daemon_stdout':
options.Option(
'--app_daemon_stdout',
default='/dev/null',
dest='twitter_common_app_daemon_stdout',
help="Direct this app's stdout to this file if daemonized."),
'daemon_stderr':
options.Option(
'--app_daemon_stderr',
default='/dev/null',
dest='twitter_common_app_daemon_stderr',
help="Direct this app's stderr to this file if daemonized."),
'pidfile':
options.Option(
'--app_pidfile',
default=None,
dest='twitter_common_app_pidfile',
help="The pidfile to use if --app_daemonize is specified."),
'debug':
options.Option(
'--app_debug',
action='store_true',
default=False,
dest='twitter_common_app_debug',
help=
"Print extra debugging information during application initialization."
),
'profiling':
options.Option(
'--app_profiling',
action='store_true',
default=False,
dest='twitter_common_app_profiling',
help=
"Run profiler on the code while it runs. Note this can cause slowdowns."
),
'profile_output':
options.Option(
'--app_profile_output',
default=None,
metavar='FILENAME',
dest='twitter_common_app_profile_output',
help="Dump the profiling output to a binary profiling format."),
'rc_filename':
options.Option('--app_rc_filename',
action='store_true',
default=False,
dest='twitter_common_app_rc_filename',
help="Print the filename for the rc file and quit."),
'ignore_rc_file':
options.Option(IGNORE_RC_FLAG,
action='store_true',
default=False,
dest='twitter_common_app_ignore_rc_file',
help="Ignore default arguments from the rc file."),
}
NO_COMMAND = 'DEFAULT'
OPTIONS = 'options'
OPTIONS_ATTR = '__options__'
def __init__(self):
self._name = None
self._registered_modules = []
self._init_modules = []
self._option_targets = defaultdict(dict)
self._global_options = {}
self._interspersed_args = False
self._main_options = Application.HELP_OPTIONS[:]
self._usage = ""
self._profiler = None
self._commands = {}
self._reset()
for opt in Application.APP_OPTIONS.values():
self.add_option(opt)
self._configure_options(None, Application.APP_OPTIONS)
def _raise_if_initialized(
self, msg="Cannot perform operation after initialization!"):
if self.initialized:
raise Application.Error(msg)
def _raise_if_uninitialized(
self, msg="Cannot perform operation before initialization!"):
if not self.initialized:
raise Application.Error(msg)
def _reset(self):
"""
Resets the state set up by init() so that init() may be called again.
"""
self.initialized = False
self._option_values = options.Values()
self._argv = []
def interspersed_args(self, value):
self._interspersed_args = bool(value)
def _configure_options(self, module, option_dict):
for opt_name, opt in option_dict.items():
self._option_targets[module][opt_name] = opt.dest
def configure(self, module=None, **kw):
"""
Configure the application object or its activated modules.
Typically application modules export flags that can be defined on the
command-line. In order to allow the application to override defaults,
these modules may export named parameters to be overridden. For example,
the Application object itself exports named variables such as "debug" or
"profiling", which can be enabled via:
app.configure(debug=True)
and
app.configure(profiling=True)
respectively. They can also be enabled with their command-line argument
counterpart, e.g.
./my_application --app_debug --app_profiling
Some modules export named options, e.g. twitter.common.app.modules.http exports
'enable', 'host', 'port'. The command-line arguments still take precedence and
will override any defaults set by the application in app.configure. To activate
these options, just pass along the module name:
app.configure(module='twitter.common.app.modules.http', enable=True)
"""
if module not in self._option_targets:
if not self._import_module(module):
raise Application.Error('Unknown module to configure: %s' %
module)
def configure_option(name, value):
if name not in self._option_targets[module]:
raise Application.Error('Module %s has no option %s' %
(module, name))
self.set_option(self._option_targets[module][name], value)
for option_name, option_value in kw.items():
configure_option(option_name, option_value)
def _main_parser(self):
return (options.parser().interspersed_arguments(
self._interspersed_args).options(self._main_options).usage(
self._usage))
def command_parser(self, command):
assert command in self._commands
values_copy = copy.deepcopy(self._option_values)
parser = self._main_parser()
command_group = options.new_group(('For %s only' %
command) if command else 'Default')
for option in getattr(self._commands[command],
Application.OPTIONS_ATTR):
op = copy.deepcopy(option)
if not hasattr(values_copy, op.dest):
setattr(
values_copy, op.dest,
op.default if op.default != optparse.NO_DEFAULT else None)
Application.rewrite_help(op)
op.default = optparse.NO_DEFAULT
command_group.add_option(op)
parser = parser.groups([command_group]).values(values_copy)
usage = self._commands[command].__doc__
if usage:
parser = parser.usage(usage)
return parser
def _construct_partial_parser(self):
"""
Construct an options parser containing only options added by __main__
or global help options registered by the application.
"""
if hasattr(self._commands.get(self._command),
Application.OPTIONS_ATTR):
return self.command_parser(self._command)
else:
return self._main_parser().values(
copy.deepcopy(self._option_values))
def _construct_full_parser(self):
"""
Construct an options parser containing both local and global (module-level) options.
"""
return self._construct_partial_parser().groups(
self._global_options.values())
def _rc_filename(self):
rc_short_filename = '~/.%src' % self.name()
return os.path.expanduser(rc_short_filename)
def _add_default_options(self, argv):
"""
Return an argument list with options from the rc file prepended.
"""
rc_filename = self._rc_filename()
options = argv
if Application.IGNORE_RC_FLAG not in argv and os.path.exists(
rc_filename):
command = self._command or Application.NO_COMMAND
rc_config = ConfigParser.SafeConfigParser()
rc_config.read(rc_filename)
if rc_config.has_option(command, Application.OPTIONS):
default_options_str = rc_config.get(command,
Application.OPTIONS)
default_options = shlex.split(default_options_str, True)
options = default_options + options
return options
def _parse_options(self, force_args=None):
"""
Parse options and set self.option_values and self.argv to the values to be passed into
the application's main() method.
"""
argv = sys.argv[1:] if force_args is None else force_args
if argv and argv[0] in self._commands:
self._command = argv.pop(0)
else:
self._command = None
parser = self._construct_full_parser()
self._option_values, self._argv = parser.parse(
self._add_default_options(argv))
def _short_help(self, option, opt, value, parser):
self._construct_partial_parser().print_help()
sys.exit(1)
def _long_help(self, option, opt, value, parser):
self._construct_full_parser().print_help()
sys.exit(1)
def _setup_modules(self):
"""
Setup all initialized modules.
"""
module_registry = AppModule.module_registry()
for bundle in topological_sort(AppModule.module_dependencies()):
for module_label in bundle:
assert module_label in module_registry
module = module_registry[module_label]
self._debug_log('Initializing: %s (%s)' %
(module.label(), module.description()))
try:
module.setup_function()
except AppModule.Unimplemented:
pass
self._init_modules.append(module.label())
def _teardown_modules(self):
"""
Teardown initialized module in reverse initialization order.
"""
module_registry = AppModule.module_registry()
for module_label in reversed(self._init_modules):
assert module_label in module_registry
module = module_registry[module_label]
self._debug_log('Running exit function for %s (%s)' %
(module_label, module.description()))
try:
module.teardown_function()
except AppModule.Unimplemented:
pass
def _maybe_daemonize(self):
if self._option_values.twitter_common_app_daemonize:
daemonize(
pidfile=self._option_values.twitter_common_app_pidfile,
stdout=self._option_values.twitter_common_app_daemon_stdout,
stderr=self._option_values.twitter_common_app_daemon_stderr)
# ------- public exported methods -------
def init(self, force_args=None):
"""
Initialize the state necessary to run the application's main() function but
without actually invoking main. Mostly useful for testing. If force_args
specified, use those arguments instead of sys.argv[1:].
"""
self._raise_if_initialized(
"init cannot be called twice. Use reinit if necessary.")
self._parse_options(force_args)
self._maybe_daemonize()
self._setup_modules()
self.initialized = True
def reinit(self, force_args=None):
"""
Reinitialize the application. This clears the stateful parts of the application
framework and reruns init(). Mostly useful for testing.
"""
self._reset()
self.init(force_args)
def argv(self):
self._raise_if_uninitialized(
"Must call app.init() before you may access argv.")
return self._argv
def add_module_path(self, name, path):
"""
Add all app.Modules defined by name at path.
Typical usage (e.g. from the __init__.py of something containing many
app modules):
app.add_module_path(__name__, __path__)
"""
import pkgutil
for _, mod, ispkg in pkgutil.iter_modules(path):
if ispkg: continue
fq_module = '.'.join([name, mod])
__import__(fq_module)
for (kls_name, kls) in inspect.getmembers(sys.modules[fq_module],
inspect.isclass):
if issubclass(kls, AppModule):
self.register_module(kls())
def register_module(self, module):
"""
Register an app.Module and all its options.
"""
if not isinstance(module, AppModule):
raise TypeError(
'register_module should be called with a subclass of AppModule'
)
if module.label() in self._registered_modules:
# Do not reregister.
return
if hasattr(module, 'OPTIONS'):
if not isinstance(module.OPTIONS, dict):
raise Application.Error(
'Registered app.Module %s has invalid OPTIONS.' %
module.__module__)
for opt in module.OPTIONS.values():
self._add_option(module.__module__, opt)
self._configure_options(module.label(), module.OPTIONS)
self._registered_modules.append(module.label())
@staticmethod
def _get_module_key(module):
return 'From module %s' % module
def _add_main_option(self, option):
self._main_options.append(option)
def _add_module_option(self, module, option):
calling_module = Application._get_module_key(module)
if calling_module not in self._global_options:
self._global_options[calling_module] = options.new_group(
calling_module)
self._global_options[calling_module].add_option(option)
@staticmethod
def rewrite_help(op):
if hasattr(op, 'help') and isinstance(op.help, Compatibility.string):
if op.help.find(
'%default') != -1 and op.default != optparse.NO_DEFAULT:
op.help = op.help.replace('%default', str(op.default))
else:
op.help = op.help + ((' [default: %s]' % str(op.default)) if
op.default != optparse.NO_DEFAULT else '')
def _add_option(self, calling_module, option):
op = copy.deepcopy(option)
if op.dest and hasattr(op, 'default'):
self.set_option(
op.dest,
op.default if op.default != optparse.NO_DEFAULT else None,
force=False)
Application.rewrite_help(op)
op.default = optparse.NO_DEFAULT
if calling_module == '__main__':
self._add_main_option(op)
else:
self._add_module_option(calling_module, op)
def _get_option_from_args(self, args, kwargs):
if len(args) == 1 and kwargs == {} and isinstance(
args[0], options.Option):
return args[0]
else:
return options.TwitterOption(*args, **kwargs)
def add_option(self, *args, **kwargs):
"""
Add an option to the application.
You may pass either an Option object from the optparse/options module, or
pass the *args/**kwargs necessary to construct an Option.
"""
self._raise_if_initialized("Cannot call add_option() after main()!")
calling_module = Inspection.find_calling_module()
added_option = self._get_option_from_args(args, kwargs)
self._add_option(calling_module, added_option)
def command(self, function=None, name=None):
"""
Decorator to turn a function into an application command.
To add a command foo, the following patterns will both work:
@app.command
def foo(args, options):
...
@app.command(name='foo')
def bar(args, options):
...
"""
if name is None:
return self._register_command(function)
else:
return partial(self._register_command, command_name=name)
def _register_command(self, function, command_name=None):
"""
Registers function as the handler for command_name. Uses function.__name__ if command_name
is None.
"""
if Inspection.find_calling_module() == '__main__':
if command_name is None:
command_name = function.__name__
if command_name in self._commands:
raise Application.Error(
'Found two definitions for command %s' % command_name)
self._commands[command_name] = function
return function
def default_command(self, function):
"""
Decorator to make a command default.
"""
if Inspection.find_calling_module() == '__main__':
if None in self._commands:
defaults = (self._commands[None].__name__, function.__name__)
raise Application.Error(
'Found two default commands: %s and %s' % defaults)
self._commands[None] = function
return function
def command_option(self, *args, **kwargs):
"""
Decorator to add an option only for a specific command.
"""
def register_option(function):
added_option = self._get_option_from_args(args, kwargs)
if not hasattr(function, Application.OPTIONS_ATTR):
setattr(function, Application.OPTIONS_ATTR, deque())
getattr(function,
Application.OPTIONS_ATTR).appendleft(added_option)
return function
return register_option
def copy_command_options(self, command_function):
"""
Decorator to copy command options from another command.
"""
def register_options(function):
if hasattr(command_function, Application.OPTIONS_ATTR):
if not hasattr(function, Application.OPTIONS_ATTR):
setattr(function, Application.OPTIONS_ATTR, deque())
command_options = getattr(command_function,
Application.OPTIONS_ATTR)
getattr(function,
Application.OPTIONS_ATTR).extendleft(command_options)
return function
return register_options
def add_command_options(self, command_function):
"""
Function to add all options from a command
"""
module = inspect.getmodule(command_function).__name__
for option in getattr(command_function, Application.OPTIONS_ATTR, []):
self._add_option(module, option)
def _debug_log(self, msg):
if hasattr(self._option_values, 'twitter_common_app_debug') and (
self._option_values.twitter_common_app_debug):
print('twitter.common.app debug: %s' % msg, file=sys.stderr)
def set_option(self, dest, value, force=True):
"""
Set a global option value either pre- or post-initialization.
If force=False, do not set the default if already overridden by a manual call to
set_option.
"""
if hasattr(self._option_values, dest) and not force:
return
setattr(self._option_values, dest, value)
def get_options(self):
"""
Return all application options, both registered by __main__ and all imported modules.
"""
return self._option_values
def get_commands(self):
"""
Return all valid commands registered by __main__
"""
return filter(None, self._commands.keys())
def get_commands_and_docstrings(self):
"""
Generate all valid commands together with their docstrings
"""
for command, function in self._commands.items():
if command is not None:
yield command, function.__doc__
def get_local_options(self):
"""
Return the options only defined by __main__.
"""
new_values = options.Values()
for opt in self._main_options:
if opt.dest:
setattr(new_values, opt.dest,
getattr(self._option_values, opt.dest))
return new_values
def set_usage(self, usage):
"""
Set the usage message should the user call --help or invalidly specify options.
"""
self._usage = usage
def error(self, message):
"""
Print the application help message, an error message, then exit.
"""
self._construct_partial_parser().error(message)
def help(self):
"""
Print the application help message and exit.
"""
self._short_help(*(None, ) * 4)
def set_name(self, application_name):
"""
Set the application name. (Autodetect otherwise.)
"""
self._raise_if_initialized("Cannot set application name.")
self._name = application_name
def name(self):
"""
Return the name of the application. If set_name was never explicitly called,
the application framework will attempt to autodetect the name of the application
based upon the location of __main__.
"""
if self._name is not None:
return self._name
else:
try:
return Inspection.find_application_name()
except:
return 'unknown'
def quit(self, rc, exit_function=sys.exit):
self._debug_log('Shutting application down.')
self._teardown_modules()
self._debug_log('Finishing up module teardown.')
nondaemons = 0
self.dump_profile()
for thr in threading.enumerate():
self._debug_log(' Active thread%s: %s' %
(' (daemon)' if thr.isDaemon() else '', thr))
if thr is not threading.current_thread() and not thr.isDaemon():
nondaemons += 1
if nondaemons:
self._debug_log(
'More than one active non-daemon thread, your application may hang!'
)
else:
self._debug_log('Exiting cleanly.')
exit_function(rc)
def profiler(self):
if self._option_values.twitter_common_app_profiling:
if self._profiler is None:
try:
import cProfile as profile
except ImportError:
import profile
self._profiler = profile.Profile()
return self._profiler
else:
return None
def dump_profile(self):
if self._option_values.twitter_common_app_profiling:
if self._option_values.twitter_common_app_profile_output:
self.profiler().dump_stats(
self._option_values.twitter_common_app_profile_output)
else:
self.profiler().print_stats(sort='time')
def _run_main(self, main_method, *args, **kwargs):
try:
if self.profiler():
rc = self.profiler().runcall(main_method, *args, **kwargs)
else:
rc = main_method(*args, **kwargs)
except SystemExit as e:
rc = e.code
self._debug_log('main_method exited with return code = %s' %
repr(rc))
except KeyboardInterrupt as e:
rc = None
self._debug_log('main_method exited with ^C')
return rc
def _import_module(self, name):
"""
Import the module, return True on success, False if the import failed.
"""
try:
__import__(name)
return True
except ImportError:
return False
def main(self):
"""
If called from __main__ module, run script's main() method with arguments passed
and global options parsed.
The following patterns are acceptable for the main method:
main()
main(args)
main(args, options)
"""
main_module = Inspection.find_calling_module()
if main_module != '__main__':
# only support if __name__ == '__main__'
return
# Pull in modules in twitter.common.app.modules
if not self._import_module('twitter.common.app.modules'):
print('Unable to import twitter app modules!', file=sys.stderr)
sys.exit(1)
# defer init as long as possible.
self.init()
if self._option_values.twitter_common_app_rc_filename:
print('RC filename: %s' % self._rc_filename())
return
try:
caller_main = Inspection.find_main_from_caller()
except Inspection.InternalError:
caller_main = None
if None in self._commands:
assert caller_main is None, "Error: Cannot define both main and a default command."
else:
self._commands[None] = caller_main
main_method = self._commands[self._command]
if main_method is None:
commands = sorted(self.get_commands())
if commands:
print('Must supply one of the following commands:',
', '.join(commands),
file=sys.stderr)
else:
print(
'No main() or command defined! Application must define one of these.',
file=sys.stderr)
sys.exit(1)
try:
argspec = inspect.getargspec(main_method)
except TypeError as e:
print('Malformed main(): %s' % e, file=sys.stderr)
sys.exit(1)
if len(argspec.args) == 1:
args = [self._argv]
elif len(argspec.args) == 2:
args = [self._argv, self._option_values]
else:
if len(self._argv) != 0:
print(
'main() takes no arguments but got leftover arguments: %s!'
% ' '.join(self._argv),
file=sys.stderr)
sys.exit(1)
args = []
rc = self._run_main(main_method, *args)
self.quit(rc)
class AppScribeExceptionHandler(app.Module):
"""
An application module that logs or scribes uncaught exceptions.
"""
OPTIONS = {
'port':
options.Option('--scribe_exception_port',
default=1463,
type='int',
metavar='PORT',
dest='twitter_common_scribe_port',
help='The port on which scribe aggregator listens.'),
'host':
options.Option('--scribe_exception_host',
default='localhost',
type='string',
metavar='HOSTNAME',
dest='twitter_common_scribe_host',
help='The host to which scribe exceptions should be written.'),
'category':
options.Option('--scribe_exception_category',
default='python_default',
type='string',
metavar='CATEGORY',
dest='twitter_common_scribe_category',
help='The scribe category into which we write exceptions.')
}
def __init__(self):
app.Module.__init__(self, __name__, description="twitter.common.log handler.")
def setup_function(self):
self._builtin_hook = sys.excepthook
def forwarding_handler(*args, **kw):
AppScribeExceptionHandler.scribe_error(*args, **kw)
self._builtin_hook(*args, **kw)
sys.excepthook = forwarding_handler
def teardown_function(self):
sys.excepthook = getattr(self, '_builtin_hook', sys.__excepthook__)
@staticmethod
def log_error(msg):
try:
from twitter.common import log
log.error(msg)
except ImportError:
sys.stderr.write(msg + '\n')
@staticmethod
def scribe_error(*args, **kw):
options = app.get_options()
socket = TSocket.TSocket(host=options.twitter_common_scribe_host,
port=options.twitter_common_scribe_port)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
value = BasicExceptionHandler.format(*args, **kw)
log_entry = scribe.LogEntry(category=options.twitter_common_scribe_category,
message=value)
try:
transport.open()
result = client.Log(messages=[log_entry])
transport.close()
if result != scribe.ResultCode.OK:
AppScribeExceptionHandler.log_error('Failed to scribe exception!')
except TTransport.TTransportException:
AppScribeExceptionHandler.log_error('Could not connect to scribe!')
def _get_option_from_args(self, args, kwargs):
if len(args) == 1 and kwargs == {} and isinstance(
args[0], options.Option):
return args[0]
else:
return options.Option(*args, **kwargs)
class Application(object):
class Error(Exception):
pass
# enforce a quasi-singleton interface (for resettable applications in test)
_GLOBAL = None
HELP_OPTIONS = [
options.Option("-h",
"--help",
"--short-help",
action="callback",
callback=lambda *args, **kwargs: Application.active().
_short_help(*args, **kwargs),
help="show this help message and exit."),
options.Option(
"--long-help",
action="callback",
callback=lambda *args, **kwargs: Application.active()._long_help(
*args, **kwargs),
help=
"show options from all registered modules, not just the __main__ module."
)
]
IGNORE_RC_FLAG = '--app_ignore_rc_file'
APP_OPTIONS = {
'daemonize':
options.Option('--app_daemonize',
action='store_true',
default=False,
dest='twitter_common_app_daemonize',
help="Daemonize this application."),
'daemon_stdout':
options.Option(
'--app_daemon_stdout',
default='/dev/null',
dest='twitter_common_app_daemon_stdout',
help="Direct this app's stdout to this file if daemonized."),
'daemon_stderr':
options.Option(
'--app_daemon_stderr',
default='/dev/null',
dest='twitter_common_app_daemon_stderr',
help="Direct this app's stderr to this file if daemonized."),
'pidfile':
options.Option(
'--app_pidfile',
default=None,
dest='twitter_common_app_pidfile',
help="The pidfile to use if --app_daemonize is specified."),
'debug':
options.Option(
'--app_debug',
action='store_true',
default=False,
dest='twitter_common_app_debug',
help=
"Print extra debugging information during application initialization."
),
'profiling':
options.Option(
'--app_profiling',
action='store_true',
default=False,
dest='twitter_common_app_profiling',
help=
"Run profiler on the code while it runs. Note this can cause slowdowns."
),
'profile_output':
options.Option(
'--app_profile_output',
default=None,
metavar='FILENAME',
dest='twitter_common_app_profile_output',
help="Dump the profiling output to a binary profiling format."),
'rc_filename':
options.Option('--app_rc_filename',
action='store_true',
default=False,
dest='twitter_common_app_rc_filename',
help="Print the filename for the rc file and quit."),
'ignore_rc_file':
options.Option(IGNORE_RC_FLAG,
action='store_true',
default=False,
dest='twitter_common_app_ignore_rc_file',
help="Ignore default arguments from the rc file."),
}
OPTIONS = 'options'
OPTIONS_ATTR = '__options__'
NO_COMMAND = 'DEFAULT'
SIGINT_RETURN_CODE = 130 # see http://tldp.org/LDP/abs/html/exitcodes.html
INITIALIZING = 1
INITIALIZED = 2
RUNNING = 3
ABORTING = 4
SHUTDOWN = 5
@classmethod
def reset(cls):
"""Reset the global application. Only useful for testing."""
cls._GLOBAL = cls()
@classmethod
def active(cls):
"""Return the current resident application object."""
return cls._GLOBAL
def __init__(self, exit_function=sys.exit, force_args=None):
self._name = None
self._exit_function = exit_function
self._force_args = force_args
self._registered_modules = []
self._init_modules = []
self._option_targets = defaultdict(dict)
self._global_options = {}
self._interspersed_args = False
self._main_options = self.HELP_OPTIONS[:]
self._main_thread = None
self._shutdown_commands = []
self._usage = ""
self._profiler = None
self._commands = {}
self._state = self.INITIALIZING
self._reset()
for opt in self.APP_OPTIONS.values():
self.add_option(opt)
self._configure_options(None, self.APP_OPTIONS)
def pre_initialization(method):
@wraps(method)
def wrapped_method(self, *args, **kw):
if self._state > self.INITIALIZING:
raise self.Error(
"Cannot perform operation after initialization!")
return method(self, *args, **kw)
return wrapped_method
def post_initialization(method):
@wraps(method)
def wrapped_method(self, *args, **kw):
if self._state < self.INITIALIZED:
raise self.Error(
"Cannot perform operation before initialization!")
return method(self, *args, **kw)
return wrapped_method
def _reset(self):
"""
Resets the state set up by init() so that init() may be called again.
"""
self._state = self.INITIALIZING
self._option_values = options.Values()
self._argv = []
def interspersed_args(self, value):
self._interspersed_args = bool(value)
def _configure_options(self, module, option_dict):
for opt_name, opt in option_dict.items():
self._option_targets[module][opt_name] = opt.dest
@pre_initialization
def configure(self, module=None, **kw):
"""
Configure the application object or its activated modules.
Typically application modules export flags that can be defined on the
command-line. In order to allow the application to override defaults,
these modules may export named parameters to be overridden. For example,
the Application object itself exports named variables such as "debug" or
"profiling", which can be enabled via:
app.configure(debug=True)
and
app.configure(profiling=True)
respectively. They can also be enabled with their command-line argument
counterpart, e.g.
./my_application --app_debug --app_profiling
Some modules export named options, e.g. twitter.common.app.modules.http exports
'enable', 'host', 'port'. The command-line arguments still take precedence and
will override any defaults set by the application in app.configure. To activate
these options, just pass along the module name:
app.configure(module='twitter.common.app.modules.http', enable=True)
"""
if module not in self._option_targets:
if not self._import_module(module):
raise self.Error('Unknown module to configure: %s' % module)
def configure_option(name, value):
if name not in self._option_targets[module]:
raise self.Error('Module %s has no option %s' % (module, name))
self.set_option(self._option_targets[module][name], value)
for option_name, option_value in kw.items():
configure_option(option_name, option_value)
def _main_parser(self):
return (options.parser().interspersed_arguments(
self._interspersed_args).options(self._main_options).usage(
self._usage))
def command_parser(self, command):
assert command in self._commands
values_copy = copy.deepcopy(self._option_values)
parser = self._main_parser()
command_group = options.new_group(('For %s only' %
command) if command else 'Default')
for option in getattr(self._commands[command],
Application.OPTIONS_ATTR, []):
op = copy.deepcopy(option)
if not hasattr(values_copy, op.dest):
setattr(
values_copy, op.dest,
op.default if op.default != optparse.NO_DEFAULT else None)
self.rewrite_help(op)
op.default = optparse.NO_DEFAULT
command_group.add_option(op)
parser = parser.groups([command_group]).values(values_copy)
usage = self._commands[command].__doc__
if usage:
parser = parser.usage(usage)
return parser
def _construct_partial_parser(self):
"""
Construct an options parser containing only options added by __main__
or global help options registered by the application.
"""
if hasattr(self._commands.get(self._command), self.OPTIONS_ATTR):
return self.command_parser(self._command)
else:
return self._main_parser().values(
copy.deepcopy(self._option_values))
def _construct_full_parser(self):
"""
Construct an options parser containing both local and global (module-level) options.
"""
return self._construct_partial_parser().groups(
self._global_options.values())
def _rc_filename(self):
rc_short_filename = '~/.%src' % self.name()
return os.path.expanduser(rc_short_filename)
def _add_default_options(self, argv):
"""
Return an argument list with options from the rc file prepended.
"""
rc_filename = self._rc_filename()
options = argv
if self.IGNORE_RC_FLAG not in argv and os.path.exists(rc_filename):
command = self._command or self.NO_COMMAND
rc_config = ConfigParser.SafeConfigParser()
rc_config.read(rc_filename)
if rc_config.has_option(command, self.OPTIONS):
default_options_str = rc_config.get(command, self.OPTIONS)
default_options = shlex.split(default_options_str, True)
options = default_options + options
return options
def _parse_options(self, force_args=None):
"""
Parse options and set self.option_values and self.argv to the values to be passed into
the application's main() method.
"""
argv = sys.argv[1:] if force_args is None else force_args
if argv and argv[0] in self._commands:
self._command = argv.pop(0)
else:
self._command = None
parser = self._construct_full_parser()
self._option_values, self._argv = parser.parse(
self._add_default_options(argv))
def _short_help(self, option, opt, value, parser):
self._construct_partial_parser().print_help()
self._exit_function(1)
return
def _long_help(self, option, opt, value, parser):
self._construct_full_parser().print_help()
self._exit_function(1)
return
@pre_initialization
def _setup_modules(self):
"""
Setup all initialized modules.
"""
module_registry = AppModule.module_registry()
for bundle in topological_sort(AppModule.module_dependencies()):
for module_label in bundle:
assert module_label in module_registry
module = module_registry[module_label]
self._debug_log('Initializing: %s (%s)' %
(module.label(), module.description()))
try:
module.setup_function()
except AppModule.Unimplemented:
pass
self._init_modules.append(module.label())
def _teardown_modules(self):
"""
Teardown initialized module in reverse initialization order.
"""
if self._state != self.SHUTDOWN:
raise self.Error('Expected application to be in SHUTDOWN state!')
module_registry = AppModule.module_registry()
for module_label in reversed(self._init_modules):
assert module_label in module_registry
module = module_registry[module_label]
self._debug_log('Running exit function for %s (%s)' %
(module_label, module.description()))
try:
module.teardown_function()
except AppModule.Unimplemented:
pass
def _maybe_daemonize(self):
if self._option_values.twitter_common_app_daemonize:
daemonize(
pidfile=self._option_values.twitter_common_app_pidfile,
stdout=self._option_values.twitter_common_app_daemon_stdout,
stderr=self._option_values.twitter_common_app_daemon_stderr)
# ------- public exported methods -------
@pre_initialization
def init(self):
"""
Initialize the state necessary to run the application's main() function but
without actually invoking main.
"""
self._parse_options(self._force_args)
self._maybe_daemonize()
self._setup_modules()
self._state = self.INITIALIZED
def reinit(self, force_args=None):
"""
Reinitialize the application. This clears the stateful parts of the application
framework and reruns init(). Mostly useful for testing.
"""
self._reset()
self.init(force_args)
@post_initialization
def argv(self):
return self._argv
@pre_initialization
def add_module_path(self, name, path):
"""
Add all app.Modules defined by name at path.
Typical usage (e.g. from the __init__.py of something containing many
app modules):
app.add_module_path(__name__, __path__)
"""
import pkgutil
for _, mod, ispkg in pkgutil.iter_modules(path):
if ispkg:
continue
fq_module = '.'.join([name, mod])
__import__(fq_module)
for (kls_name, kls) in inspect.getmembers(sys.modules[fq_module],
inspect.isclass):
if issubclass(kls, AppModule):
self.register_module(kls())
@pre_initialization
def register_module(self, module):
"""
Register an app.Module and all its options.
"""
if not isinstance(module, AppModule):
raise TypeError(
'register_module should be called with a subclass of AppModule'
)
if module.label() in self._registered_modules:
# Do not reregister.
return
if hasattr(module, 'OPTIONS'):
if not isinstance(module.OPTIONS, dict):
raise self.Error(
'Registered app.Module %s has invalid OPTIONS.' %
module.__module__)
for opt in module.OPTIONS.values():
self._add_option(module.__module__, opt)
self._configure_options(module.label(), module.OPTIONS)
self._registered_modules.append(module.label())
@classmethod
def _get_module_key(cls, module):
return 'From module %s' % module
@pre_initialization
def _add_main_option(self, option):
self._main_options.append(option)
@pre_initialization
def _add_module_option(self, module, option):
calling_module = self._get_module_key(module)
if calling_module not in self._global_options:
self._global_options[calling_module] = options.new_group(
calling_module)
self._global_options[calling_module].add_option(option)
@staticmethod
def rewrite_help(op):
if hasattr(op, 'help') and isinstance(op.help, Compatibility.string):
if op.help.find(
'%default') != -1 and op.default != optparse.NO_DEFAULT:
op.help = op.help.replace('%default', str(op.default))
else:
op.help = op.help + ((' [default: %s]' % str(op.default)) if
op.default != optparse.NO_DEFAULT else '')
def _add_option(self, calling_module, option):
op = copy.deepcopy(option)
if op.dest and hasattr(op, 'default'):
self.set_option(
op.dest,
op.default if op.default != optparse.NO_DEFAULT else None,
force=False)
self.rewrite_help(op)
op.default = optparse.NO_DEFAULT
if calling_module == '__main__':
self._add_main_option(op)
else:
self._add_module_option(calling_module, op)
def _get_option_from_args(self, args, kwargs):
if len(args) == 1 and kwargs == {} and isinstance(
args[0], options.Option):
return args[0]
else:
return options.TwitterOption(*args, **kwargs)
@pre_initialization
def add_option(self, *args, **kwargs):
"""
Add an option to the application.
You may pass either an Option object from the optparse/options module, or
pass the *args/**kwargs necessary to construct an Option.
"""
calling_module = Inspection.find_calling_module()
added_option = self._get_option_from_args(args, kwargs)
self._add_option(calling_module, added_option)
def _set_command_origin(self, function, command_name):
function.__app_command_origin__ = (self, command_name)
def _get_command_name(self, function):
assert self._is_app_command(function)
return function.__app_command_origin__[1]
def _is_app_command(self, function):
return callable(function) and (getattr(
function, '__app_command_origin__', (None, None))[0] == self)
def command(self, function=None, name=None):
"""
Decorator to turn a function into an application command.
To add a command foo, the following patterns will both work:
@app.command
def foo(args, options):
...
@app.command(name='foo')
def bar(args, options):
...
"""
if name is None:
return self._command(function)
else:
return partial(self._command, name=name)
def _command(self, function, name=None):
command_name = name or function.__name__
self._set_command_origin(function, command_name)
if Inspection.find_calling_module() == '__main__':
self._register_command(function, command_name)
return function
def register_commands_from(self, *modules):
"""
Given an imported module, walk the module for commands that have been
annotated with @app.command and register them against this
application.
"""
for module in modules:
for _, function in inspect.getmembers(
module, predicate=lambda fn: callable(fn)):
if self._is_app_command(function):
self._register_command(function,
self._get_command_name(function))
@pre_initialization
def _register_command(self, function, command_name):
"""
Registers function as the handler for command_name. Uses function.__name__ if command_name
is None.
"""
if command_name in self._commands:
raise self.Error('Found two definitions for command %s' %
command_name)
self._commands[command_name] = function
return function
def default_command(self, function):
"""
Decorator to make a command default.
"""
if Inspection.find_calling_module() == '__main__':
if None in self._commands:
defaults = (self._commands[None].__name__, function.__name__)
raise self.Error('Found two default commands: %s and %s' %
defaults)
self._commands[None] = function
return function
@pre_initialization
def command_option(self, *args, **kwargs):
"""
Decorator to add an option only for a specific command.
"""
def register_option(function):
added_option = self._get_option_from_args(args, kwargs)
if not hasattr(function, self.OPTIONS_ATTR):
setattr(function, self.OPTIONS_ATTR, deque())
getattr(function, self.OPTIONS_ATTR).appendleft(added_option)
return function
return register_option
@pre_initialization
def copy_command_options(self, command_function):
"""
Decorator to copy command options from another command.
"""
def register_options(function):
if hasattr(command_function, self.OPTIONS_ATTR):
if not hasattr(function, self.OPTIONS_ATTR):
setattr(function, self.OPTIONS_ATTR, deque())
command_options = getattr(command_function, self.OPTIONS_ATTR)
getattr(function,
self.OPTIONS_ATTR).extendleft(command_options)
return function
return register_options
def add_command_options(self, command_function):
"""
Function to add all options from a command
"""
module = inspect.getmodule(command_function).__name__
for option in getattr(command_function, self.OPTIONS_ATTR, ()):
self._add_option(module, option)
def _debug_log(self, msg):
if hasattr(self._option_values, 'twitter_common_app_debug') and (
self._option_values.twitter_common_app_debug):
print('twitter.common.app debug: %s' % msg, file=sys.stderr)
def set_option(self, dest, value, force=True):
"""
Set a global option value either pre- or post-initialization.
If force=False, do not set the default if already overridden by a manual call to
set_option.
"""
if hasattr(self._option_values, dest) and not force:
return
setattr(self._option_values, dest, value)
def get_options(self):
"""
Return all application options, both registered by __main__ and all imported modules.
"""
return self._option_values
def get_commands(self):
"""
Return all valid commands registered by __main__
"""
return list(filter(None, self._commands.keys()))
def get_commands_and_docstrings(self):
"""
Generate all valid commands together with their docstrings
"""
for command, function in self._commands.items():
if command is not None:
yield command, function.__doc__
def get_local_options(self):
"""
Return the options only defined by __main__.
"""
new_values = options.Values()
for opt in self._main_options:
if opt.dest:
setattr(new_values, opt.dest,
getattr(self._option_values, opt.dest))
return new_values
@pre_initialization
def set_usage(self, usage):
"""
Set the usage message should the user call --help or invalidly specify options.
"""
self._usage = usage
def set_usage_based_on_commands(self):
"""
Sets the usage message automatically, to show the available commands.
"""
self.set_usage(
'Please run with one of the following commands:\n' + '\n'.join([
' %-22s%s' %
(command, self._set_string_margin(docstring or '', 0, 24))
for (command, docstring) in self.get_commands_and_docstrings()
]))
@staticmethod
def _set_string_margin(s, first_line_indentation, other_lines_indentation):
"""
Given a multi-line string, resets the indentation to the given number of spaces.
"""
lines = s.strip().splitlines()
lines = ([
' ' * first_line_indentation + line.strip() for line in lines[:1]
] + [
' ' * other_lines_indentation + line.strip() for line in lines[1:]
])
return '\n'.join(lines)
def error(self, message):
"""
Print the application help message, an error message, then exit.
"""
self._construct_partial_parser().error(message)
def help(self):
"""
Print the application help message and exit.
"""
self._short_help(None, None, None, None)
@pre_initialization
def set_name(self, application_name):
"""
Set the application name. (Autodetect otherwise.)
"""
self._name = application_name
def name(self):
"""
Return the name of the application. If set_name was never explicitly called,
the application framework will attempt to autodetect the name of the application
based upon the location of __main__.
"""
if self._name is not None:
return self._name
else:
try:
return Inspection.find_application_name()
# TODO(wickman) Be more specific
except Exception:
return 'unknown'
def quit(self, return_code):
nondaemons = 0
for thr in threading.enumerate():
self._debug_log(' Active thread%s: %s' %
(' (daemon)' if thr.isDaemon() else '', thr))
if thr is not threading.current_thread() and not thr.isDaemon():
nondaemons += 1
if nondaemons:
self._debug_log(
'More than one active non-daemon thread, your application may hang!'
)
else:
self._debug_log('Exiting cleanly.')
self._exit_function(return_code)
def profiler(self):
if self._option_values.twitter_common_app_profiling:
if self._profiler is None:
try:
import cProfile as profile
except ImportError:
import profile
self._profiler = profile.Profile()
return self._profiler
else:
return None
def dump_profile(self):
if self._option_values.twitter_common_app_profiling:
if self._option_values.twitter_common_app_profile_output:
self.profiler().dump_stats(
self._option_values.twitter_common_app_profile_output)
else:
self.profiler().print_stats(sort='time')
# The thread module provides the interrupt_main() function which does
# precisely what it says, sending a KeyboardInterrupt to MainThread. The
# only problem is that it only delivers the exception while the MainThread
# is running. If one does time.sleep(10000000) it will simply block
# forever. Sending an actual SIGINT seems to be the only way around this.
# Of course, applications can trap SIGINT and prevent the quitquitquit
# handlers from working.
#
# Furthermore, the following cannot work:
#
# def main():
# shutdown_event = threading.Event()
# app.register_shutdown_command(lambda rc: shutdown_event.set())
# shutdown_event.wait()
#
# because threading.Event.wait() is uninterruptible. This is why
# abortabortabort is so severe. An application that traps SIGTERM will
# render the framework unable to abort it, so SIGKILL is really the only
# way to be sure to force termination because it cannot be trapped.
#
# For the particular case where the bulk of the work is taking place in
# background threads, use app.wait_forever().
def quitquitquit(self):
self._state = self.ABORTING
os.kill(os.getpid(), signal.SIGINT)
def abortabortabort(self):
self._state = self.SHUTDOWN
os.kill(os.getpid(), signal.SIGKILL)
def register_shutdown_command(self, command):
if not callable(command):
raise TypeError('Shutdown command must be a callable.')
if self._state >= self.ABORTING:
raise self.Error(
'Cannot register a shutdown command while shutting down.')
self._shutdown_commands.append(command)
def _wrap_method(self, method, method_name=None):
method_name = method_name or method.__name__
try:
return_code = method()
except SystemExit as e:
self._debug_log('%s sys.exited' % method_name)
return_code = e.code
except KeyboardInterrupt as e:
if self._state >= self.ABORTING:
self._debug_log('%s being shutdown' % method_name)
return_code = 0
else:
self._debug_log('%s exited with ^C' % method_name)
return_code = self.SIGINT_RETURN_CODE
except Exception as e:
return_code = 1
self._debug_log('%s excepted with %s' % (method_name, type(e)))
sys.excepthook(*sys.exc_info())
return return_code
@post_initialization
def _run_main(self, main_method, *args, **kwargs):
if self.profiler():
main = lambda: self.profiler().runcall(main_method, *args, **kwargs
)
else:
main = lambda: main_method(*args, **kwargs)
self._state = self.RUNNING
return self._wrap_method(main, method_name='main')
def _run_shutdown_commands(self, return_code):
while self._state != self.SHUTDOWN and self._shutdown_commands:
command = self._shutdown_commands.pop(0)
command(return_code)
def _run_module_teardown(self):
if self._state != self.SHUTDOWN:
raise self.Error('Expected application to be in SHUTDOWN state!')
self._debug_log('Shutting application down.')
self._teardown_modules()
self._debug_log('Finishing up module teardown.')
self.dump_profile()
def _import_module(self, name):
"""
Import the module, return True on success, False if the import failed.
"""
try:
__import__(name)
return True
except ImportError:
return False
def _validate_main_module(self):
main_module = Inspection.find_calling_module()
return main_module == '__main__'
def _default_command_is_defined(self):
return None in self._commands
# Allow for overrides in test
def _find_main_method(self):
try:
return Inspection.find_main_from_caller()
except Inspection.InternalError:
pass
def _get_main_method(self):
caller_main = self._find_main_method()
if self._default_command_is_defined() and caller_main is not None:
print('Error: Cannot define both main and a default command.',
file=sys.stderr)
self._exit_function(1)
return
main_method = self._commands.get(self._command) or caller_main
if main_method is None:
commands = sorted(self.get_commands())
if commands:
print('Must supply one of the following commands:',
', '.join(commands),
file=sys.stderr)
else:
print(
'No main() or command defined! Application must define one of these.',
file=sys.stderr)
return main_method
def wait_forever(self):
"""Convenience function to block the application until it is terminated
by ^C or lifecycle functions."""
while True:
time.sleep(0.5)
def shutdown(self, return_code):
self._wrap_method(lambda: self._run_shutdown_commands(return_code),
method_name='shutdown commands')
self._state = self.SHUTDOWN
self._run_module_teardown()
self.quit(return_code)
def main(self):
"""
If called from __main__ module, run script's main() method with arguments passed
and global options parsed.
The following patterns are acceptable for the main method:
main()
main(args)
main(args, options)
"""
if not self._validate_main_module():
# only support if __name__ == '__main__'
return
# Pull in modules in twitter.common.app.modules
if not self._import_module('twitter.common.app.modules'):
print('Unable to import twitter app modules!', file=sys.stderr)
self._exit_function(1)
return
# defer init as long as possible.
self.init()
if self._option_values.twitter_common_app_rc_filename:
print('RC filename: %s' % self._rc_filename())
return
main_method = self._get_main_method()
if main_method is None:
self._exit_function(1)
return
try:
argspec = inspect.getargspec(main_method)
except TypeError as e:
print('Malformed main(): %s' % e, file=sys.stderr)
self._exit_function(1)
return
if len(argspec.args) == 1:
args = [self._argv]
elif len(argspec.args) == 2:
args = [self._argv, self._option_values]
else:
if len(self._argv) != 0:
print(
'main() takes no arguments but got leftover arguments: %s!'
% ' '.join(self._argv),
file=sys.stderr)
self._exit_function(1)
return
args = []
self.shutdown(self._run_main(main_method, *args))
del post_initialization
del pre_initialization
class ServerSetModule(app.Module):
"""
Binds this application to a Zookeeper ServerSet.
"""
OPTIONS = {
'serverset-enable':
options.Option(
'--serverset-enable',
default=False,
action='store_true',
dest='serverset_module_enable',
help=
'Enable the ServerSet module. Requires --serverset-path and --serverset-primary.'
),
'serverset-ensemble':
options.Option(
'--serverset-ensemble',
default='zookeeper.local.twitter.com:2181',
dest='serverset_module_ensemble',
metavar='HOST[:PORT]',
help=
'The serverset ensemble to talk to. HOST or HOST:PORT pair. If the HOST is a RR DNS '
'record, we fan out to the entire ensemble. If no port is specified, 2181 assumed.'
),
'serverset-path':
options.Option(
'--serverset-path',
default=None,
dest='serverset_module_path',
metavar='PATH',
type='str',
help=
'The serverset path to join, preferably /twitter/service/(role)/(service)/(env) '
'where env is prod, staging, devel.'),
'serverset-primary':
options.Option('--serverset-primary',
type='int',
metavar='PORT',
dest='serverset_module_primary_port',
default=None,
help='Port on which to bind the primary endpoint.'),
'serverset-shard-id':
options.Option('--serverset-shard-id',
type='int',
metavar='INT',
dest='serverset_module_shard_id',
default=None,
help='Shard id to assign this serverset entry.'),
'serverset-extra':
options.Option(
'--serverset-extra',
default={},
type='string',
nargs=1,
action='callback',
metavar='NAME:PORT',
callback=add_port_to('serverset_module_extra'),
dest='serverset_module_extra',
help=
'Additional endpoints to bind. Format NAME:PORT. May be specified multiple times.'
),
'serverset-persistence':
options.Option(
'--serverset-persistence',
'--no-serverset-persistence',
action='callback',
callback=set_bool,
dest='serverset_module_persistence',
default=True,
help=
'If serverset persistence is enabled, if the serverset connection is dropped for any '
'reason, we will retry to connect forever. If serverset persistence is turned off, '
'the application will commit seppuku -- sys.exit(1) -- upon session disconnection.'
),
}
def __init__(self):
app.Module.__init__(self, __name__, description="ServerSet module")
self._zookeeper = None
self._serverset = None
self._membership = None
self._join_args = None
self._torndown = False
self._rejoin_event = threading.Event()
self._joiner = None
@property
def serverset(self):
return self._serverset
@property
def zh(self):
if self._zookeeper:
return self._zookeeper._zh
def _assert_valid_inputs(self, options):
if not options.serverset_module_enable:
return
assert options.serverset_module_path is not None, (
'If serverset module enabled, serverset path must be specified.')
assert options.serverset_module_primary_port is not None, (
'If serverset module enabled, serverset primary port must be specified.'
)
assert isinstance(
options.serverset_module_extra,
dict), ('Serverset additional endpoints must be a dictionary!')
for name, value in options.serverset_module_extra.items():
assert isinstance(
name, str), 'Additional endpoints must be named by strings!'
assert isinstance(
value, int), 'Additional endpoint ports must be integers!'
try:
primary_port = int(options.serverset_module_primary_port)
except ValueError as e:
raise ValueError('Could not parse serverset primary port: %s' % e)
def _construct_serverset(self, options):
import socket
import threading
import zookeeper
from twitter.common.zookeeper.client import ZooKeeper
from twitter.common.zookeeper.serverset import Endpoint, ServerSet
log.debug('ServerSet module constructing serverset.')
hostname = socket.gethostname()
primary_port = int(options.serverset_module_primary_port)
primary = Endpoint(hostname, primary_port)
additional = dict((port_name, Endpoint(hostname, port_number))
for port_name, port_number in
options.serverset_module_extra.items())
# TODO(wickman) Add timeout parameterization here.
self._zookeeper = ZooKeeper(options.serverset_module_ensemble)
self._serverset = ServerSet(self._zookeeper,
options.serverset_module_path)
self._join_args = (primary, additional)
self._join_kwargs = ({
'shard': options.serverset_module_shard_id
} if options.serverset_module_shard_id else {})
def _join(self):
log.debug('ServerSet module joining serverset.')
primary, additional = self._join_args
self._membership = self._serverset.join(
primary,
additional,
expire_callback=self.on_expiration,
**self._join_kwargs)
def on_expiration(self):
if self._torndown:
return
log.debug('Serverset session expired.')
if not app.get_options().serverset_module_persistence:
log.debug('Committing seppuku...')
sys.exit(1)
else:
log.debug('Rejoining...')
self._rejoin_event.set()
def setup_function(self):
options = app.get_options()
if options.serverset_module_enable:
self._assert_valid_inputs(options)
self._construct_serverset(options)
self._thread = ServerSetJoinThread(self._rejoin_event, self._join)
self._thread.start()
self._rejoin_event.set()
def teardown_function(self):
self._torndown = True
if self._membership:
self._serverset.cancel(self._membership)
self._zookeeper.stop()
class RootServer(HttpServer, app.Module):
"""
A root singleton server for all your http endpoints to bind to.
"""
OPTIONS = {
'enable':
options.Option(
'--enable_http',
default=False,
action='store_true',
dest='twitter_common_http_root_server_enabled',
help=
'Enable root http server for various subsystems, e.g. metrics exporting.'
),
'port':
options.Option(
'--http_port',
default=8888,
type='int',
metavar='PORT',
dest='twitter_common_http_root_server_port',
help='The port the root http server will be listening on.'),
'host':
options.Option(
'--http_host',
default='localhost',
type='string',
metavar='HOSTNAME',
dest='twitter_common_http_root_server_host',
help='The host the root http server will be listening on.'),
'framework':
options.Option(
'--http_framework',
default='wsgiref',
type='string',
metavar='FRAMEWORK',
dest='twitter_common_http_root_server_framework',
help=
'The framework that will be running the integrated http server.')
}
def __init__(self):
self._thread = None
HttpServer.__init__(self)
app.Module.__init__(self, __name__, description="Http subsystem.")
def setup_function(self):
assert self._thread is None, "Attempting to call start() after server has been started!"
options = app.get_options()
parent = self
self.mount_routes(DiagnosticsEndpoints())
class RootServerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
def run(self):
rs = parent
rs.run(
options.twitter_common_http_root_server_host,
options.twitter_common_http_root_server_port,
server=options.twitter_common_http_root_server_framework)
if options.twitter_common_http_root_server_enabled:
self._thread = RootServerThread()
self._thread.start()
class VarsSubsystem(app.Module):
"""
Exports a /vars endpoint on the root http server bound to twitter.common.metrics.RootMetrics.
"""
OPTIONS = {
'sampling_delay':
options.Option(
'--vars-sampling-delay-ms',
default=1000,
type='int',
metavar='MILLISECONDS',
dest='twitter_common_metrics_vars_sampling_delay_ms',
help='How long between taking samples of the vars subsystem.'),
'trace_endpoints':
options.Option(
'--vars-trace-endpoints',
'--no-vars-trace-endpoints',
default=True,
action='callback',
callback=set_bool,
dest='twitter_common_app_modules_varz_trace_endpoints',
help='Trace all registered http endpoints in this application.'),
'trace_namespace':
options.Option('--trace-namespace',
default='http',
dest='twitter_common_app_modules_varz_trace_namespace',
help='The prefix for http request metrics.'),
'stats_filter':
options.Option(
'--vars-stats-filter',
default=[],
action='append',
dest='twitter_common_app_modules_varz_stats_filter',
help='Full-match regexes to filter metrics on-demand when requested '
'with `filtered=1`.')
}
def __init__(self):
app.Module.__init__(self,
__name__,
description='Vars subsystem',
dependencies='twitter.common.app.modules.http')
def setup_function(self):
options = app.get_options()
rs = RootServer()
if rs:
varz = VarsEndpoint(
period=Amount(
options.twitter_common_metrics_vars_sampling_delay_ms,
Time.MILLISECONDS),
stats_filter=self.compile_stats_filters(
options.twitter_common_app_modules_varz_stats_filter))
rs.mount_routes(varz)
register_diagnostics()
register_build_properties()
if options.twitter_common_app_modules_varz_trace_endpoints:
plugin = EndpointTracePlugin()
rs.install(plugin)
RootMetrics().register_observable(
options.twitter_common_app_modules_varz_trace_namespace,
plugin)
def compile_stats_filters(self, regexes_list):
if len(regexes_list) > 0:
# safeguard against partial matches
full_regexes = ['^' + regex + '$' for regex in regexes_list]
return re.compile('(' + ")|(".join(full_regexes) + ')')
else:
return None
