def test_value_inheritance(self): option_list = [ options.Option('-a', dest='a'), options.Option('-b', dest='b') ] values, leftovers = options.parser().options(option_list).parse([]) assert not hasattr(values, 'a') assert not hasattr(values, 'b') # w/ option values, leftovers = options.parser().options(option_list).parse( ['-a', 'value_a']) assert hasattr(values, 'a') assert values.a == 'value_a' assert not hasattr(values, 'b') # w/ inherited option values, leftovers = options.parser().values(values).options( option_list).parse(['-b', 'value_b']) assert values.a == 'value_a' assert values.b == 'value_b' # w/ inherits w/o parsing any new args values, leftovers = options.parser().values(values).options( option_list).parse([]) assert values.a == 'value_a' assert values.b == 'value_b' # w/ overwrites despite inheriting values, leftovers = options.parser().values(values).options( option_list).parse(['-a', 'new_value_a']) assert values.a == 'new_value_a' assert values.b == 'value_b'
def test_multiple_option_inheritance(self): option_a = options.Option('-a', dest='a') option_b = options.Option('-b', dest='b') values, leftovers = (options.parser().options([option_a]).options( [option_b])).parse(['-a', 'value_a', '-b', 'value_b']) assert values.a == 'value_a' assert values.b == 'value_b'
class VarsSubsystem(app.Module): """ Exports a /vars endpoint on the root http server bound to twitter.common.metrics.RootMetrics. """ OPTIONS = { 'sampling_delay': options.Option( '--vars-sampling-delay-ms', default=1000, type='int', metavar='MILLISECONDS', dest='twitter_common_metrics_vars_sampling_delay_ms', help='How long between taking samples of the vars subsystem.'), 'trace_endpoints': options.Option( '--vars-trace-endpoints', '--no-vars-trace-endpoints', default=True, action='callback', callback=set_bool, dest='twitter_common_app_modules_varz_trace_endpoints', help='Trace all registered http endpoints in this application.'), 'trace_namespace': options.Option('--trace-namespace', default='http', dest='twitter_common_app_modules_varz_trace_namespace', help='The prefix for http request metrics.') } def __init__(self): app.Module.__init__(self, __name__, description='Vars subsystem', dependencies='twitter.common.app.modules.http') def setup_function(self): options = app.get_options() rs = RootServer() if rs: varz = VarsEndpoint(period=Amount( options.twitter_common_metrics_vars_sampling_delay_ms, Time.MILLISECONDS)) rs.mount_routes(varz) register_diagnostics() register_build_properties() if options.twitter_common_app_modules_varz_trace_endpoints: plugin = EndpointTracePlugin() rs.install(plugin) RootMetrics().register_observable( options.twitter_common_app_modules_varz_trace_namespace, plugin)
def test_app_add_options_with_Option(self): # options.Option opt = options.Option('--option1', dest='option1') app.add_option(opt) app.init(force_args=['--option1', 'option1value', 'extraargs']) assert app.get_options().option1 == 'option1value' assert app.argv() == ['extraargs']
class VarsSubsystem(app.Module): """ Exports a /vars endpoint on the root http server bound to twitter.common.metrics.RootMetrics. """ OPTIONS = { 'sampling_delay': options.Option( '--vars_sampling_delay_ms', default=1000, type='int', metavar='MILLISECONDS', dest='twitter_common_metrics_vars_sampling_delay_ms', help='How long between taking samples of the vars subsystem.') } def __init__(self): app.Module.__init__(self, __name__, description="Vars subsystem", dependencies='twitter.common.app.modules.http') def setup_function(self): options = app.get_options() rs = RootServer() if rs: varz = VarsEndpoint(period=Amount( options.twitter_common_metrics_vars_sampling_delay_ms, Time.MILLISECONDS)) rs.mount_routes(varz) register_diagnostics() register_build_properties()
def test_basic_parsing(self): option = options.Option('-m', '--my_option', dest='my_option') # w/o option values, leftovers = options.parser().options([option]).parse([]) assert not hasattr(values, 'my_option') assert leftovers == [] # w/ option values, leftovers = options.parser().options([option ]).parse(['-m', 'poop']) assert values.my_option == 'poop' assert leftovers == [] # w/ long option values, leftovers = options.parser().options([option]).parse( ['--my_option', 'plork']) assert values.my_option == 'plork' assert leftovers == [] # w/ option and leftover values, leftovers = options.parser().options([option]).parse( ['--my_option', 'plork', 'hork']) assert values.my_option == 'plork' assert leftovers == ['hork']
def test_default_parsing(self): option = options.Option('-m', '--my_option', default="specified", dest='my_option') values, leftovers = options.parser().options([option]).parse([]) assert hasattr(values, 'my_option') assert leftovers == [] assert values.my_option == 'specified'
def test_multiple_value_inheritance(self): option_list = [ options.Option('-a', dest='a'), options.Option('-b', dest='b') ] values_with_a, _ = options.parser().options(option_list).parse( ['-a', 'value_a']) values_with_b, _ = options.parser().options(option_list).parse( ['-b', 'value_b']) values, leftovers = (options.parser().options(option_list).values( values_with_a).values(values_with_b)).parse([]) assert values.a == 'value_a' assert values.b == 'value_b' # and parsed values overwrite values, leftovers = (options.parser().options(option_list).values( values_with_a).values(values_with_b)).parse(['-a', 'new_value_a']) assert values.a == 'new_value_a' assert values.b == 'value_b'
def test_groups(self): option_a = options.Option('-a', dest='a') option_b = options.Option('-b', dest='b') option_group_a = options.group('a') option_group_b = options.group('b') option_group_a.add_option(options.Option('--a1', dest='a1'), options.Option('--a2', dest='a2')) option_group_b.add_option(options.Option('--b1', dest='b1'), options.Option('--b2', dest='b2')) partial_parser = (options.parser().interspersed_arguments(True).groups( [option_group_a, option_group_b])) full_parser = partial_parser.options([option_a, option_b]) parameters = [ '--a1', 'value_a1', '--a2', 'value_a2', '--b1', 'value_b1', '--b2', 'value_b2' ] full_parameters = parameters + ['-a', 'value_a', '-b', 'value_b'] values, leftovers = partial_parser.parse(parameters) assert values.a1 == 'value_a1' assert values.a2 == 'value_a2' assert values.b1 == 'value_b1' assert values.b2 == 'value_b2' assert leftovers == [] values, leftovers = full_parser.parse(full_parameters) assert values.a1 == 'value_a1' assert values.a2 == 'value_a2' assert values.b1 == 'value_b1' assert values.b2 == 'value_b2' assert values.a == 'value_a' assert values.b == 'value_b' assert leftovers == []
class Application(object): class Error(Exception): pass # enforce a quasi-singleton interface (for resettable applications in test) _Global = None @staticmethod def reset(): """Reset the global application. Only useful for testing.""" Application._Global = Application() @staticmethod def active(): """Return the current resident application object.""" return Application._Global HELP_OPTIONS = [ options.Option("-h", "--help", "--short-help", action="callback", callback=lambda *args, **kwargs: Application.active(). _short_help(*args, **kwargs), help="show this help message and exit."), options.Option( "--long-help", action="callback", callback=lambda *args, **kwargs: Application.active()._long_help( *args, **kwargs), help= "show options from all registered modules, not just the __main__ module." ) ] IGNORE_RC_FLAG = '--app_ignore_rc_file' APP_OPTIONS = { 'daemonize': options.Option('--app_daemonize', action='store_true', default=False, dest='twitter_common_app_daemonize', help="Daemonize this application."), 'daemon_stdout': options.Option( '--app_daemon_stdout', default='/dev/null', dest='twitter_common_app_daemon_stdout', help="Direct this app's stdout to this file if daemonized."), 'daemon_stderr': options.Option( '--app_daemon_stderr', default='/dev/null', dest='twitter_common_app_daemon_stderr', help="Direct this app's stderr to this file if daemonized."), 'pidfile': options.Option( '--app_pidfile', default=None, dest='twitter_common_app_pidfile', help="The pidfile to use if --app_daemonize is specified."), 'debug': options.Option( '--app_debug', action='store_true', default=False, dest='twitter_common_app_debug', help= "Print extra debugging information during application initialization." ), 'profiling': options.Option( '--app_profiling', action='store_true', default=False, dest='twitter_common_app_profiling', help= "Run profiler on the code while it runs. Note this can cause slowdowns." ), 'profile_output': options.Option( '--app_profile_output', default=None, metavar='FILENAME', dest='twitter_common_app_profile_output', help="Dump the profiling output to a binary profiling format."), 'rc_filename': options.Option('--app_rc_filename', action='store_true', default=False, dest='twitter_common_app_rc_filename', help="Print the filename for the rc file and quit."), 'ignore_rc_file': options.Option(IGNORE_RC_FLAG, action='store_true', default=False, dest='twitter_common_app_ignore_rc_file', help="Ignore default arguments from the rc file."), } NO_COMMAND = 'DEFAULT' OPTIONS = 'options' OPTIONS_ATTR = '__options__' def __init__(self): self._name = None self._registered_modules = [] self._init_modules = [] self._option_targets = defaultdict(dict) self._global_options = {} self._interspersed_args = False self._main_options = Application.HELP_OPTIONS[:] self._usage = "" self._profiler = None self._commands = {} self._reset() for opt in Application.APP_OPTIONS.values(): self.add_option(opt) self._configure_options(None, Application.APP_OPTIONS) def _raise_if_initialized( self, msg="Cannot perform operation after initialization!"): if self.initialized: raise Application.Error(msg) def _raise_if_uninitialized( self, msg="Cannot perform operation before initialization!"): if not self.initialized: raise Application.Error(msg) def _reset(self): """ Resets the state set up by init() so that init() may be called again. """ self.initialized = False self._option_values = options.Values() self._argv = [] def interspersed_args(self, value): self._interspersed_args = bool(value) def _configure_options(self, module, option_dict): for opt_name, opt in option_dict.items(): self._option_targets[module][opt_name] = opt.dest def configure(self, module=None, **kw): """ Configure the application object or its activated modules. Typically application modules export flags that can be defined on the command-line. In order to allow the application to override defaults, these modules may export named parameters to be overridden. For example, the Application object itself exports named variables such as "debug" or "profiling", which can be enabled via: app.configure(debug=True) and app.configure(profiling=True) respectively. They can also be enabled with their command-line argument counterpart, e.g. ./my_application --app_debug --app_profiling Some modules export named options, e.g. twitter.common.app.modules.http exports 'enable', 'host', 'port'. The command-line arguments still take precedence and will override any defaults set by the application in app.configure. To activate these options, just pass along the module name: app.configure(module='twitter.common.app.modules.http', enable=True) """ if module not in self._option_targets: if not self._import_module(module): raise Application.Error('Unknown module to configure: %s' % module) def configure_option(name, value): if name not in self._option_targets[module]: raise Application.Error('Module %s has no option %s' % (module, name)) self.set_option(self._option_targets[module][name], value) for option_name, option_value in kw.items(): configure_option(option_name, option_value) def _main_parser(self): return (options.parser().interspersed_arguments( self._interspersed_args).options(self._main_options).usage( self._usage)) def command_parser(self, command): assert command in self._commands values_copy = copy.deepcopy(self._option_values) parser = self._main_parser() command_group = options.new_group(('For %s only' % command) if command else 'Default') for option in getattr(self._commands[command], Application.OPTIONS_ATTR): op = copy.deepcopy(option) if not hasattr(values_copy, op.dest): setattr( values_copy, op.dest, op.default if op.default != optparse.NO_DEFAULT else None) Application.rewrite_help(op) op.default = optparse.NO_DEFAULT command_group.add_option(op) parser = parser.groups([command_group]).values(values_copy) usage = self._commands[command].__doc__ if usage: parser = parser.usage(usage) return parser def _construct_partial_parser(self): """ Construct an options parser containing only options added by __main__ or global help options registered by the application. """ if hasattr(self._commands.get(self._command), Application.OPTIONS_ATTR): return self.command_parser(self._command) else: return self._main_parser().values( copy.deepcopy(self._option_values)) def _construct_full_parser(self): """ Construct an options parser containing both local and global (module-level) options. """ return self._construct_partial_parser().groups( self._global_options.values()) def _rc_filename(self): rc_short_filename = '~/.%src' % self.name() return os.path.expanduser(rc_short_filename) def _add_default_options(self, argv): """ Return an argument list with options from the rc file prepended. """ rc_filename = self._rc_filename() options = argv if Application.IGNORE_RC_FLAG not in argv and os.path.exists( rc_filename): command = self._command or Application.NO_COMMAND rc_config = ConfigParser.SafeConfigParser() rc_config.read(rc_filename) if rc_config.has_option(command, Application.OPTIONS): default_options_str = rc_config.get(command, Application.OPTIONS) default_options = shlex.split(default_options_str, True) options = default_options + options return options def _parse_options(self, force_args=None): """ Parse options and set self.option_values and self.argv to the values to be passed into the application's main() method. """ argv = sys.argv[1:] if force_args is None else force_args if argv and argv[0] in self._commands: self._command = argv.pop(0) else: self._command = None parser = self._construct_full_parser() self._option_values, self._argv = parser.parse( self._add_default_options(argv)) def _short_help(self, option, opt, value, parser): self._construct_partial_parser().print_help() sys.exit(1) def _long_help(self, option, opt, value, parser): self._construct_full_parser().print_help() sys.exit(1) def _setup_modules(self): """ Setup all initialized modules. """ module_registry = AppModule.module_registry() for bundle in topological_sort(AppModule.module_dependencies()): for module_label in bundle: assert module_label in module_registry module = module_registry[module_label] self._debug_log('Initializing: %s (%s)' % (module.label(), module.description())) try: module.setup_function() except AppModule.Unimplemented: pass self._init_modules.append(module.label()) def _teardown_modules(self): """ Teardown initialized module in reverse initialization order. """ module_registry = AppModule.module_registry() for module_label in reversed(self._init_modules): assert module_label in module_registry module = module_registry[module_label] self._debug_log('Running exit function for %s (%s)' % (module_label, module.description())) try: module.teardown_function() except AppModule.Unimplemented: pass def _maybe_daemonize(self): if self._option_values.twitter_common_app_daemonize: daemonize( pidfile=self._option_values.twitter_common_app_pidfile, stdout=self._option_values.twitter_common_app_daemon_stdout, stderr=self._option_values.twitter_common_app_daemon_stderr) # ------- public exported methods ------- def init(self, force_args=None): """ Initialize the state necessary to run the application's main() function but without actually invoking main. Mostly useful for testing. If force_args specified, use those arguments instead of sys.argv[1:]. """ self._raise_if_initialized( "init cannot be called twice. Use reinit if necessary.") self._parse_options(force_args) self._maybe_daemonize() self._setup_modules() self.initialized = True def reinit(self, force_args=None): """ Reinitialize the application. This clears the stateful parts of the application framework and reruns init(). Mostly useful for testing. """ self._reset() self.init(force_args) def argv(self): self._raise_if_uninitialized( "Must call app.init() before you may access argv.") return self._argv def add_module_path(self, name, path): """ Add all app.Modules defined by name at path. Typical usage (e.g. from the __init__.py of something containing many app modules): app.add_module_path(__name__, __path__) """ import pkgutil for _, mod, ispkg in pkgutil.iter_modules(path): if ispkg: continue fq_module = '.'.join([name, mod]) __import__(fq_module) for (kls_name, kls) in inspect.getmembers(sys.modules[fq_module], inspect.isclass): if issubclass(kls, AppModule): self.register_module(kls()) def register_module(self, module): """ Register an app.Module and all its options. """ if not isinstance(module, AppModule): raise TypeError( 'register_module should be called with a subclass of AppModule' ) if module.label() in self._registered_modules: # Do not reregister. return if hasattr(module, 'OPTIONS'): if not isinstance(module.OPTIONS, dict): raise Application.Error( 'Registered app.Module %s has invalid OPTIONS.' % module.__module__) for opt in module.OPTIONS.values(): self._add_option(module.__module__, opt) self._configure_options(module.label(), module.OPTIONS) self._registered_modules.append(module.label()) @staticmethod def _get_module_key(module): return 'From module %s' % module def _add_main_option(self, option): self._main_options.append(option) def _add_module_option(self, module, option): calling_module = Application._get_module_key(module) if calling_module not in self._global_options: self._global_options[calling_module] = options.new_group( calling_module) self._global_options[calling_module].add_option(option) @staticmethod def rewrite_help(op): if hasattr(op, 'help') and isinstance(op.help, Compatibility.string): if op.help.find( '%default') != -1 and op.default != optparse.NO_DEFAULT: op.help = op.help.replace('%default', str(op.default)) else: op.help = op.help + ((' [default: %s]' % str(op.default)) if op.default != optparse.NO_DEFAULT else '') def _add_option(self, calling_module, option): op = copy.deepcopy(option) if op.dest and hasattr(op, 'default'): self.set_option( op.dest, op.default if op.default != optparse.NO_DEFAULT else None, force=False) Application.rewrite_help(op) op.default = optparse.NO_DEFAULT if calling_module == '__main__': self._add_main_option(op) else: self._add_module_option(calling_module, op) def _get_option_from_args(self, args, kwargs): if len(args) == 1 and kwargs == {} and isinstance( args[0], options.Option): return args[0] else: return options.TwitterOption(*args, **kwargs) def add_option(self, *args, **kwargs): """ Add an option to the application. You may pass either an Option object from the optparse/options module, or pass the *args/**kwargs necessary to construct an Option. """ self._raise_if_initialized("Cannot call add_option() after main()!") calling_module = Inspection.find_calling_module() added_option = self._get_option_from_args(args, kwargs) self._add_option(calling_module, added_option) def command(self, function=None, name=None): """ Decorator to turn a function into an application command. To add a command foo, the following patterns will both work: @app.command def foo(args, options): ... @app.command(name='foo') def bar(args, options): ... """ if name is None: return self._register_command(function) else: return partial(self._register_command, command_name=name) def _register_command(self, function, command_name=None): """ Registers function as the handler for command_name. Uses function.__name__ if command_name is None. """ if Inspection.find_calling_module() == '__main__': if command_name is None: command_name = function.__name__ if command_name in self._commands: raise Application.Error( 'Found two definitions for command %s' % command_name) self._commands[command_name] = function return function def default_command(self, function): """ Decorator to make a command default. """ if Inspection.find_calling_module() == '__main__': if None in self._commands: defaults = (self._commands[None].__name__, function.__name__) raise Application.Error( 'Found two default commands: %s and %s' % defaults) self._commands[None] = function return function def command_option(self, *args, **kwargs): """ Decorator to add an option only for a specific command. """ def register_option(function): added_option = self._get_option_from_args(args, kwargs) if not hasattr(function, Application.OPTIONS_ATTR): setattr(function, Application.OPTIONS_ATTR, deque()) getattr(function, Application.OPTIONS_ATTR).appendleft(added_option) return function return register_option def copy_command_options(self, command_function): """ Decorator to copy command options from another command. """ def register_options(function): if hasattr(command_function, Application.OPTIONS_ATTR): if not hasattr(function, Application.OPTIONS_ATTR): setattr(function, Application.OPTIONS_ATTR, deque()) command_options = getattr(command_function, Application.OPTIONS_ATTR) getattr(function, Application.OPTIONS_ATTR).extendleft(command_options) return function return register_options def add_command_options(self, command_function): """ Function to add all options from a command """ module = inspect.getmodule(command_function).__name__ for option in getattr(command_function, Application.OPTIONS_ATTR, []): self._add_option(module, option) def _debug_log(self, msg): if hasattr(self._option_values, 'twitter_common_app_debug') and ( self._option_values.twitter_common_app_debug): print('twitter.common.app debug: %s' % msg, file=sys.stderr) def set_option(self, dest, value, force=True): """ Set a global option value either pre- or post-initialization. If force=False, do not set the default if already overridden by a manual call to set_option. """ if hasattr(self._option_values, dest) and not force: return setattr(self._option_values, dest, value) def get_options(self): """ Return all application options, both registered by __main__ and all imported modules. """ return self._option_values def get_commands(self): """ Return all valid commands registered by __main__ """ return filter(None, self._commands.keys()) def get_commands_and_docstrings(self): """ Generate all valid commands together with their docstrings """ for command, function in self._commands.items(): if command is not None: yield command, function.__doc__ def get_local_options(self): """ Return the options only defined by __main__. """ new_values = options.Values() for opt in self._main_options: if opt.dest: setattr(new_values, opt.dest, getattr(self._option_values, opt.dest)) return new_values def set_usage(self, usage): """ Set the usage message should the user call --help or invalidly specify options. """ self._usage = usage def error(self, message): """ Print the application help message, an error message, then exit. """ self._construct_partial_parser().error(message) def help(self): """ Print the application help message and exit. """ self._short_help(*(None, ) * 4) def set_name(self, application_name): """ Set the application name. (Autodetect otherwise.) """ self._raise_if_initialized("Cannot set application name.") self._name = application_name def name(self): """ Return the name of the application. If set_name was never explicitly called, the application framework will attempt to autodetect the name of the application based upon the location of __main__. """ if self._name is not None: return self._name else: try: return Inspection.find_application_name() except: return 'unknown' def quit(self, rc, exit_function=sys.exit): self._debug_log('Shutting application down.') self._teardown_modules() self._debug_log('Finishing up module teardown.') nondaemons = 0 self.dump_profile() for thr in threading.enumerate(): self._debug_log(' Active thread%s: %s' % (' (daemon)' if thr.isDaemon() else '', thr)) if thr is not threading.current_thread() and not thr.isDaemon(): nondaemons += 1 if nondaemons: self._debug_log( 'More than one active non-daemon thread, your application may hang!' ) else: self._debug_log('Exiting cleanly.') exit_function(rc) def profiler(self): if self._option_values.twitter_common_app_profiling: if self._profiler is None: try: import cProfile as profile except ImportError: import profile self._profiler = profile.Profile() return self._profiler else: return None def dump_profile(self): if self._option_values.twitter_common_app_profiling: if self._option_values.twitter_common_app_profile_output: self.profiler().dump_stats( self._option_values.twitter_common_app_profile_output) else: self.profiler().print_stats(sort='time') def _run_main(self, main_method, *args, **kwargs): try: if self.profiler(): rc = self.profiler().runcall(main_method, *args, **kwargs) else: rc = main_method(*args, **kwargs) except SystemExit as e: rc = e.code self._debug_log('main_method exited with return code = %s' % repr(rc)) except KeyboardInterrupt as e: rc = None self._debug_log('main_method exited with ^C') return rc def _import_module(self, name): """ Import the module, return True on success, False if the import failed. """ try: __import__(name) return True except ImportError: return False def main(self): """ If called from __main__ module, run script's main() method with arguments passed and global options parsed. The following patterns are acceptable for the main method: main() main(args) main(args, options) """ main_module = Inspection.find_calling_module() if main_module != '__main__': # only support if __name__ == '__main__' return # Pull in modules in twitter.common.app.modules if not self._import_module('twitter.common.app.modules'): print('Unable to import twitter app modules!', file=sys.stderr) sys.exit(1) # defer init as long as possible. self.init() if self._option_values.twitter_common_app_rc_filename: print('RC filename: %s' % self._rc_filename()) return try: caller_main = Inspection.find_main_from_caller() except Inspection.InternalError: caller_main = None if None in self._commands: assert caller_main is None, "Error: Cannot define both main and a default command." else: self._commands[None] = caller_main main_method = self._commands[self._command] if main_method is None: commands = sorted(self.get_commands()) if commands: print('Must supply one of the following commands:', ', '.join(commands), file=sys.stderr) else: print( 'No main() or command defined! Application must define one of these.', file=sys.stderr) sys.exit(1) try: argspec = inspect.getargspec(main_method) except TypeError as e: print('Malformed main(): %s' % e, file=sys.stderr) sys.exit(1) if len(argspec.args) == 1: args = [self._argv] elif len(argspec.args) == 2: args = [self._argv, self._option_values] else: if len(self._argv) != 0: print( 'main() takes no arguments but got leftover arguments: %s!' % ' '.join(self._argv), file=sys.stderr) sys.exit(1) args = [] rc = self._run_main(main_method, *args) self.quit(rc)
class AppScribeExceptionHandler(app.Module): """ An application module that logs or scribes uncaught exceptions. """ OPTIONS = { 'port': options.Option('--scribe_exception_port', default=1463, type='int', metavar='PORT', dest='twitter_common_scribe_port', help='The port on which scribe aggregator listens.'), 'host': options.Option('--scribe_exception_host', default='localhost', type='string', metavar='HOSTNAME', dest='twitter_common_scribe_host', help='The host to which scribe exceptions should be written.'), 'category': options.Option('--scribe_exception_category', default='python_default', type='string', metavar='CATEGORY', dest='twitter_common_scribe_category', help='The scribe category into which we write exceptions.') } def __init__(self): app.Module.__init__(self, __name__, description="twitter.common.log handler.") def setup_function(self): self._builtin_hook = sys.excepthook def forwarding_handler(*args, **kw): AppScribeExceptionHandler.scribe_error(*args, **kw) self._builtin_hook(*args, **kw) sys.excepthook = forwarding_handler def teardown_function(self): sys.excepthook = getattr(self, '_builtin_hook', sys.__excepthook__) @staticmethod def log_error(msg): try: from twitter.common import log log.error(msg) except ImportError: sys.stderr.write(msg + '\n') @staticmethod def scribe_error(*args, **kw): options = app.get_options() socket = TSocket.TSocket(host=options.twitter_common_scribe_host, port=options.twitter_common_scribe_port) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) value = BasicExceptionHandler.format(*args, **kw) log_entry = scribe.LogEntry(category=options.twitter_common_scribe_category, message=value) try: transport.open() result = client.Log(messages=[log_entry]) transport.close() if result != scribe.ResultCode.OK: AppScribeExceptionHandler.log_error('Failed to scribe exception!') except TTransport.TTransportException: AppScribeExceptionHandler.log_error('Could not connect to scribe!')
def _get_option_from_args(self, args, kwargs): if len(args) == 1 and kwargs == {} and isinstance( args[0], options.Option): return args[0] else: return options.Option(*args, **kwargs)
class Application(object): class Error(Exception): pass # enforce a quasi-singleton interface (for resettable applications in test) _GLOBAL = None HELP_OPTIONS = [ options.Option("-h", "--help", "--short-help", action="callback", callback=lambda *args, **kwargs: Application.active(). _short_help(*args, **kwargs), help="show this help message and exit."), options.Option( "--long-help", action="callback", callback=lambda *args, **kwargs: Application.active()._long_help( *args, **kwargs), help= "show options from all registered modules, not just the __main__ module." ) ] IGNORE_RC_FLAG = '--app_ignore_rc_file' APP_OPTIONS = { 'daemonize': options.Option('--app_daemonize', action='store_true', default=False, dest='twitter_common_app_daemonize', help="Daemonize this application."), 'daemon_stdout': options.Option( '--app_daemon_stdout', default='/dev/null', dest='twitter_common_app_daemon_stdout', help="Direct this app's stdout to this file if daemonized."), 'daemon_stderr': options.Option( '--app_daemon_stderr', default='/dev/null', dest='twitter_common_app_daemon_stderr', help="Direct this app's stderr to this file if daemonized."), 'pidfile': options.Option( '--app_pidfile', default=None, dest='twitter_common_app_pidfile', help="The pidfile to use if --app_daemonize is specified."), 'debug': options.Option( '--app_debug', action='store_true', default=False, dest='twitter_common_app_debug', help= "Print extra debugging information during application initialization." ), 'profiling': options.Option( '--app_profiling', action='store_true', default=False, dest='twitter_common_app_profiling', help= "Run profiler on the code while it runs. Note this can cause slowdowns." ), 'profile_output': options.Option( '--app_profile_output', default=None, metavar='FILENAME', dest='twitter_common_app_profile_output', help="Dump the profiling output to a binary profiling format."), 'rc_filename': options.Option('--app_rc_filename', action='store_true', default=False, dest='twitter_common_app_rc_filename', help="Print the filename for the rc file and quit."), 'ignore_rc_file': options.Option(IGNORE_RC_FLAG, action='store_true', default=False, dest='twitter_common_app_ignore_rc_file', help="Ignore default arguments from the rc file."), } OPTIONS = 'options' OPTIONS_ATTR = '__options__' NO_COMMAND = 'DEFAULT' SIGINT_RETURN_CODE = 130 # see http://tldp.org/LDP/abs/html/exitcodes.html INITIALIZING = 1 INITIALIZED = 2 RUNNING = 3 ABORTING = 4 SHUTDOWN = 5 @classmethod def reset(cls): """Reset the global application. Only useful for testing.""" cls._GLOBAL = cls() @classmethod def active(cls): """Return the current resident application object.""" return cls._GLOBAL def __init__(self, exit_function=sys.exit, force_args=None): self._name = None self._exit_function = exit_function self._force_args = force_args self._registered_modules = [] self._init_modules = [] self._option_targets = defaultdict(dict) self._global_options = {} self._interspersed_args = False self._main_options = self.HELP_OPTIONS[:] self._main_thread = None self._shutdown_commands = [] self._usage = "" self._profiler = None self._commands = {} self._state = self.INITIALIZING self._reset() for opt in self.APP_OPTIONS.values(): self.add_option(opt) self._configure_options(None, self.APP_OPTIONS) def pre_initialization(method): @wraps(method) def wrapped_method(self, *args, **kw): if self._state > self.INITIALIZING: raise self.Error( "Cannot perform operation after initialization!") return method(self, *args, **kw) return wrapped_method def post_initialization(method): @wraps(method) def wrapped_method(self, *args, **kw): if self._state < self.INITIALIZED: raise self.Error( "Cannot perform operation before initialization!") return method(self, *args, **kw) return wrapped_method def _reset(self): """ Resets the state set up by init() so that init() may be called again. """ self._state = self.INITIALIZING self._option_values = options.Values() self._argv = [] def interspersed_args(self, value): self._interspersed_args = bool(value) def _configure_options(self, module, option_dict): for opt_name, opt in option_dict.items(): self._option_targets[module][opt_name] = opt.dest @pre_initialization def configure(self, module=None, **kw): """ Configure the application object or its activated modules. Typically application modules export flags that can be defined on the command-line. In order to allow the application to override defaults, these modules may export named parameters to be overridden. For example, the Application object itself exports named variables such as "debug" or "profiling", which can be enabled via: app.configure(debug=True) and app.configure(profiling=True) respectively. They can also be enabled with their command-line argument counterpart, e.g. ./my_application --app_debug --app_profiling Some modules export named options, e.g. twitter.common.app.modules.http exports 'enable', 'host', 'port'. The command-line arguments still take precedence and will override any defaults set by the application in app.configure. To activate these options, just pass along the module name: app.configure(module='twitter.common.app.modules.http', enable=True) """ if module not in self._option_targets: if not self._import_module(module): raise self.Error('Unknown module to configure: %s' % module) def configure_option(name, value): if name not in self._option_targets[module]: raise self.Error('Module %s has no option %s' % (module, name)) self.set_option(self._option_targets[module][name], value) for option_name, option_value in kw.items(): configure_option(option_name, option_value) def _main_parser(self): return (options.parser().interspersed_arguments( self._interspersed_args).options(self._main_options).usage( self._usage)) def command_parser(self, command): assert command in self._commands values_copy = copy.deepcopy(self._option_values) parser = self._main_parser() command_group = options.new_group(('For %s only' % command) if command else 'Default') for option in getattr(self._commands[command], Application.OPTIONS_ATTR, []): op = copy.deepcopy(option) if not hasattr(values_copy, op.dest): setattr( values_copy, op.dest, op.default if op.default != optparse.NO_DEFAULT else None) self.rewrite_help(op) op.default = optparse.NO_DEFAULT command_group.add_option(op) parser = parser.groups([command_group]).values(values_copy) usage = self._commands[command].__doc__ if usage: parser = parser.usage(usage) return parser def _construct_partial_parser(self): """ Construct an options parser containing only options added by __main__ or global help options registered by the application. """ if hasattr(self._commands.get(self._command), self.OPTIONS_ATTR): return self.command_parser(self._command) else: return self._main_parser().values( copy.deepcopy(self._option_values)) def _construct_full_parser(self): """ Construct an options parser containing both local and global (module-level) options. """ return self._construct_partial_parser().groups( self._global_options.values()) def _rc_filename(self): rc_short_filename = '~/.%src' % self.name() return os.path.expanduser(rc_short_filename) def _add_default_options(self, argv): """ Return an argument list with options from the rc file prepended. """ rc_filename = self._rc_filename() options = argv if self.IGNORE_RC_FLAG not in argv and os.path.exists(rc_filename): command = self._command or self.NO_COMMAND rc_config = ConfigParser.SafeConfigParser() rc_config.read(rc_filename) if rc_config.has_option(command, self.OPTIONS): default_options_str = rc_config.get(command, self.OPTIONS) default_options = shlex.split(default_options_str, True) options = default_options + options return options def _parse_options(self, force_args=None): """ Parse options and set self.option_values and self.argv to the values to be passed into the application's main() method. """ argv = sys.argv[1:] if force_args is None else force_args if argv and argv[0] in self._commands: self._command = argv.pop(0) else: self._command = None parser = self._construct_full_parser() self._option_values, self._argv = parser.parse( self._add_default_options(argv)) def _short_help(self, option, opt, value, parser): self._construct_partial_parser().print_help() self._exit_function(1) return def _long_help(self, option, opt, value, parser): self._construct_full_parser().print_help() self._exit_function(1) return @pre_initialization def _setup_modules(self): """ Setup all initialized modules. """ module_registry = AppModule.module_registry() for bundle in topological_sort(AppModule.module_dependencies()): for module_label in bundle: assert module_label in module_registry module = module_registry[module_label] self._debug_log('Initializing: %s (%s)' % (module.label(), module.description())) try: module.setup_function() except AppModule.Unimplemented: pass self._init_modules.append(module.label()) def _teardown_modules(self): """ Teardown initialized module in reverse initialization order. """ if self._state != self.SHUTDOWN: raise self.Error('Expected application to be in SHUTDOWN state!') module_registry = AppModule.module_registry() for module_label in reversed(self._init_modules): assert module_label in module_registry module = module_registry[module_label] self._debug_log('Running exit function for %s (%s)' % (module_label, module.description())) try: module.teardown_function() except AppModule.Unimplemented: pass def _maybe_daemonize(self): if self._option_values.twitter_common_app_daemonize: daemonize( pidfile=self._option_values.twitter_common_app_pidfile, stdout=self._option_values.twitter_common_app_daemon_stdout, stderr=self._option_values.twitter_common_app_daemon_stderr) # ------- public exported methods ------- @pre_initialization def init(self): """ Initialize the state necessary to run the application's main() function but without actually invoking main. """ self._parse_options(self._force_args) self._maybe_daemonize() self._setup_modules() self._state = self.INITIALIZED def reinit(self, force_args=None): """ Reinitialize the application. This clears the stateful parts of the application framework and reruns init(). Mostly useful for testing. """ self._reset() self.init(force_args) @post_initialization def argv(self): return self._argv @pre_initialization def add_module_path(self, name, path): """ Add all app.Modules defined by name at path. Typical usage (e.g. from the __init__.py of something containing many app modules): app.add_module_path(__name__, __path__) """ import pkgutil for _, mod, ispkg in pkgutil.iter_modules(path): if ispkg: continue fq_module = '.'.join([name, mod]) __import__(fq_module) for (kls_name, kls) in inspect.getmembers(sys.modules[fq_module], inspect.isclass): if issubclass(kls, AppModule): self.register_module(kls()) @pre_initialization def register_module(self, module): """ Register an app.Module and all its options. """ if not isinstance(module, AppModule): raise TypeError( 'register_module should be called with a subclass of AppModule' ) if module.label() in self._registered_modules: # Do not reregister. return if hasattr(module, 'OPTIONS'): if not isinstance(module.OPTIONS, dict): raise self.Error( 'Registered app.Module %s has invalid OPTIONS.' % module.__module__) for opt in module.OPTIONS.values(): self._add_option(module.__module__, opt) self._configure_options(module.label(), module.OPTIONS) self._registered_modules.append(module.label()) @classmethod def _get_module_key(cls, module): return 'From module %s' % module @pre_initialization def _add_main_option(self, option): self._main_options.append(option) @pre_initialization def _add_module_option(self, module, option): calling_module = self._get_module_key(module) if calling_module not in self._global_options: self._global_options[calling_module] = options.new_group( calling_module) self._global_options[calling_module].add_option(option) @staticmethod def rewrite_help(op): if hasattr(op, 'help') and isinstance(op.help, Compatibility.string): if op.help.find( '%default') != -1 and op.default != optparse.NO_DEFAULT: op.help = op.help.replace('%default', str(op.default)) else: op.help = op.help + ((' [default: %s]' % str(op.default)) if op.default != optparse.NO_DEFAULT else '') def _add_option(self, calling_module, option): op = copy.deepcopy(option) if op.dest and hasattr(op, 'default'): self.set_option( op.dest, op.default if op.default != optparse.NO_DEFAULT else None, force=False) self.rewrite_help(op) op.default = optparse.NO_DEFAULT if calling_module == '__main__': self._add_main_option(op) else: self._add_module_option(calling_module, op) def _get_option_from_args(self, args, kwargs): if len(args) == 1 and kwargs == {} and isinstance( args[0], options.Option): return args[0] else: return options.TwitterOption(*args, **kwargs) @pre_initialization def add_option(self, *args, **kwargs): """ Add an option to the application. You may pass either an Option object from the optparse/options module, or pass the *args/**kwargs necessary to construct an Option. """ calling_module = Inspection.find_calling_module() added_option = self._get_option_from_args(args, kwargs) self._add_option(calling_module, added_option) def _set_command_origin(self, function, command_name): function.__app_command_origin__ = (self, command_name) def _get_command_name(self, function): assert self._is_app_command(function) return function.__app_command_origin__[1] def _is_app_command(self, function): return callable(function) and (getattr( function, '__app_command_origin__', (None, None))[0] == self) def command(self, function=None, name=None): """ Decorator to turn a function into an application command. To add a command foo, the following patterns will both work: @app.command def foo(args, options): ... @app.command(name='foo') def bar(args, options): ... """ if name is None: return self._command(function) else: return partial(self._command, name=name) def _command(self, function, name=None): command_name = name or function.__name__ self._set_command_origin(function, command_name) if Inspection.find_calling_module() == '__main__': self._register_command(function, command_name) return function def register_commands_from(self, *modules): """ Given an imported module, walk the module for commands that have been annotated with @app.command and register them against this application. """ for module in modules: for _, function in inspect.getmembers( module, predicate=lambda fn: callable(fn)): if self._is_app_command(function): self._register_command(function, self._get_command_name(function)) @pre_initialization def _register_command(self, function, command_name): """ Registers function as the handler for command_name. Uses function.__name__ if command_name is None. """ if command_name in self._commands: raise self.Error('Found two definitions for command %s' % command_name) self._commands[command_name] = function return function def default_command(self, function): """ Decorator to make a command default. """ if Inspection.find_calling_module() == '__main__': if None in self._commands: defaults = (self._commands[None].__name__, function.__name__) raise self.Error('Found two default commands: %s and %s' % defaults) self._commands[None] = function return function @pre_initialization def command_option(self, *args, **kwargs): """ Decorator to add an option only for a specific command. """ def register_option(function): added_option = self._get_option_from_args(args, kwargs) if not hasattr(function, self.OPTIONS_ATTR): setattr(function, self.OPTIONS_ATTR, deque()) getattr(function, self.OPTIONS_ATTR).appendleft(added_option) return function return register_option @pre_initialization def copy_command_options(self, command_function): """ Decorator to copy command options from another command. """ def register_options(function): if hasattr(command_function, self.OPTIONS_ATTR): if not hasattr(function, self.OPTIONS_ATTR): setattr(function, self.OPTIONS_ATTR, deque()) command_options = getattr(command_function, self.OPTIONS_ATTR) getattr(function, self.OPTIONS_ATTR).extendleft(command_options) return function return register_options def add_command_options(self, command_function): """ Function to add all options from a command """ module = inspect.getmodule(command_function).__name__ for option in getattr(command_function, self.OPTIONS_ATTR, ()): self._add_option(module, option) def _debug_log(self, msg): if hasattr(self._option_values, 'twitter_common_app_debug') and ( self._option_values.twitter_common_app_debug): print('twitter.common.app debug: %s' % msg, file=sys.stderr) def set_option(self, dest, value, force=True): """ Set a global option value either pre- or post-initialization. If force=False, do not set the default if already overridden by a manual call to set_option. """ if hasattr(self._option_values, dest) and not force: return setattr(self._option_values, dest, value) def get_options(self): """ Return all application options, both registered by __main__ and all imported modules. """ return self._option_values def get_commands(self): """ Return all valid commands registered by __main__ """ return list(filter(None, self._commands.keys())) def get_commands_and_docstrings(self): """ Generate all valid commands together with their docstrings """ for command, function in self._commands.items(): if command is not None: yield command, function.__doc__ def get_local_options(self): """ Return the options only defined by __main__. """ new_values = options.Values() for opt in self._main_options: if opt.dest: setattr(new_values, opt.dest, getattr(self._option_values, opt.dest)) return new_values @pre_initialization def set_usage(self, usage): """ Set the usage message should the user call --help or invalidly specify options. """ self._usage = usage def set_usage_based_on_commands(self): """ Sets the usage message automatically, to show the available commands. """ self.set_usage( 'Please run with one of the following commands:\n' + '\n'.join([ ' %-22s%s' % (command, self._set_string_margin(docstring or '', 0, 24)) for (command, docstring) in self.get_commands_and_docstrings() ])) @staticmethod def _set_string_margin(s, first_line_indentation, other_lines_indentation): """ Given a multi-line string, resets the indentation to the given number of spaces. """ lines = s.strip().splitlines() lines = ([ ' ' * first_line_indentation + line.strip() for line in lines[:1] ] + [ ' ' * other_lines_indentation + line.strip() for line in lines[1:] ]) return '\n'.join(lines) def error(self, message): """ Print the application help message, an error message, then exit. """ self._construct_partial_parser().error(message) def help(self): """ Print the application help message and exit. """ self._short_help(None, None, None, None) @pre_initialization def set_name(self, application_name): """ Set the application name. (Autodetect otherwise.) """ self._name = application_name def name(self): """ Return the name of the application. If set_name was never explicitly called, the application framework will attempt to autodetect the name of the application based upon the location of __main__. """ if self._name is not None: return self._name else: try: return Inspection.find_application_name() # TODO(wickman) Be more specific except Exception: return 'unknown' def quit(self, return_code): nondaemons = 0 for thr in threading.enumerate(): self._debug_log(' Active thread%s: %s' % (' (daemon)' if thr.isDaemon() else '', thr)) if thr is not threading.current_thread() and not thr.isDaemon(): nondaemons += 1 if nondaemons: self._debug_log( 'More than one active non-daemon thread, your application may hang!' ) else: self._debug_log('Exiting cleanly.') self._exit_function(return_code) def profiler(self): if self._option_values.twitter_common_app_profiling: if self._profiler is None: try: import cProfile as profile except ImportError: import profile self._profiler = profile.Profile() return self._profiler else: return None def dump_profile(self): if self._option_values.twitter_common_app_profiling: if self._option_values.twitter_common_app_profile_output: self.profiler().dump_stats( self._option_values.twitter_common_app_profile_output) else: self.profiler().print_stats(sort='time') # The thread module provides the interrupt_main() function which does # precisely what it says, sending a KeyboardInterrupt to MainThread. The # only problem is that it only delivers the exception while the MainThread # is running. If one does time.sleep(10000000) it will simply block # forever. Sending an actual SIGINT seems to be the only way around this. # Of course, applications can trap SIGINT and prevent the quitquitquit # handlers from working. # # Furthermore, the following cannot work: # # def main(): # shutdown_event = threading.Event() # app.register_shutdown_command(lambda rc: shutdown_event.set()) # shutdown_event.wait() # # because threading.Event.wait() is uninterruptible. This is why # abortabortabort is so severe. An application that traps SIGTERM will # render the framework unable to abort it, so SIGKILL is really the only # way to be sure to force termination because it cannot be trapped. # # For the particular case where the bulk of the work is taking place in # background threads, use app.wait_forever(). def quitquitquit(self): self._state = self.ABORTING os.kill(os.getpid(), signal.SIGINT) def abortabortabort(self): self._state = self.SHUTDOWN os.kill(os.getpid(), signal.SIGKILL) def register_shutdown_command(self, command): if not callable(command): raise TypeError('Shutdown command must be a callable.') if self._state >= self.ABORTING: raise self.Error( 'Cannot register a shutdown command while shutting down.') self._shutdown_commands.append(command) def _wrap_method(self, method, method_name=None): method_name = method_name or method.__name__ try: return_code = method() except SystemExit as e: self._debug_log('%s sys.exited' % method_name) return_code = e.code except KeyboardInterrupt as e: if self._state >= self.ABORTING: self._debug_log('%s being shutdown' % method_name) return_code = 0 else: self._debug_log('%s exited with ^C' % method_name) return_code = self.SIGINT_RETURN_CODE except Exception as e: return_code = 1 self._debug_log('%s excepted with %s' % (method_name, type(e))) sys.excepthook(*sys.exc_info()) return return_code @post_initialization def _run_main(self, main_method, *args, **kwargs): if self.profiler(): main = lambda: self.profiler().runcall(main_method, *args, **kwargs ) else: main = lambda: main_method(*args, **kwargs) self._state = self.RUNNING return self._wrap_method(main, method_name='main') def _run_shutdown_commands(self, return_code): while self._state != self.SHUTDOWN and self._shutdown_commands: command = self._shutdown_commands.pop(0) command(return_code) def _run_module_teardown(self): if self._state != self.SHUTDOWN: raise self.Error('Expected application to be in SHUTDOWN state!') self._debug_log('Shutting application down.') self._teardown_modules() self._debug_log('Finishing up module teardown.') self.dump_profile() def _import_module(self, name): """ Import the module, return True on success, False if the import failed. """ try: __import__(name) return True except ImportError: return False def _validate_main_module(self): main_module = Inspection.find_calling_module() return main_module == '__main__' def _default_command_is_defined(self): return None in self._commands # Allow for overrides in test def _find_main_method(self): try: return Inspection.find_main_from_caller() except Inspection.InternalError: pass def _get_main_method(self): caller_main = self._find_main_method() if self._default_command_is_defined() and caller_main is not None: print('Error: Cannot define both main and a default command.', file=sys.stderr) self._exit_function(1) return main_method = self._commands.get(self._command) or caller_main if main_method is None: commands = sorted(self.get_commands()) if commands: print('Must supply one of the following commands:', ', '.join(commands), file=sys.stderr) else: print( 'No main() or command defined! Application must define one of these.', file=sys.stderr) return main_method def wait_forever(self): """Convenience function to block the application until it is terminated by ^C or lifecycle functions.""" while True: time.sleep(0.5) def shutdown(self, return_code): self._wrap_method(lambda: self._run_shutdown_commands(return_code), method_name='shutdown commands') self._state = self.SHUTDOWN self._run_module_teardown() self.quit(return_code) def main(self): """ If called from __main__ module, run script's main() method with arguments passed and global options parsed. The following patterns are acceptable for the main method: main() main(args) main(args, options) """ if not self._validate_main_module(): # only support if __name__ == '__main__' return # Pull in modules in twitter.common.app.modules if not self._import_module('twitter.common.app.modules'): print('Unable to import twitter app modules!', file=sys.stderr) self._exit_function(1) return # defer init as long as possible. self.init() if self._option_values.twitter_common_app_rc_filename: print('RC filename: %s' % self._rc_filename()) return main_method = self._get_main_method() if main_method is None: self._exit_function(1) return try: argspec = inspect.getargspec(main_method) except TypeError as e: print('Malformed main(): %s' % e, file=sys.stderr) self._exit_function(1) return if len(argspec.args) == 1: args = [self._argv] elif len(argspec.args) == 2: args = [self._argv, self._option_values] else: if len(self._argv) != 0: print( 'main() takes no arguments but got leftover arguments: %s!' % ' '.join(self._argv), file=sys.stderr) self._exit_function(1) return args = [] self.shutdown(self._run_main(main_method, *args)) del post_initialization del pre_initialization
class ServerSetModule(app.Module): """ Binds this application to a Zookeeper ServerSet. """ OPTIONS = { 'serverset-enable': options.Option( '--serverset-enable', default=False, action='store_true', dest='serverset_module_enable', help= 'Enable the ServerSet module. Requires --serverset-path and --serverset-primary.' ), 'serverset-ensemble': options.Option( '--serverset-ensemble', default='zookeeper.local.twitter.com:2181', dest='serverset_module_ensemble', metavar='HOST[:PORT]', help= 'The serverset ensemble to talk to. HOST or HOST:PORT pair. If the HOST is a RR DNS ' 'record, we fan out to the entire ensemble. If no port is specified, 2181 assumed.' ), 'serverset-path': options.Option( '--serverset-path', default=None, dest='serverset_module_path', metavar='PATH', type='str', help= 'The serverset path to join, preferably /twitter/service/(role)/(service)/(env) ' 'where env is prod, staging, devel.'), 'serverset-primary': options.Option('--serverset-primary', type='int', metavar='PORT', dest='serverset_module_primary_port', default=None, help='Port on which to bind the primary endpoint.'), 'serverset-shard-id': options.Option('--serverset-shard-id', type='int', metavar='INT', dest='serverset_module_shard_id', default=None, help='Shard id to assign this serverset entry.'), 'serverset-extra': options.Option( '--serverset-extra', default={}, type='string', nargs=1, action='callback', metavar='NAME:PORT', callback=add_port_to('serverset_module_extra'), dest='serverset_module_extra', help= 'Additional endpoints to bind. Format NAME:PORT. May be specified multiple times.' ), 'serverset-persistence': options.Option( '--serverset-persistence', '--no-serverset-persistence', action='callback', callback=set_bool, dest='serverset_module_persistence', default=True, help= 'If serverset persistence is enabled, if the serverset connection is dropped for any ' 'reason, we will retry to connect forever. If serverset persistence is turned off, ' 'the application will commit seppuku -- sys.exit(1) -- upon session disconnection.' ), } def __init__(self): app.Module.__init__(self, __name__, description="ServerSet module") self._zookeeper = None self._serverset = None self._membership = None self._join_args = None self._torndown = False self._rejoin_event = threading.Event() self._joiner = None @property def serverset(self): return self._serverset @property def zh(self): if self._zookeeper: return self._zookeeper._zh def _assert_valid_inputs(self, options): if not options.serverset_module_enable: return assert options.serverset_module_path is not None, ( 'If serverset module enabled, serverset path must be specified.') assert options.serverset_module_primary_port is not None, ( 'If serverset module enabled, serverset primary port must be specified.' ) assert isinstance( options.serverset_module_extra, dict), ('Serverset additional endpoints must be a dictionary!') for name, value in options.serverset_module_extra.items(): assert isinstance( name, str), 'Additional endpoints must be named by strings!' assert isinstance( value, int), 'Additional endpoint ports must be integers!' try: primary_port = int(options.serverset_module_primary_port) except ValueError as e: raise ValueError('Could not parse serverset primary port: %s' % e) def _construct_serverset(self, options): import socket import threading import zookeeper from twitter.common.zookeeper.client import ZooKeeper from twitter.common.zookeeper.serverset import Endpoint, ServerSet log.debug('ServerSet module constructing serverset.') hostname = socket.gethostname() primary_port = int(options.serverset_module_primary_port) primary = Endpoint(hostname, primary_port) additional = dict((port_name, Endpoint(hostname, port_number)) for port_name, port_number in options.serverset_module_extra.items()) # TODO(wickman) Add timeout parameterization here. self._zookeeper = ZooKeeper(options.serverset_module_ensemble) self._serverset = ServerSet(self._zookeeper, options.serverset_module_path) self._join_args = (primary, additional) self._join_kwargs = ({ 'shard': options.serverset_module_shard_id } if options.serverset_module_shard_id else {}) def _join(self): log.debug('ServerSet module joining serverset.') primary, additional = self._join_args self._membership = self._serverset.join( primary, additional, expire_callback=self.on_expiration, **self._join_kwargs) def on_expiration(self): if self._torndown: return log.debug('Serverset session expired.') if not app.get_options().serverset_module_persistence: log.debug('Committing seppuku...') sys.exit(1) else: log.debug('Rejoining...') self._rejoin_event.set() def setup_function(self): options = app.get_options() if options.serverset_module_enable: self._assert_valid_inputs(options) self._construct_serverset(options) self._thread = ServerSetJoinThread(self._rejoin_event, self._join) self._thread.start() self._rejoin_event.set() def teardown_function(self): self._torndown = True if self._membership: self._serverset.cancel(self._membership) self._zookeeper.stop()
class RootServer(HttpServer, app.Module): """ A root singleton server for all your http endpoints to bind to. """ OPTIONS = { 'enable': options.Option( '--enable_http', default=False, action='store_true', dest='twitter_common_http_root_server_enabled', help= 'Enable root http server for various subsystems, e.g. metrics exporting.' ), 'port': options.Option( '--http_port', default=8888, type='int', metavar='PORT', dest='twitter_common_http_root_server_port', help='The port the root http server will be listening on.'), 'host': options.Option( '--http_host', default='localhost', type='string', metavar='HOSTNAME', dest='twitter_common_http_root_server_host', help='The host the root http server will be listening on.'), 'framework': options.Option( '--http_framework', default='wsgiref', type='string', metavar='FRAMEWORK', dest='twitter_common_http_root_server_framework', help= 'The framework that will be running the integrated http server.') } def __init__(self): self._thread = None HttpServer.__init__(self) app.Module.__init__(self, __name__, description="Http subsystem.") def setup_function(self): assert self._thread is None, "Attempting to call start() after server has been started!" options = app.get_options() parent = self self.mount_routes(DiagnosticsEndpoints()) class RootServerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.daemon = True def run(self): rs = parent rs.run( options.twitter_common_http_root_server_host, options.twitter_common_http_root_server_port, server=options.twitter_common_http_root_server_framework) if options.twitter_common_http_root_server_enabled: self._thread = RootServerThread() self._thread.start()
class VarsSubsystem(app.Module): """ Exports a /vars endpoint on the root http server bound to twitter.common.metrics.RootMetrics. """ OPTIONS = { 'sampling_delay': options.Option( '--vars-sampling-delay-ms', default=1000, type='int', metavar='MILLISECONDS', dest='twitter_common_metrics_vars_sampling_delay_ms', help='How long between taking samples of the vars subsystem.'), 'trace_endpoints': options.Option( '--vars-trace-endpoints', '--no-vars-trace-endpoints', default=True, action='callback', callback=set_bool, dest='twitter_common_app_modules_varz_trace_endpoints', help='Trace all registered http endpoints in this application.'), 'trace_namespace': options.Option('--trace-namespace', default='http', dest='twitter_common_app_modules_varz_trace_namespace', help='The prefix for http request metrics.'), 'stats_filter': options.Option( '--vars-stats-filter', default=[], action='append', dest='twitter_common_app_modules_varz_stats_filter', help='Full-match regexes to filter metrics on-demand when requested ' 'with `filtered=1`.') } def __init__(self): app.Module.__init__(self, __name__, description='Vars subsystem', dependencies='twitter.common.app.modules.http') def setup_function(self): options = app.get_options() rs = RootServer() if rs: varz = VarsEndpoint( period=Amount( options.twitter_common_metrics_vars_sampling_delay_ms, Time.MILLISECONDS), stats_filter=self.compile_stats_filters( options.twitter_common_app_modules_varz_stats_filter)) rs.mount_routes(varz) register_diagnostics() register_build_properties() if options.twitter_common_app_modules_varz_trace_endpoints: plugin = EndpointTracePlugin() rs.install(plugin) RootMetrics().register_observable( options.twitter_common_app_modules_varz_trace_namespace, plugin) def compile_stats_filters(self, regexes_list): if len(regexes_list) > 0: # safeguard against partial matches full_regexes = ['^' + regex + '$' for regex in regexes_list] return re.compile('(' + ")|(".join(full_regexes) + ')') else: return None