def test_basic_07_argparse_multilevel_class_expansion(self):
option_definitions = self.setup_configman_namespace()
other_value_source = {"gamma": [38, 28, 18, 8]}
other_definition_source = Namespace()
other_definition_source.add_option(
"a_class",
default="configman.tests.test_val_for_modules.Alpha",
from_string_converter=class_converter)
cm = ConfigurationManager(
definition_source=[option_definitions, other_definition_source],
values_source_list=[other_value_source, command_line],
argv_source=[
"0", "--admin.expose_secrets", "--delta",
'--gamma="8 18 28 38"',
'--a_class=configman.tests.test_val_for_modules.Delta',
'--messy=34'
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha":
0,
"beta":
'the second',
"gamma": [8, 18, 28, 38],
"delta":
True,
"admin.print_conf":
None,
"admin.dump_conf":
'',
"admin.strict":
False,
"admin.expose_secrets":
True,
"a_class":
class_converter("configman.tests.test_val_for_modules.Delta"),
"messy":
34,
"dd":
class_converter("configman.tests.test_val_for_modules.Beta"),
'b':
23,
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def jobs_converter(path_or_jobs):
"""Takes a Python dotted path or a jobs spec and returns crontabber jobs
Example Python dotted path::
jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS')
Example jobs spec::
jobs_converter('socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1h')
:arg str path_or_jobs: a Python dotted path or a crontabber jobs spec
:returns: crontabber jobs InnerClassList
"""
if '|' not in path_or_jobs:
# NOTE(willkg): configman's class_converter returns the value pointed
# to by a Python dotted path
input_str = class_converter(path_or_jobs)
else:
input_str = path_or_jobs
from_string_converter = classes_in_namespaces_converter_with_compression(
reference_namespace=Namespace(),
list_splitter_fn=line_splitter,
class_extractor=pipe_splitter,
extra_extractor=get_extra_as_options)
return from_string_converter(input_str)
def jobs_converter(path_or_jobs):
"""Takes a Python dotted path or a jobs spec and returns crontabber jobs
Example Python dotted path::
jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS')
Example jobs spec::
jobs_converter('socorro.cron.jobs.archivescraper.ArchiveScraperCronApp|1h')
:arg str path_or_jobs: a Python dotted path or a crontabber jobs spec
:returns: crontabber jobs InnerClassList
"""
if '|' not in path_or_jobs:
# NOTE(willkg): configman's class_converter returns the value pointed
# to by a Python dotted path
input_str = class_converter(path_or_jobs)
else:
input_str = path_or_jobs
# At this point, we have a big string of jobs
from_string_converter = classes_in_namespaces_converter_with_compression(
reference_namespace=Namespace())
return from_string_converter(input_str)
class HBaseClientApp(generic_app.App):
app_name = "hbase_client.py"
app_version = "0.1"
app_description = __doc__
required_config = Namespace()
required_config.add_option(
'hbase_crash_storage_class',
default=HappyBaseCrashStorage,
doc='the class responsible for proving an hbase connection',
from_string_converter=class_converter)
required_config.add_option(
'command',
default=help,
doc='command to use',
is_argument=True,
from_string_converter=lambda s: class_converter(__name__ + '.' + s))
required_config.add_option(
'json',
default=False,
short_form='j',
doc='json output instead of a pretty printed mapping',
)
def main(self):
self.storage = self.config.hbase_crash_storage_class(self.config)
self.config.command(self).run()
def main(initial_app, values_source_list=None, config_path=None):
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
if config_path is None:
default = './config'
config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH', default)
if config_path != default:
# you tried to set it, then it must be a valid directory
if not os.path.isdir(config_path):
raise IOError('%s is not a valid directory' % config_path)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the application',
default=initial_app,
from_string_converter=class_converter)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError, x:
raise AppDetailMissingError(str(x))
def main(initial_app, values_source_list=None):
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the '
'application',
default=initial_app,
from_string_converter=class_converter
)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError, x:
raise AppDetailMissingError(str(x))
class InnerClassList(RequiredConfig):
"""This nested class is a proxy list for the classes. It collects
all the config requirements for the listed classes and places them
each into their own Namespace.
"""
# we're dynamically creating a class here. The following block of
# code is actually adding class level attributes to this new class
# 1st requirement for configman
required_config = Namespace()
# to help the programmer know what Namespaces we added
subordinate_namespace_names = []
# save the template for future reference
namespace_template = template_for_namespace
# for display
original_input = class_list_str.replace('\n', '\\n')
# for each class in the class list
class_list = []
for namespace_index, class_list_element in enumerate(
class_str_list):
try:
a_class = class_converter(
class_extractor(class_list_element))
except AttributeError:
raise JobNotFoundError(class_list_element)
class_list.append((a_class.__name__, a_class))
# figure out the Namespace name
namespace_name_dict = {
'name': a_class.__name__,
'index': namespace_index
}
namespace_name = template_for_namespace % namespace_name_dict
subordinate_namespace_names.append(namespace_name)
# create the new Namespace
required_config.namespace(namespace_name)
a_class_namespace = required_config[namespace_name]
# add options for the 'extra data'
try:
extra_options = extra_extractor(class_list_element)
a_class_namespace.update(extra_options)
except NotImplementedError:
pass
# add options frr the classes required config
try:
for k, v in a_class.get_required_config().iteritems():
if k not in reference_namespace:
a_class_namespace[k] = v
except AttributeError: # a_class has no get_required_config
pass
@classmethod
def to_str(cls):
"""this method takes this inner class object and turns it back
into the original string of classnames. This is used
primarily as for the output of the 'help' option"""
return cls.original_input
def jobs_converter(path_or_jobs):
"""Takes a Python dotted path or a jobs spec and returns crontabber jobs
Example Python dotted path::
jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS')
Example jobs spec::
jobs_converter('socorro.cron.jobs.archivescraper.ArchiveScraperCronApp|1h')
:arg str path_or_jobs: a Python dotted path or a crontabber jobs spec
:returns: crontabber jobs InnerClassList
"""
if '|' not in path_or_jobs:
# NOTE(willkg): configman's class_converter returns the value pointed
# to by a Python dotted path
input_str = class_converter(path_or_jobs)
else:
input_str = path_or_jobs
# At this point, we have a big string of jobs
from_string_converter = classes_in_namespaces_converter_with_compression(
reference_namespace=Namespace()
)
return from_string_converter(input_str)
def main(initial_app, values_source_list=None, config_path=None):
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
if config_path is None:
default = './config'
config_path = os.environ.get(
'DEFAULT_SOCORRO_CONFIG_PATH',
default
)
if config_path != default:
# you tried to set it, then it must be a valid directory
if not os.path.isdir(config_path):
raise IOError('%s is not a valid directory' % config_path)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the application',
default=initial_app,
from_string_converter=class_converter
)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError, x:
raise AppDetailMissingError(str(x))
def test_basic_07_argparse_multilevel_class_expansion(self):
option_definitions = self.setup_configman_namespace()
other_value_source = {
"gamma": [38, 28, 18, 8]
}
other_definition_source = Namespace()
other_definition_source.add_option(
"a_class",
default="configman.tests.test_val_for_modules.Alpha",
from_string_converter=class_converter
)
cm = ConfigurationManager(
definition_source=[option_definitions, other_definition_source],
values_source_list=[other_value_source, command_line],
argv_source=[
"0",
"--admin.expose_secrets",
"--delta",
'--gamma="8 18 28 38"',
'--a_class=configman.tests.test_val_for_modules.Delta',
'--messy=34'
],
use_auto_help=False,
)
config = cm.get_config()
expected = {
"alpha": 0,
"beta": 'the second',
"gamma": [8, 18, 28, 38],
"delta": True,
"admin.print_conf": None,
"admin.dump_conf": '',
"admin.strict": False,
"admin.expose_secrets": True,
"a_class": class_converter(
"configman.tests.test_val_for_modules.Delta"
),
"messy": 34,
"dd": class_converter(
"configman.tests.test_val_for_modules.Beta"
),
'b': 23,
}
for k in config.keys_breadth_first():
self.assertEqual(config[k], expected[k])
def main(app_object=None):
if isinstance(app_object, basestring):
app_object = class_converter(app_object)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.admin = admin = Namespace()
admin.add_option('application',
doc='the fully qualified module or class of the '
'application',
default=app_object,
from_string_converter=class_converter
)
app_name = getattr(app_object, 'app_name', 'unknown')
app_version = getattr(app_object, 'app_version', '0.0')
app_description = getattr(app_object, 'app_description', 'no idea')
# create an iterable collection of value sources
# the order is important as these will supply values for the sources
# defined in the_definition_source. The values will be overlain in turn.
# First the os.environ values will be applied. Then any values from an ini
# file parsed by getopt. Finally any values supplied on the command line
# will be applied.
value_sources = (ConfigFileFutureProxy, # alias for allowing the user
# to specify a config file on
# the command line
environment, # alias for os.environ
command_line) # alias for getopt
# set up the manager with the definitions and values
# it isn't necessary to provide the app_name because the
# app_object passed in or loaded by the ConfigurationManager will alredy
# have that information.
config_manager = ConfigurationManager(app_definition,
value_sources,
app_name=app_name,
app_version=app_version,
app_description=app_description,
)
config = config_manager.get_config()
app_object = config.admin.application
if isinstance(app_object, type):
# invocation of the app if the app_object was a class
instance = app_object(config)
instance.main()
elif inspect.ismodule(app_object):
# invocation of the app if the app_object was a module
app_object.main(config)
elif inspect.isfunction(app_object):
# invocation of the app if the app_object was a function
app_object(config)
def main(app_object=None):
if isinstance(app_object, basestring):
app_object = class_converter(app_object)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.admin = admin = Namespace()
admin.add_option('application',
doc='the fully qualified module or class of the '
'application',
default=app_object,
from_string_converter=class_converter)
app_name = getattr(app_object, 'app_name', 'unknown')
app_version = getattr(app_object, 'app_version', '0.0')
app_description = getattr(app_object, 'app_description', 'no idea')
# create an iterable collection of value sources
# the order is important as these will supply values for the sources
# defined in the_definition_source. The values will be overlain in turn.
# First the os.environ values will be applied. Then any values from an ini
# file parsed by getopt. Finally any values supplied on the command line
# will be applied.
value_sources = (
ConfigFileFutureProxy, # alias for allowing the user
# to specify a config file on
# the command line
environment, # alias for os.environ
command_line) # alias for getopt
# set up the manager with the definitions and values
# it isn't necessary to provide the app_name because the
# app_object passed in or loaded by the ConfigurationManager will alredy
# have that information.
config_manager = ConfigurationManager(
app_definition,
value_sources,
app_name=app_name,
app_version=app_version,
app_description=app_description,
)
config = config_manager.get_config()
app_object = config.admin.application
if isinstance(app_object, type):
# invocation of the app if the app_object was a class
instance = app_object(config)
instance.main()
elif inspect.ismodule(app_object):
# invocation of the app if the app_object was a module
app_object.main(config)
elif inspect.isfunction(app_object):
# invocation of the app if the app_object was a function
app_object(config)
def action_converter(action):
try:
return action_dispatch[action]
except KeyError:
try:
f = class_converter(action)
except Exception:
raise Exception("'%s' is not a valid action" % action)
if f in action_dispatch.values():
return f
raise Exception("'%s' is not a valid action" % action)
def action_converter(action):
try:
return action_dispatch[action]
except KeyError:
try:
f = class_converter(action)
except Exception:
raise Exception("'%s' is not a valid action" % action)
if f in action_dispatch.values():
return f
raise Exception("'%s' is not a valid action" % action)
def __init__(self, source, the_config_manager=None):
if isinstance(source, (six.binary_type, six.text_type)):
source = class_converter(source)
module_as_dotdict = DotDict()
try:
ignore_symbol_list = source.ignore_symbol_list
if 'ignore_symbol_list' not in ignore_symbol_list:
ignore_symbol_list.append('ignore_symbol_list')
except AttributeError:
ignore_symbol_list = []
try:
self.always_ignore_mismatches = source.always_ignore_mismatches
except AttributeError:
pass # don't need to do anything - mismatches will not be ignored
for key, value in six.iteritems(source.__dict__):
if key.startswith('__') and key != "__doc__":
continue
if key in ignore_symbol_list:
continue
module_as_dotdict[key] = value
self.module = source
self.source = module_as_dotdict
def main(initial_app, values_source_list=None, config_path='./config'):
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the application',
default=initial_app,
from_string_converter=class_converter
)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError, x:
raise AppDetailMissingError(str(x))
def _do_main(
initial_app,
values_source_list=None,
config_path=None,
config_manager_cls=ConfigurationManager
):
global restart
restart = False
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
if config_path is None:
default = './config'
config_path = os.environ.get(
'DEFAULT_SOCORRO_CONFIG_PATH',
default
)
if config_path != default:
# you tried to set it, then it must be a valid directory
if not os.path.isdir(config_path):
raise IOError('%s is not a valid directory' % config_path)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the application',
default=initial_app,
from_string_converter=class_converter
)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError as x:
raise AppDetailMissingError(x)
app_definition.add_aggregation(
'logger',
functools.partial(setup_logger, app_name)
)
definitions = (
app_definition,
logging_required_config(app_name)
)
config_manager = config_manager_cls(
definitions,
app_name=app_name,
app_version=app_version,
app_description=app_description,
values_source_list=values_source_list,
config_pathname=config_path
)
def fix_exit_code(code):
# some apps don't return a code so you might get None
# which isn't good enough to send to sys.exit()
if code is None:
return 0
return code
with config_manager.context() as config:
#config.logger.config = config
config.executor_identity = lambda: threading.currentThread().getName()
config_manager.log_config(config.logger)
# install the signal handler for SIGHUP to be the action defined in
# 'respond_to_SIGHUP'
respond_to_SIGHUP_with_logging = functools.partial(
respond_to_SIGHUP,
logger=config.logger
)
signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)
# get the app class from configman. Why bother since we have it aleady
# with the 'initial_app' name? In most cases initial_app == app,
# it might not always be that way. The user always has the ability
# to specify on the command line a new app class that will override
# 'initial_app'.
app = config.application
if isinstance(app, type):
# invocation of the app if the app_object was a class
instance = app(config)
instance.config_manager = config_manager
return_code = fix_exit_code(instance.main())
elif inspect.ismodule(app):
# invocation of the app if the app_object was a module
return_code = fix_exit_code(app.main(config))
elif inspect.isfunction(app):
# invocation of the app if the app_object was a function
return_code = fix_exit_code(app(config))
config.logger.info('done.')
return return_code
raise NotImplementedError("The app did not have a callable main function")
