def jobs_converter(path_or_jobs):
"""Takes a Python dotted path or a jobs spec and returns crontabber jobs
Example Python dotted path::
jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS')
Example jobs spec::
jobs_converter('socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1h')
:arg str path_or_jobs: a Python dotted path or a crontabber jobs spec
:returns: crontabber jobs InnerClassList
"""
if '|' not in path_or_jobs:
# NOTE(willkg): crontabber's class_converter returns the value pointed
# to by a Python dotted path
input_str = class_converter(path_or_jobs)
else:
input_str = path_or_jobs
from_string_converter = classes_in_namespaces_converter_with_compression(
reference_namespace=Namespace(),
list_splitter_fn=line_splitter,
class_extractor=pipe_splitter,
extra_extractor=get_extra_as_options
)
return from_string_converter(input_str)
def test_app_converter(self):
eq_(
self.proxy.str_to_application_class('CollectorApp'),
class_converter('socorro.collector.collector_app.CollectorApp')
)
eq_(
self.proxy.str_to_application_class('CrashMoverApp'),
class_converter('socorro.collector.crashmover_app.CrashMoverApp')
)
eq_(
self.proxy.str_to_application_class('SubmitterApp'),
class_converter('socorro.collector.submitter_app.SubmitterApp')
)
#eq_(
#self.proxy.str_to_application_class('CronTabberApp'),
#class_converter('socorro.cron.crontabber_app.CronTabberApp')
#)
eq_(
self.proxy.str_to_application_class('MiddlewareApp'),
class_converter('socorro.middleware.middleware_app.MiddlewareApp')
)
eq_(
self.proxy.str_to_application_class('ProcessorApp'),
class_converter('socorro.processor.processor_app.ProcessorApp')
)
eq_(
self.proxy.str_to_application_class('HBaseClientApp'),
class_converter('socorro.external.hb.hbase_client.HBaseClientApp')
)
eq_(
self.proxy.str_to_application_class(
'socorro.external.hb.hbase_client.HBaseClientApp'
),
class_converter('socorro.external.hb.hbase_client.HBaseClientApp')
)
def __init__(self, predicate,
predicate_args,
predicate_kwargs,
action,
action_args,
action_kwargs):
"""construct a new predicate/action rule pair.
input parameters:
pedicate - the name of a function to serve as a predicate. The
function must accept two dicts followed by any number
of constant args or kwargs
predicate_args - arguments to be passed on to the predicate
function in addition to the two required dicts.
predicate_kwargs - kwargs to be passed on to the predicate
function in addition to the two required dicts.
action - the name of a function to be run if the predicate returns
True
action_args - arguments to be passed on to the action function
in addition to the two required dicts
action_kwargs - kwargs to be passed on to the action function in
addition to the two required dicts
"""
try:
self.predicate = configman.converters.class_converter(predicate)
except TypeError:
self.predicate = predicate
try:
if predicate_args in ('', None):
self.predicate_args = ()
elif isinstance(predicate_args, tuple):
self.predicate_args = predicate_args
else:
self.predicate_args = tuple([eval(x.strip())
for x in predicate_args.split(',')])
except AttributeError:
self.predicate_args = ()
self.predicate_kwargs = kw_str_parse(predicate_kwargs)
try:
self.action = configman.class_converter(action)
except TypeError:
self.action = action
try:
if action_args in ('', None):
self.action_args = ()
elif isinstance(action_args, tuple):
self.action_args = action_args
else:
self.action_args = tuple([eval(x.strip())
for x in action_args.split(',')])
except AttributeError:
self.action_args = ()
self.action_kwargs = kw_str_parse(action_kwargs)
def test_app_converter(self):
eq_(
self.proxy.str_to_application_class('collector'),
class_converter('socorro.collector.collector_app.CollectorApp')
)
eq_(
self.proxy.str_to_application_class('crashmover'),
class_converter('socorro.collector.crashmover_app.CrashMoverApp')
)
eq_(
self.proxy.str_to_application_class('submitter'),
class_converter('socorro.collector.submitter_app.SubmitterApp')
)
#eq_(
#self.proxy.str_to_application_class('crontabber'),
#class_converter('socorro.cron.crontabber_app.CronTabberApp')
#)
eq_(
self.proxy.str_to_application_class('middleware'),
class_converter('socorro.middleware.middleware_app.MiddlewareApp')
)
eq_(
self.proxy.str_to_application_class('processor'),
class_converter('socorro.processor.processor_app.ProcessorApp')
)
eq_(
self.proxy.str_to_application_class(
'socorro.external.hb.hbase_client.HBaseClientApp'
),
class_converter('socorro.external.hb.hbase_client.HBaseClientApp')
)
class InnerClassList(RequiredConfig):
"""This nested class is a proxy list for the classes. It collects
all the config requirements for the listed classes and places them
each into their own Namespace.
"""
# we're dynamically creating a class here. The following block of
# code is actually adding class level attributes to this new class
# 1st requirement for configman
required_config = Namespace()
# to help the programmer know what Namespaces we added
subordinate_namespace_names = []
# save the template for future reference
namespace_template = template_for_namespace
# for display
original_input = class_list_str.replace('\n', '\\n')
# for each class in the class list
class_list = []
for namespace_index, class_list_element in enumerate(
class_str_list):
a_class = class_converter(class_extractor(class_list_element))
# figure out the Namespace name
namespace_name_dict = {
'name': a_class.__name__,
'index': namespace_index
}
namespace_name = template_for_namespace % namespace_name_dict
class_list.append((a_class.__name__, a_class, namespace_name))
subordinate_namespace_names.append(namespace_name)
# create the new Namespace
required_config.namespace(namespace_name)
a_class_namespace = required_config[namespace_name]
a_class_namespace.add_option(
name_of_class_option,
doc='fully qualified classname',
default=class_list_element,
from_string_converter=class_converter,
likely_to_be_changed=True,
)
@classmethod
def to_str(cls):
"""this method takes this inner class object and turns it back
into the original string of classnames. This is used
primarily as for the output of the 'help' option"""
return "'%s'" % cls.original_input
class InnerClassList(RequiredConfig):
"""This nested class is a proxy list for the classes. It collects
all the config requirements for the listed classes and places them
each into their own Namespace.
"""
# we're dynamically creating a class here. The following block of
# code is actually adding class level attributes to this new class
# 1st requirement for configman
required_config = Namespace()
# to help the programmer know what Namespaces we added
subordinate_namespace_names = []
# save the template for future reference
namespace_template = template_for_namespace
# for display
original_input = class_list_str.replace('\n', '\\n')
# for each class in the class list
class_list = []
for namespace_index, class_list_element in enumerate(
class_str_list):
a_class = class_converter(class_extractor(class_list_element))
class_list.append((a_class.__name__, a_class))
# figure out the Namespace name
namespace_name_dict = {
'name': a_class.__name__,
'index': namespace_index
}
namespace_name = template_for_namespace % namespace_name_dict
subordinate_namespace_names.append(namespace_name)
# create the new Namespace
required_config.namespace(namespace_name)
a_class_namespace = required_config[namespace_name]
# add options for the classes required config
try:
for k, v in a_class.get_required_config().iteritems():
a_class_namespace[k] = v
except AttributeError: # a_class has no get_required_config
pass
@classmethod
def to_str(cls):
"""this method takes this inner class object and turns it back
into the original string of classnames. This is used
primarily as for the output of the 'help' option"""
return cls.original_input
def __init__(self, predicate, predicate_args, predicate_kwargs, action,
action_args, action_kwargs):
"""construct a new predicate/action rule pair.
input parameters:
pedicate - the name of a function to serve as a predicate. The
function must accept two dicts followed by any number
of constant args or kwargs
predicate_args - arguments to be passed on to the predicate
function in addition to the two required dicts.
predicate_kwargs - kwargs to be passed on to the predicate
function in addition to the two required dicts.
action - the name of a function to be run if the predicate returns
True
action_args - arguments to be passed on to the action function
in addition to the two required dicts
action_kwargs - kwargs to be passed on to the action function in
addition to the two required dicts
"""
try:
self.predicate = configman.converters.class_converter(predicate)
except TypeError:
self.predicate = predicate
try:
if predicate_args in ('', None):
self.predicate_args = ()
elif isinstance(predicate_args, tuple):
self.predicate_args = predicate_args
else:
self.predicate_args = tuple(
[eval(x.strip()) for x in predicate_args.split(',')])
except AttributeError:
self.predicate_args = ()
self.predicate_kwargs = kw_str_parse(predicate_kwargs)
try:
self.action = configman.class_converter(action)
except TypeError:
self.action = action
try:
if action_args in ('', None):
self.action_args = ()
elif isinstance(action_args, tuple):
self.action_args = action_args
else:
self.action_args = tuple(
[eval(x.strip()) for x in action_args.split(',')])
except AttributeError:
self.action_args = ()
self.action_kwargs = kw_str_parse(action_kwargs)
def _do_main(
initial_app,
values_source_list=None,
config_path=None,
config_manager_cls=ConfigurationManager
):
if values_source_list is None:
values_source_list = [
ConfigFileFutureProxy,
environment,
command_line
]
global restart
restart = False
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
if config_path is None:
default = './config'
config_path = os.environ.get(
'DEFAULT_SOCORRO_CONFIG_PATH',
default
)
if config_path != default:
# you tried to set it, then it must be a valid directory
if not os.path.isdir(config_path):
raise IOError('%s is not a valid directory' % config_path)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the application',
default=initial_app,
from_string_converter=class_converter
)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError as x:
raise AppDetailMissingError(x)
app_definition.add_aggregation(
'logger',
functools.partial(setup_logger, app_name)
)
definitions = (
app_definition,
logging_required_config(app_name)
)
config_manager = config_manager_cls(
definitions,
app_name=app_name,
app_version=app_version,
app_description=app_description,
values_source_list=values_source_list,
config_pathname=config_path
)
def fix_exit_code(code):
# some apps don't return a code so you might get None
# which isn't good enough to send to sys.exit()
if code is None:
return 0
return code
with config_manager.context() as config:
#config.logger.config = config
config.executor_identity = lambda: threading.currentThread().getName()
config_manager.log_config(config.logger)
# install the signal handler for SIGHUP to be the action defined in
# 'respond_to_SIGHUP'
respond_to_SIGHUP_with_logging = functools.partial(
respond_to_SIGHUP,
logger=config.logger
)
signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)
# get the app class from configman. Why bother since we have it aleady
# with the 'initial_app' name? In most cases initial_app == app,
# it might not always be that way. The user always has the ability
# to specify on the command line a new app class that will override
# 'initial_app'.
app = config.application
if isinstance(app, type):
# invocation of the app if the app_object was a class
instance = app(config)
instance.config_manager = config_manager
return_code = fix_exit_code(instance.main())
elif inspect.ismodule(app):
# invocation of the app if the app_object was a module
return_code = fix_exit_code(app.main(config))
elif inspect.isfunction(app):
# invocation of the app if the app_object was a function
return_code = fix_exit_code(app(config))
config.logger.info('done.')
return return_code
raise NotImplementedError("The app did not have a callable main function")
def __init__(self, predicate,
predicate_args,
predicate_kwargs,
action,
action_args,
action_kwargs,
config=None):
"""construct a new predicate/action rule pair.
input parameters:
pedicate - the name of a function to serve as a predicate. The
function must accept two dicts followed by any number
of constant args or kwargs. Alternatively, this could
be a classname for a class that has a method called
'predicate' with the aforementioned charactistics
predicate_args - arguments to be passed on to the predicate
function in addition to the two required dicts.
predicate_kwargs - kwargs to be passed on to the predicate
function in addition to the two required dicts.
action - the name of a function to be run if the predicate returns
True. The method must accept two dicts followed by
any number of args or kwargs. Alternatively, this could
be a classname for a class that has a method called
'predicate' with the aforementioned charactistics
action_args - arguments to be passed on to the action function
in addition to the two required dicts
action_kwargs - kwargs to be passed on to the action function in
addition to the two required dicts
"""
try:
self.predicate = configman.converters.class_converter(predicate)
except TypeError:
# conversion failed, let's assume it was already function or a
# callable object
self.predicate = predicate
if inspect.isclass(self.predicate):
# the predicate is a class, instantiate it and set the predicate
# function to the object's 'predicate' method
self._predicate_implementation = self.predicate(config)
self.predicate = self._predicate_implementation.predicate
else:
self._predicate_implementation = type(self.predicate)
try:
if predicate_args in ('', None):
self.predicate_args = ()
elif isinstance(predicate_args, tuple):
self.predicate_args = predicate_args
else:
self.predicate_args = tuple(
[eval(x.strip()) for x in predicate_args.split(',')]
)
except AttributeError:
self.predicate_args = ()
self.predicate_kwargs = kw_str_parse(predicate_kwargs)
try:
self.action = configman.class_converter(action)
except TypeError:
# the conversion failed, let's assume that the action was passed in
# as something callable.
self.action = action
if inspect.isclass(self.action):
# the action is actually a class, go on and instantiate it, then
# assign the 'action' to be the object's 'action' method
if self._predicate_implementation.__class__ is self.action:
# if the predicate and the action are implemented in the same
# class, only instantiate one copy.
self._action_implementation = self._predicate_implementation
else:
self._action_implementation = self.action(config)
self.action = self._action_implementation.action
try:
if action_args in ('', None):
self.action_args = ()
elif isinstance(action_args, tuple):
self.action_args = action_args
else:
self.action_args = tuple(
[eval(x.strip()) for x in action_args.split(',')]
)
except AttributeError:
self.action_args = ()
self.action_kwargs = kw_str_parse(action_kwargs)
def __init__(self, predicate, predicate_args, predicate_kwargs, action,
action_args, action_kwargs):
"""construct a new predicate/action rule pair.
input parameters:
pedicate - the name of a function to serve as a predicate. The
function must accept two dicts followed by any number
of constant args or kwargs. Alternatively, this could
be a classname for a class that has a method called
'predicate' with the aforementioned charactistics
predicate_args - arguments to be passed on to the predicate
function in addition to the two required dicts.
predicate_kwargs - kwargs to be passed on to the predicate
function in addition to the two required dicts.
action - the name of a function to be run if the predicate returns
True. The method must accept two dicts followed by
any number of args or kwargs. Alternatively, this could
be a classname for a class that has a method called
'predicate' with the aforementioned charactistics
action_args - arguments to be passed on to the action function
in addition to the two required dicts
action_kwargs - kwargs to be passed on to the action function in
addition to the two required dicts
"""
try:
self.predicate = configman.converters.class_converter(predicate)
except TypeError:
# conversion failed, let's assume it was already function or a
# callable object
self.predicate = predicate
if inspect.isclass(self.predicate):
# the predicate is a class, instantiate it and set the predicate
# function to the object's 'predicate' method
self._predicitate_implementation = self.predicate()
self.predicate = self._predicitate_implementation.predicate
else:
self._predicitate_implementation = type(self.predicate)
try:
if predicate_args in ('', None):
self.predicate_args = ()
elif isinstance(predicate_args, tuple):
self.predicate_args = predicate_args
else:
self.predicate_args = tuple(
[eval(x.strip()) for x in predicate_args.split(',')])
except AttributeError:
self.predicate_args = ()
self.predicate_kwargs = kw_str_parse(predicate_kwargs)
try:
self.action = configman.class_converter(action)
except TypeError:
# the conversion failed, let's assume that the action was passed in
# as something callable.
self.action = action
if inspect.isclass(self.action):
# the action is actually a class, go on and instantiate it, then
# assign the 'action' to be the object's 'action' method
if self._predicitate_implementation.__class__ is self.action:
# if the predicate and the action are implemented in the same
# class, only instantiate one copy.
self._action_implementation = self._predicitate_implementation
else:
self._action_implementation = self.action()
self.action = self._action_implementation.action
try:
if action_args in ('', None):
self.action_args = ()
elif isinstance(action_args, tuple):
self.action_args = action_args
else:
self.action_args = tuple(
[eval(x.strip()) for x in action_args.split(',')])
except AttributeError:
self.action_args = ()
self.action_kwargs = kw_str_parse(action_kwargs)
def _do_main(initial_app,
values_source_list=None,
config_path=None,
config_manager_cls=ConfigurationManager):
if values_source_list is None:
values_source_list = [ConfigFileFutureProxy, environment, command_line]
global restart
restart = False
if isinstance(initial_app, basestring):
initial_app = class_converter(initial_app)
if config_path is None:
default = './config'
config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH', default)
if config_path != default:
# you tried to set it, then it must be a valid directory
if not os.path.isdir(config_path):
raise IOError('%s is not a valid directory' % config_path)
# the only config parameter is a special one that refers to a class or
# module that defines an application. In order to qualify, a class must
# have a constructor that accepts a DotDict derivative as the sole
# input parameter. It must also have a 'main' function that accepts no
# parameters. For a module to be acceptable, it must have a main
# function that accepts a DotDict derivative as its input parameter.
app_definition = Namespace()
app_definition.add_option(
'application',
doc='the fully qualified module or class of the application',
default=initial_app,
from_string_converter=class_converter)
try:
app_name = initial_app.app_name # this will be used as the default
# b
app_version = initial_app.app_version
app_description = initial_app.app_description
except AttributeError as x:
raise AppDetailMissingError(x)
app_definition.add_aggregation('logger',
functools.partial(setup_logger, app_name))
definitions = (app_definition, logging_required_config(app_name))
config_manager = config_manager_cls(definitions,
app_name=app_name,
app_version=app_version,
app_description=app_description,
values_source_list=values_source_list,
config_pathname=config_path)
def fix_exit_code(code):
# some apps don't return a code so you might get None
# which isn't good enough to send to sys.exit()
if code is None:
return 0
return code
with config_manager.context() as config:
#config.logger.config = config
config.executor_identity = lambda: threading.currentThread().getName()
config_manager.log_config(config.logger)
# install the signal handler for SIGHUP to be the action defined in
# 'respond_to_SIGHUP'
respond_to_SIGHUP_with_logging = functools.partial(
respond_to_SIGHUP, logger=config.logger)
signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)
# get the app class from configman. Why bother since we have it aleady
# with the 'initial_app' name? In most cases initial_app == app,
# it might not always be that way. The user always has the ability
# to specify on the command line a new app class that will override
# 'initial_app'.
app = config.application
if isinstance(app, type):
# invocation of the app if the app_object was a class
instance = app(config)
instance.config_manager = config_manager
return_code = fix_exit_code(instance.main())
elif inspect.ismodule(app):
# invocation of the app if the app_object was a module
return_code = fix_exit_code(app.main(config))
elif inspect.isfunction(app):
# invocation of the app if the app_object was a function
return_code = fix_exit_code(app(config))
config.logger.info('done.')
return return_code
raise NotImplementedError("The app did not have a callable main function")