示例#1
0
def jobs_converter(path_or_jobs):
    """Takes a Python dotted path or a jobs spec and returns crontabber jobs

    Example Python dotted path::

        jobs_converter('socorro.cron.crontabber_app.DEFAULT_JOBS')

    Example jobs spec::

        jobs_converter('socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1h')


    :arg str path_or_jobs: a Python dotted path or a crontabber jobs spec

    :returns: crontabber jobs InnerClassList

    """
    if '|' not in path_or_jobs:
        # NOTE(willkg): crontabber's class_converter returns the value pointed
        # to by a Python dotted path
        input_str = class_converter(path_or_jobs)
    else:
        input_str = path_or_jobs

    from_string_converter = classes_in_namespaces_converter_with_compression(
        reference_namespace=Namespace(),
        list_splitter_fn=line_splitter,
        class_extractor=pipe_splitter,
        extra_extractor=get_extra_as_options
    )
    return from_string_converter(input_str)
 def test_app_converter(self):
     eq_(
         self.proxy.str_to_application_class('CollectorApp'),
         class_converter('socorro.collector.collector_app.CollectorApp')
     )
     eq_(
         self.proxy.str_to_application_class('CrashMoverApp'),
         class_converter('socorro.collector.crashmover_app.CrashMoverApp')
     )
     eq_(
         self.proxy.str_to_application_class('SubmitterApp'),
         class_converter('socorro.collector.submitter_app.SubmitterApp')
     )
     #eq_(
         #self.proxy.str_to_application_class('CronTabberApp'),
         #class_converter('socorro.cron.crontabber_app.CronTabberApp')
     #)
     eq_(
         self.proxy.str_to_application_class('MiddlewareApp'),
         class_converter('socorro.middleware.middleware_app.MiddlewareApp')
     )
     eq_(
         self.proxy.str_to_application_class('ProcessorApp'),
         class_converter('socorro.processor.processor_app.ProcessorApp')
     )
     eq_(
         self.proxy.str_to_application_class('HBaseClientApp'),
         class_converter('socorro.external.hb.hbase_client.HBaseClientApp')
     )
     eq_(
         self.proxy.str_to_application_class(
             'socorro.external.hb.hbase_client.HBaseClientApp'
         ),
         class_converter('socorro.external.hb.hbase_client.HBaseClientApp')
     )
示例#3
0
    def __init__(self, predicate,
                       predicate_args,
                       predicate_kwargs,
                       action,
                       action_args,
                       action_kwargs):
        """construct a new predicate/action rule pair.
        input parameters:
            pedicate - the name of a function to serve as a predicate.  The
                       function must accept two dicts followed by any number
                       of constant args or kwargs
            predicate_args - arguments to be passed on to the predicate
                             function in addition to the two required dicts.
            predicate_kwargs - kwargs to be passed on to the predicate
                               function in addition to the two required dicts.
            action - the name of a function to be run if the predicate returns
                     True
            action_args - arguments to be passed on to the action function
                          in addition to the two required dicts
            action_kwargs - kwargs to be passed on to the action function in
                            addition to the two required dicts
        """
        try:
            self.predicate = configman.converters.class_converter(predicate)
        except TypeError:
            self.predicate = predicate

        try:
            if predicate_args in ('', None):
                self.predicate_args = ()
            elif isinstance(predicate_args, tuple):
                self.predicate_args = predicate_args
            else:
                self.predicate_args = tuple([eval(x.strip())
                                       for x in predicate_args.split(',')])
        except AttributeError:
            self.predicate_args = ()

        self.predicate_kwargs = kw_str_parse(predicate_kwargs)
        try:
            self.action = configman.class_converter(action)
        except TypeError:
            self.action = action
        try:
            if action_args in ('', None):
                self.action_args = ()
            elif isinstance(action_args, tuple):
                self.action_args = action_args
            else:
                self.action_args = tuple([eval(x.strip())
                                    for x in action_args.split(',')])
        except AttributeError:
            self.action_args = ()
        self.action_kwargs = kw_str_parse(action_kwargs)
 def test_app_converter(self):
     eq_(
         self.proxy.str_to_application_class('collector'),
         class_converter('socorro.collector.collector_app.CollectorApp')
     )
     eq_(
         self.proxy.str_to_application_class('crashmover'),
         class_converter('socorro.collector.crashmover_app.CrashMoverApp')
     )
     eq_(
         self.proxy.str_to_application_class('submitter'),
         class_converter('socorro.collector.submitter_app.SubmitterApp')
     )
     #eq_(
         #self.proxy.str_to_application_class('crontabber'),
         #class_converter('socorro.cron.crontabber_app.CronTabberApp')
     #)
     eq_(
         self.proxy.str_to_application_class('middleware'),
         class_converter('socorro.middleware.middleware_app.MiddlewareApp')
     )
     eq_(
         self.proxy.str_to_application_class('processor'),
         class_converter('socorro.processor.processor_app.ProcessorApp')
     )
     eq_(
         self.proxy.str_to_application_class(
             'socorro.external.hb.hbase_client.HBaseClientApp'
         ),
         class_converter('socorro.external.hb.hbase_client.HBaseClientApp')
     )
示例#5
0
        class InnerClassList(RequiredConfig):
            """This nested class is a proxy list for the classes.  It collects
            all the config requirements for the listed classes and places them
            each into their own Namespace.
            """
            # we're dynamically creating a class here.  The following block of
            # code is actually adding class level attributes to this new class

            # 1st requirement for configman
            required_config = Namespace()

            # to help the programmer know what Namespaces we added
            subordinate_namespace_names = []

            # save the template for future reference
            namespace_template = template_for_namespace

            # for display
            original_input = class_list_str.replace('\n', '\\n')

            # for each class in the class list
            class_list = []
            for namespace_index, class_list_element in enumerate(
                    class_str_list):
                a_class = class_converter(class_extractor(class_list_element))

                # figure out the Namespace name
                namespace_name_dict = {
                    'name': a_class.__name__,
                    'index': namespace_index
                }
                namespace_name = template_for_namespace % namespace_name_dict
                class_list.append((a_class.__name__, a_class, namespace_name))
                subordinate_namespace_names.append(namespace_name)
                # create the new Namespace
                required_config.namespace(namespace_name)
                a_class_namespace = required_config[namespace_name]
                a_class_namespace.add_option(
                    name_of_class_option,
                    doc='fully qualified classname',
                    default=class_list_element,
                    from_string_converter=class_converter,
                    likely_to_be_changed=True,
                )

            @classmethod
            def to_str(cls):
                """this method takes this inner class object and turns it back
                into the original string of classnames.  This is used
                primarily as for the output of the 'help' option"""
                return "'%s'" % cls.original_input
示例#6
0
        class InnerClassList(RequiredConfig):
            """This nested class is a proxy list for the classes.  It collects
            all the config requirements for the listed classes and places them
            each into their own Namespace.
            """
            # we're dynamically creating a class here.  The following block of
            # code is actually adding class level attributes to this new class

            # 1st requirement for configman
            required_config = Namespace()

            # to help the programmer know what Namespaces we added
            subordinate_namespace_names = []

            # save the template for future reference
            namespace_template = template_for_namespace

            # for display
            original_input = class_list_str.replace('\n', '\\n')

            # for each class in the class list
            class_list = []
            for namespace_index, class_list_element in enumerate(
                    class_str_list):
                a_class = class_converter(class_extractor(class_list_element))

                class_list.append((a_class.__name__, a_class))
                # figure out the Namespace name
                namespace_name_dict = {
                    'name': a_class.__name__,
                    'index': namespace_index
                }
                namespace_name = template_for_namespace % namespace_name_dict
                subordinate_namespace_names.append(namespace_name)
                # create the new Namespace
                required_config.namespace(namespace_name)
                a_class_namespace = required_config[namespace_name]
                # add options for the classes required config
                try:
                    for k, v in a_class.get_required_config().iteritems():
                        a_class_namespace[k] = v
                except AttributeError:  # a_class has no get_required_config
                    pass

            @classmethod
            def to_str(cls):
                """this method takes this inner class object and turns it back
                into the original string of classnames.  This is used
                primarily as for the output of the 'help' option"""
                return cls.original_input
示例#7
0
    def __init__(self, predicate, predicate_args, predicate_kwargs, action,
                 action_args, action_kwargs):
        """construct a new predicate/action rule pair.
        input parameters:
            pedicate - the name of a function to serve as a predicate.  The
                       function must accept two dicts followed by any number
                       of constant args or kwargs
            predicate_args - arguments to be passed on to the predicate
                             function in addition to the two required dicts.
            predicate_kwargs - kwargs to be passed on to the predicate
                               function in addition to the two required dicts.
            action - the name of a function to be run if the predicate returns
                     True
            action_args - arguments to be passed on to the action function
                          in addition to the two required dicts
            action_kwargs - kwargs to be passed on to the action function in
                            addition to the two required dicts
        """
        try:
            self.predicate = configman.converters.class_converter(predicate)
        except TypeError:
            self.predicate = predicate

        try:
            if predicate_args in ('', None):
                self.predicate_args = ()
            elif isinstance(predicate_args, tuple):
                self.predicate_args = predicate_args
            else:
                self.predicate_args = tuple(
                    [eval(x.strip()) for x in predicate_args.split(',')])
        except AttributeError:
            self.predicate_args = ()

        self.predicate_kwargs = kw_str_parse(predicate_kwargs)
        try:
            self.action = configman.class_converter(action)
        except TypeError:
            self.action = action
        try:
            if action_args in ('', None):
                self.action_args = ()
            elif isinstance(action_args, tuple):
                self.action_args = action_args
            else:
                self.action_args = tuple(
                    [eval(x.strip()) for x in action_args.split(',')])
        except AttributeError:
            self.action_args = ()
        self.action_kwargs = kw_str_parse(action_kwargs)
示例#8
0
def _do_main(
    initial_app,
    values_source_list=None,
    config_path=None,
    config_manager_cls=ConfigurationManager
):
    if values_source_list is None:
        values_source_list = [
            ConfigFileFutureProxy,
            environment,
            command_line
        ]


    global restart
    restart = False
    if isinstance(initial_app, basestring):
        initial_app = class_converter(initial_app)

    if config_path is None:
        default = './config'
        config_path = os.environ.get(
            'DEFAULT_SOCORRO_CONFIG_PATH',
            default
        )
        if config_path != default:
            # you tried to set it, then it must be a valid directory
            if not os.path.isdir(config_path):
                raise IOError('%s is not a valid directory' % config_path)

    # the only config parameter is a special one that refers to a class or
    # module that defines an application.  In order to qualify, a class must
    # have a constructor that accepts a DotDict derivative as the sole
    # input parameter.  It must also have a 'main' function that accepts no
    # parameters.  For a module to be acceptable, it must have a main
    # function that accepts a DotDict derivative as its input parameter.
    app_definition = Namespace()
    app_definition.add_option(
        'application',
        doc='the fully qualified module or class of the application',
        default=initial_app,
        from_string_converter=class_converter
    )
    try:
        app_name = initial_app.app_name  # this will be used as the default
                                         # b
        app_version = initial_app.app_version
        app_description = initial_app.app_description
    except AttributeError as x:
        raise AppDetailMissingError(x)

    app_definition.add_aggregation(
        'logger',
        functools.partial(setup_logger, app_name)
    )

    definitions = (
        app_definition,
        logging_required_config(app_name)
    )

    config_manager = config_manager_cls(
        definitions,
        app_name=app_name,
        app_version=app_version,
        app_description=app_description,
        values_source_list=values_source_list,
        config_pathname=config_path
    )

    def fix_exit_code(code):
        # some apps don't return a code so you might get None
        # which isn't good enough to send to sys.exit()
        if code is None:
            return 0
        return code

    with config_manager.context() as config:
        #config.logger.config = config
        config.executor_identity = lambda: threading.currentThread().getName()
        config_manager.log_config(config.logger)

        # install the signal handler for SIGHUP to be the action defined in
        # 'respond_to_SIGHUP'
        respond_to_SIGHUP_with_logging = functools.partial(
            respond_to_SIGHUP,
            logger=config.logger
        )
        signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)


        # get the app class from configman.  Why bother since we have it aleady
        # with the 'initial_app' name?  In most cases initial_app == app,
        # it might not always be that way.  The user always has the ability
        # to specify on the command line a new app class that will override
        # 'initial_app'.
        app = config.application

        if isinstance(app, type):
            # invocation of the app if the app_object was a class
            instance = app(config)
            instance.config_manager = config_manager
            return_code = fix_exit_code(instance.main())
        elif inspect.ismodule(app):
            # invocation of the app if the app_object was a module
            return_code = fix_exit_code(app.main(config))
        elif inspect.isfunction(app):
            # invocation of the app if the app_object was a function
            return_code = fix_exit_code(app(config))
        config.logger.info('done.')
        return return_code

    raise NotImplementedError("The app did not have a callable main function")
示例#9
0
    def __init__(self, predicate,
                       predicate_args,
                       predicate_kwargs,
                       action,
                       action_args,
                       action_kwargs,
                       config=None):
        """construct a new predicate/action rule pair.
        input parameters:
            pedicate - the name of a function to serve as a predicate.  The
                       function must accept two dicts followed by any number
                       of constant args or kwargs.  Alternatively, this could
                       be a classname for a class that has a method called
                       'predicate' with the aforementioned charactistics
            predicate_args - arguments to be passed on to the predicate
                             function in addition to the two required dicts.
            predicate_kwargs - kwargs to be passed on to the predicate
                               function in addition to the two required dicts.
            action - the name of a function to be run if the predicate returns
                     True.  The method must accept two dicts followed by
                     any number of args or kwargs.    Alternatively, this could
                     be a classname for a class that has a method called
                     'predicate' with the aforementioned charactistics
            action_args - arguments to be passed on to the action function
                          in addition to the two required dicts
            action_kwargs - kwargs to be passed on to the action function in
                            addition to the two required dicts
        """
        try:
            self.predicate = configman.converters.class_converter(predicate)
        except TypeError:
            # conversion failed, let's assume it was already function or a
            # callable object
            self.predicate = predicate
        if inspect.isclass(self.predicate):
            # the predicate is a class, instantiate it and set the predicate
            # function to the object's 'predicate' method
            self._predicate_implementation = self.predicate(config)
            self.predicate = self._predicate_implementation.predicate
        else:
            self._predicate_implementation = type(self.predicate)

        try:
            if predicate_args in ('', None):
                self.predicate_args = ()
            elif isinstance(predicate_args, tuple):
                self.predicate_args = predicate_args
            else:
                self.predicate_args = tuple(
                    [eval(x.strip()) for x in predicate_args.split(',')]
                )
        except AttributeError:
            self.predicate_args = ()

        self.predicate_kwargs = kw_str_parse(predicate_kwargs)
        try:
            self.action = configman.class_converter(action)
        except TypeError:
            # the conversion failed, let's assume that the action was passed in
            # as something callable.
            self.action = action
        if inspect.isclass(self.action):
            # the action is actually a class, go on and instantiate it, then
            # assign the 'action' to be the object's 'action' method
            if self._predicate_implementation.__class__ is self.action:
                # if the predicate and the action are implemented in the same
                # class, only instantiate one copy.
                self._action_implementation = self._predicate_implementation
            else:
                self._action_implementation = self.action(config)
            self.action = self._action_implementation.action

        try:
            if action_args in ('', None):
                self.action_args = ()
            elif isinstance(action_args, tuple):
                self.action_args = action_args
            else:
                self.action_args = tuple(
                    [eval(x.strip()) for x in action_args.split(',')]
                )
        except AttributeError:
            self.action_args = ()
        self.action_kwargs = kw_str_parse(action_kwargs)
示例#10
0
    def __init__(self, predicate, predicate_args, predicate_kwargs, action,
                 action_args, action_kwargs):
        """construct a new predicate/action rule pair.
        input parameters:
            pedicate - the name of a function to serve as a predicate.  The
                       function must accept two dicts followed by any number
                       of constant args or kwargs.  Alternatively, this could
                       be a classname for a class that has a method called
                       'predicate' with the aforementioned charactistics
            predicate_args - arguments to be passed on to the predicate
                             function in addition to the two required dicts.
            predicate_kwargs - kwargs to be passed on to the predicate
                               function in addition to the two required dicts.
            action - the name of a function to be run if the predicate returns
                     True.  The method must accept two dicts followed by
                     any number of args or kwargs.    Alternatively, this could
                     be a classname for a class that has a method called
                     'predicate' with the aforementioned charactistics
            action_args - arguments to be passed on to the action function
                          in addition to the two required dicts
            action_kwargs - kwargs to be passed on to the action function in
                            addition to the two required dicts
        """
        try:
            self.predicate = configman.converters.class_converter(predicate)
        except TypeError:
            # conversion failed, let's assume it was already function or a
            # callable object
            self.predicate = predicate
        if inspect.isclass(self.predicate):
            # the predicate is a class, instantiate it and set the predicate
            # function to the object's 'predicate' method
            self._predicitate_implementation = self.predicate()
            self.predicate = self._predicitate_implementation.predicate
        else:
            self._predicitate_implementation = type(self.predicate)

        try:
            if predicate_args in ('', None):
                self.predicate_args = ()
            elif isinstance(predicate_args, tuple):
                self.predicate_args = predicate_args
            else:
                self.predicate_args = tuple(
                    [eval(x.strip()) for x in predicate_args.split(',')])
        except AttributeError:
            self.predicate_args = ()

        self.predicate_kwargs = kw_str_parse(predicate_kwargs)
        try:
            self.action = configman.class_converter(action)
        except TypeError:
            # the conversion failed, let's assume that the action was passed in
            # as something callable.
            self.action = action
        if inspect.isclass(self.action):
            # the action is actually a class, go on and instantiate it, then
            # assign the 'action' to be the object's 'action' method
            if self._predicitate_implementation.__class__ is self.action:
                # if the predicate and the action are implemented in the same
                # class, only instantiate one copy.
                self._action_implementation = self._predicitate_implementation
            else:
                self._action_implementation = self.action()
            self.action = self._action_implementation.action

        try:
            if action_args in ('', None):
                self.action_args = ()
            elif isinstance(action_args, tuple):
                self.action_args = action_args
            else:
                self.action_args = tuple(
                    [eval(x.strip()) for x in action_args.split(',')])
        except AttributeError:
            self.action_args = ()
        self.action_kwargs = kw_str_parse(action_kwargs)
示例#11
0
def _do_main(initial_app,
             values_source_list=None,
             config_path=None,
             config_manager_cls=ConfigurationManager):
    if values_source_list is None:
        values_source_list = [ConfigFileFutureProxy, environment, command_line]

    global restart
    restart = False
    if isinstance(initial_app, basestring):
        initial_app = class_converter(initial_app)

    if config_path is None:
        default = './config'
        config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH', default)
        if config_path != default:
            # you tried to set it, then it must be a valid directory
            if not os.path.isdir(config_path):
                raise IOError('%s is not a valid directory' % config_path)

    # the only config parameter is a special one that refers to a class or
    # module that defines an application.  In order to qualify, a class must
    # have a constructor that accepts a DotDict derivative as the sole
    # input parameter.  It must also have a 'main' function that accepts no
    # parameters.  For a module to be acceptable, it must have a main
    # function that accepts a DotDict derivative as its input parameter.
    app_definition = Namespace()
    app_definition.add_option(
        'application',
        doc='the fully qualified module or class of the application',
        default=initial_app,
        from_string_converter=class_converter)
    try:
        app_name = initial_app.app_name  # this will be used as the default
        # b
        app_version = initial_app.app_version
        app_description = initial_app.app_description
    except AttributeError as x:
        raise AppDetailMissingError(x)

    app_definition.add_aggregation('logger',
                                   functools.partial(setup_logger, app_name))

    definitions = (app_definition, logging_required_config(app_name))

    config_manager = config_manager_cls(definitions,
                                        app_name=app_name,
                                        app_version=app_version,
                                        app_description=app_description,
                                        values_source_list=values_source_list,
                                        config_pathname=config_path)

    def fix_exit_code(code):
        # some apps don't return a code so you might get None
        # which isn't good enough to send to sys.exit()
        if code is None:
            return 0
        return code

    with config_manager.context() as config:
        #config.logger.config = config
        config.executor_identity = lambda: threading.currentThread().getName()
        config_manager.log_config(config.logger)

        # install the signal handler for SIGHUP to be the action defined in
        # 'respond_to_SIGHUP'
        respond_to_SIGHUP_with_logging = functools.partial(
            respond_to_SIGHUP, logger=config.logger)
        signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)

        # get the app class from configman.  Why bother since we have it aleady
        # with the 'initial_app' name?  In most cases initial_app == app,
        # it might not always be that way.  The user always has the ability
        # to specify on the command line a new app class that will override
        # 'initial_app'.
        app = config.application

        if isinstance(app, type):
            # invocation of the app if the app_object was a class
            instance = app(config)
            instance.config_manager = config_manager
            return_code = fix_exit_code(instance.main())
        elif inspect.ismodule(app):
            # invocation of the app if the app_object was a module
            return_code = fix_exit_code(app.main(config))
        elif inspect.isfunction(app):
            # invocation of the app if the app_object was a function
            return_code = fix_exit_code(app(config))
        config.logger.info('done.')
        return return_code

    raise NotImplementedError("The app did not have a callable main function")