Exemplo n.º 1
0
    def test_do_run(self):
        config = DotDict()
        with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
            cm.return_value.context.return_value = mock.MagicMock()
            with mock.patch('socorro.app.socorro_app.signal') as s:

                class SomeOtherApp(SocorroApp):
                    app_name = 'SomeOtherApp'
                    app_verision = '1.2.3'
                    app_description = 'a silly app'

                    def main(self):
                        expected = cm.return_value.context.return_value.__enter__.return_value
                        assert self.config is expected
                        return 17

                result = main(SomeOtherApp)
                args = cm.call_args_list
                args, kwargs = args[0]
                assert isinstance(args[0], Namespace)
                assert isinstance(kwargs['values_source_list'], list)
                assert kwargs['app_name'] == SomeOtherApp.app_name
                assert kwargs['app_version'] == SomeOtherApp.app_version
                assert kwargs[
                    'app_description'] == SomeOtherApp.app_description
                assert kwargs['config_pathname'] == './config'
                assert kwargs['values_source_list'][-1] == command_line
                assert isinstance(kwargs['values_source_list'][-2], DotDict)
                assert kwargs['values_source_list'][-3] is ConfigFileFutureProxy
                assert isinstance(kwargs['values_source_list'][0],
                                  ApplicationDefaultsProxy)
                assert result == 17
Exemplo n.º 2
0
    def test_do_run_with_alternate_values_source_list(self):
        with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
            cm.return_value.context.return_value = mock.MagicMock()
            with mock.patch('socorro.app.socorro_app.signal'):
                class SomeOtherApp(SocorroApp):
                    app_name = 'SomeOtherApp'
                    app_verision = '1.2.3'
                    app_description = 'a silly app'

                    def main(self):
                        expected = cm.return_value.context.return_value.__enter__.return_value
                        assert self.config is expected
                        return 17

                result = main(
                    SomeOtherApp,
                    config_path='my/other/path',
                    values_source_list=[{"a": 1}, {"b": 2}]
                )

                args = cm.call_args_list
                args, kwargs = args[0]
                assert isinstance(args[0], Namespace)
                assert kwargs['app_name'] == SomeOtherApp.app_name
                assert kwargs['app_version'] == SomeOtherApp.app_version
                assert kwargs['app_description'] == SomeOtherApp.app_description
                assert kwargs['config_pathname'] == 'my/other/path'
                assert isinstance(kwargs['values_source_list'], list)
                assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy)
                assert kwargs['values_source_list'][1] == {"a": 1}
                assert kwargs['values_source_list'][2] == {"b": 2}
                assert result == 17
Exemplo n.º 3
0
    def test_do_run_with_alternate_class_path(self):
        config = DotDict()
        with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
            cm.return_value.context.return_value = mock.MagicMock()
            with mock.patch('socorro.app.socorro_app.signal') as s:

                class SomeOtherApp(SocorroApp):
                    app_name = 'SomeOtherApp'
                    app_verision = '1.2.3'
                    app_description = 'a silly app'

                    def main(self):
                        ok_(self.config is cm.return_value.context.
                            return_value.__enter__.return_value)
                        return 17

                result = main(SomeOtherApp, 'my/other/path')

                args = cm.call_args_list
                args, kwargs = args[0]
                ok_(isinstance(args[0], Namespace))
                ok_(isinstance(kwargs['values_source_list'], list))
                eq_(kwargs['app_name'], SomeOtherApp.app_name)
                eq_(kwargs['app_version'], SomeOtherApp.app_version)
                eq_(kwargs['app_description'], SomeOtherApp.app_description)
                eq_(kwargs['config_pathname'], 'my/other/path')
                ok_(kwargs['values_source_list'][-1], command_line)
                ok_(isinstance(kwargs['values_source_list'][-2], DotDict))
                ok_(kwargs['values_source_list'][-3] is ConfigFileFutureProxy)
                ok_(
                    isinstance(kwargs['values_source_list'][0],
                               ApplicationDefaultsProxy))
                eq_(result, 17)
Exemplo n.º 4
0
    def test_do_run(self):
        config = DotDict()
        with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
            cm.return_value.context.return_value = mock.MagicMock()
            with mock.patch('socorro.app.socorro_app.signal') as s:
                class SomeOtherApp(SocorroApp):
                    app_name='SomeOtherApp'
                    app_verision='1.2.3'
                    app_description='a silly app'
                    def main(self):
                        ok_(
                            self.config
                            is cm.return_value.context.return_value.__enter__
                                 .return_value
                        )
                        return 17

                result = main(SomeOtherApp)
                args = cm.call_args_list
                args, kwargs = args[0]
                ok_(isinstance(args[0], Namespace))
                ok_(isinstance(kwargs['values_source_list'], list))
                eq_(kwargs['app_name'], SomeOtherApp.app_name)
                eq_(kwargs['app_version'], SomeOtherApp.app_version)
                eq_(kwargs['app_description'], SomeOtherApp.app_description)
                eq_(kwargs['config_pathname'], './config')
                ok_(kwargs['values_source_list'][-1], command_line)
                ok_(isinstance(kwargs['values_source_list'][-2], DotDict))
                ok_(kwargs['values_source_list'][-3] is ConfigFileFutureProxy)
                ok_(isinstance(
                    kwargs['values_source_list'][0],
                    ApplicationDefaultsProxy
                ))
                eq_(result, 17)
Exemplo n.º 5
0
    def test_do_run_with_alternate_values_source_list(self):
        config = DotDict()
        with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
            cm.return_value.context.return_value = mock.MagicMock()
            with mock.patch('socorro.app.socorro_app.signal') as s:
                class SomeOtherApp(SocorroApp):
                    app_name='SomeOtherApp'
                    app_verision='1.2.3'
                    app_description='a silly app'
                    def main(self):
                        expected = cm.return_value.context.return_value.__enter__.return_value
                        assert self.config is expected
                        return 17

                result = main(
                    SomeOtherApp,
                    config_path='my/other/path',
                    values_source_list=[{"a": 1}, {"b": 2}]
                )

                args = cm.call_args_list
                args, kwargs = args[0]
                assert isinstance(args[0], Namespace)
                assert kwargs['app_name'] == SomeOtherApp.app_name
                assert kwargs['app_version'] == SomeOtherApp.app_version
                assert kwargs['app_description'] == SomeOtherApp.app_description
                assert kwargs['config_pathname'] == 'my/other/path'
                assert isinstance(kwargs['values_source_list'], list)
                assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy)
                assert kwargs['values_source_list'][1] == {"a": 1}
                assert kwargs['values_source_list'][2] == {"b": 2}
                assert result == 17
Exemplo n.º 6
0
            if not self.config.get('no_staticdata'):
                self.import_staticdata(db)
            if self.config['fakedata']:
                self.generate_fakedata(db, self.config['fakedata_days'])
            db.commit()
            command.stamp(alembic_cfg, "heads")
            db.session.close()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database owner section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'database extensions section with %s',
            superuser_normaldb_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_normaldb_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            db.set_table_owner(self.config.database_username)
            db.set_default_owner(database_name, self.config.database_username)
            db.set_grants(self.config)  # config has user lists

        return 0

if __name__ == "__main__":
    sys.exit(main(SocorroDBApp))
Exemplo n.º 7
0
    """

    required_config = Namespace()
    required_config.add_option('date',
                               default=datetime.datetime.utcnow().date(),
                               doc='Date to run for',
                               from_string_converter=string_to_datetime)
    required_config.add_option(
        'crontabber_job_class',
        default='socorro.cron.jobs.ftpscraper.FTPScraperCronApp',
        doc='bla',
        from_string_converter=class_converter,
    )

    @staticmethod
    def get_application_defaults():
        return {'database.database_class': mock.MagicMock()}

    def __init__(self, config):
        self.config = config
        self.config.dry_run = True
        self.ftpscraper = config.crontabber_job_class(config, {})

    def main(self):
        assert self.config.dry_run
        self.ftpscraper.run(self.config.date)


if __name__ == '__main__':  # pragma: no cover
    sys.exit(main(FTPScraperCronAppDryRunner))
Exemplo n.º 8
0
from configman import Namespace


class ExampleApp(App):
    app_name = 'example'
    app_version = '0.1'
    app_description = __doc__

    # in this section, define any configuration requirements
    required_config = Namespace()
    required_config.add_option('name', default='Wilma', doc='a name to echo')
    required_config.add_option('time',
                               default=datetime.datetime.now(),
                               doc='the time of day')

    # implementing this constructor is only necessary when there is more
    # initialization to be done before main can be called
    # def __init__(self, config):
    #    super(ExampleApp,self).__init__(config)

    def main(self):
        # this is where we'd implement the app
        # the configuraton is already setup as self.config
        print('hello, %s. The time is: %s' %
              (self.config.name, self.config.time))


if __name__ == '__main__':
    main(ExampleApp)
Exemplo n.º 9
0
        self.config.processor_name = self.app_instance_name

        # this function will be called by the MainThread periodically
        # while the threaded_task_manager processes crashes.
        self.waiting_func = None

        self.processor = self.config.processor.processor_class(
            self.config.processor,
            self.quit_check
        )

    def close(self):
        """when  the processor shutsdown, this function cleans up"""
        try:
            self.companion_process.close()
        except AttributeError:
            # there is either no companion or it doesn't have a close method
            # we can skip on
            pass
        try:
            self.processor.close()
        except AttributeError:
            # the processor implementation does not have a close method
            # we can blithely skip on
            pass


if __name__ == '__main__':
    main(ProcessorApp)
Exemplo n.º 10
0
    #--------------------------------------------------------------------------
    def find_method(self, a_method_name):
        try:
            return getattr(self, a_method_name + '_command')
        except AttributeError:
            return partial(self.method_not_found, a_method_name)

    #--------------------------------------------------------------------------
    def main(self):
        self.crash_store = self.config.crashstorage_class(self.config)
        command = self.find_method(self.config.command)
        result = command()
        if result is not None:
            print result


commands = []
for a_symbol in dir(FetchApp):
    if a_symbol.endswith('_command'):
        commands.append(a_symbol.replace('_command', ''))

FetchApp.required_config.command.doc = (
    FetchApp.required_config.command.doc % ', '.join(commands)
)



if __name__ == '__main__':
    main(FetchApp)
Exemplo n.º 11
0

class MissingSymbolsCronAppDryRunner(App):  # pragma: no cover
    """App to test running missing-symbols right here right now.

    To run it, simply execute this file:

        $ python socorro/cron/jobs/missingsymbols.py

    """

    required_config = Namespace()
    required_config.add_option(
        'crontabber_job_class',
        default='socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp',
        doc='bla',
        from_string_converter=class_converter,
    )

    def __init__(self, config):
        self.config = config
        self.app = config.crontabber_job_class(config, {})

    def main(self):
        self.app.run()


if __name__ == '__main__':  # pragma: no cover
    import sys
    sys.exit(main(MissingSymbolsCronAppDryRunner))
Exemplo n.º 12
0
    #--------------------------------------------------------------------------
    def find_method(self, a_method_name):
        try:
            return getattr(self, a_method_name + '_command')
        except AttributeError:
            return partial(self.method_not_found, a_method_name)

    #--------------------------------------------------------------------------
    def main(self):
        self.crash_store = self.config.crashstorage_class(self.config)
        command = self.find_method(self.config.command)
        result = command()
        if result is not None:
            print result


commands = []
for a_symbol in dir(FetchApp):
    if a_symbol.endswith('_command'):
        commands.append(a_symbol.replace('_command', ''))

FetchApp.required_config.command.doc = (
    FetchApp.required_config.command.doc % ', '.join(commands)
)



if __name__ == '__main__':
    main(FetchApp)
Exemplo n.º 13
0

class MissingSymbolsCronAppDryRunner(App):  # pragma: no cover
    """App to test running missing-symbols right here right now.

    To run it, simply execute this file:

        $ python socorro/cron/jobs/missingsymbols.py

    """

    required_config = Namespace()
    required_config.add_option(
        'crontabber_job_class',
        default='socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp',
        doc='bla',
        from_string_converter=class_converter,
    )

    def __init__(self, config):
        self.config = config
        self.app = config.crontabber_job_class(config, {})

    def main(self):
        self.app.run()


if __name__ == '__main__':  # pragma: no cover
    import sys
    sys.exit(main(MissingSymbolsCronAppDryRunner))
Exemplo n.º 14
0
import os
from socorro.app.socorro_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import WSGIServer
import socorro.middleware.middleware_app

from configman import (
    ConfigFileFutureProxy,
    environment
)

if os.path.isfile('/etc/socorro/middleware.ini'):
    config_path = '/etc/socorro'
else:
    config_path = WSGIServer.get_socorro_config_path(__file__)

# invoke the socorro main function to create the configman app class and which
# will then create the wsgi app object.
main(
    MiddlewareApp,  # the socorro app class
    config_path=config_path,
    values_source_list=[
        ConfigFileFutureProxy,
        environment
    ]
)

application = socorro.middleware.middleware_app.application
Exemplo n.º 15
0
        alembic_cfg = Config(self.config.alembic_config)
        alembic_cfg.set_main_option("sqlalchemy.url", sa_url)
        with PostgreSQLAlchemyManager(sa_url, self.config.logger) as db:
            connection = db.engine.connect()
            db.setup_admin()
            if self.no_schema:
                db.commit()
                return 0
            db.create_types()
            db.create_procs()
            db.set_sequence_owner('breakpad_rw')
            db.commit()
            db.create_tables()
            db.set_table_owner('breakpad_rw')
            db.create_views()
            db.commit()
            db.set_grants(self.config)  # config has user lists
            if not self.config['no_staticdata']:
                self.import_staticdata(db)
            if self.config['fakedata']:
                self.generate_fakedata(db, self.config['fakedata_days'])
            db.commit()
            command.stamp(alembic_cfg, "head")
            db.set_default_owner(self.database_name)
            db.session.close()

        return 0

if __name__ == "__main__":
    sys.exit(main(SocorroDBApp))
Exemplo n.º 16
0
        'date',
        default=datetime.datetime.utcnow().date(),
        doc='Date to run for',
        from_string_converter=string_to_datetime
    )
    required_config.add_option(
        'crontabber_job_class',
        default='socorro.cron.jobs.ftpscraper.FTPScraperCronApp',
        doc='bla',
        from_string_converter=class_converter,
    )

    @staticmethod
    def get_application_defaults():
        return {
            'database.database_class': mock.MagicMock()
        }

    def __init__(self, config):
        self.config = config
        self.config.dry_run = True
        self.ftpscraper = config.crontabber_job_class(config, {})

    def main(self):
        assert self.config.dry_run
        self.ftpscraper.run(self.config.date)


if __name__ == '__main__':  # pragma: no cover
    sys.exit(main(FTPScraperCronAppDryRunner))
Exemplo n.º 17
0
    main
)
from socorro.webapi.servers import WSGIServer
import socorro.collector.collector_app

from configman import (
    ConfigFileFutureProxy,
    environment
)

if os.path.isfile('/etc/socorro/collector.ini'):
    config_path = '/etc/socorro'
else:
    config_path = WSGIServer.get_socorro_config_path(__file__)

# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
    # we use the generic Socorro App class. We'll rely on configuration to set
    # the 'application' class object to the appropriate collector_app class
    # for example, it could be "CollectorApp" or "Collector2015App"
    SocorroWelcomeApp,
    config_path=config_path,
    values_source_list=[
        ConfigFileFutureProxy,
        environment
    ]
)

application = socorro.collector.collector_app.application
Exemplo n.º 18
0
                    host=config.host,
                    port=config.port,
                    virtual_host=config.virtual_host,
                    credentials=pika.credentials.PlainCredentials(
                        config.rabbitmq_user, config.rabbitmq_password)))
        except Exception:
            logging.error("Failed to connect")
            raise
        self.connection = connection

    def main(self):
        self.connect()
        channel = self.connection.channel()

        channel.queue_declare(queue='socorro.reprocessing', durable=True)

        with open(self.config.reprocesscrashlist.crashes, 'r') as file:
            for uuid in file.read().splitlines():
                channel.basic_publish(
                    exchange='',
                    routing_key="socorro.reprocessing",
                    body=uuid,
                    properties=pika.BasicProperties(delivery_mode=2))
                logging.debug('submitted %s' % uuid)

        self.connection.close()


if __name__ == '__main__':
    main(ReprocessCrashlistApp)
Exemplo n.º 19
0
    def _action_after_iteration_completes(self):
        self.config.logger.info(
            'the queuing iterator is exhausted - waiting to quit')
        self.task_manager.wait_for_empty_queue(
            5, "waiting for the queue to drain before quitting")
        time.sleep(self.config.producer_consumer.number_of_threads * 2)

    def _filter_disallowed_values(self, current_value):
        """In this base class there are no disallowed values coming from the
        iterators.  Other users of these iterator may have some standards and
        can detect and reject them here

        """
        return current_value is None

    def _transform(self, crash_id):
        """Transfers raw data from the source to the destination without changing the data

        """
        if self.config.submitter.dry_run:
            print(crash_id)
        else:
            raw_crash = self.source.get_raw_crash(crash_id)
            dumps = self.source.get_raw_dumps_as_files(crash_id)
            self.destination.save_raw_crash_with_file_dumps(
                raw_crash, dumps, crash_id)


if __name__ == '__main__':
    main(SubmitterApp)