def test_do_run(self): config = DotDict() with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm: cm.return_value.context.return_value = mock.MagicMock() with mock.patch('socorro.app.socorro_app.signal') as s: class SomeOtherApp(SocorroApp): app_name = 'SomeOtherApp' app_verision = '1.2.3' app_description = 'a silly app' def main(self): expected = cm.return_value.context.return_value.__enter__.return_value assert self.config is expected return 17 result = main(SomeOtherApp) args = cm.call_args_list args, kwargs = args[0] assert isinstance(args[0], Namespace) assert isinstance(kwargs['values_source_list'], list) assert kwargs['app_name'] == SomeOtherApp.app_name assert kwargs['app_version'] == SomeOtherApp.app_version assert kwargs[ 'app_description'] == SomeOtherApp.app_description assert kwargs['config_pathname'] == './config' assert kwargs['values_source_list'][-1] == command_line assert isinstance(kwargs['values_source_list'][-2], DotDict) assert kwargs['values_source_list'][-3] is ConfigFileFutureProxy assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy) assert result == 17
def test_do_run_with_alternate_values_source_list(self): with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm: cm.return_value.context.return_value = mock.MagicMock() with mock.patch('socorro.app.socorro_app.signal'): class SomeOtherApp(SocorroApp): app_name = 'SomeOtherApp' app_verision = '1.2.3' app_description = 'a silly app' def main(self): expected = cm.return_value.context.return_value.__enter__.return_value assert self.config is expected return 17 result = main( SomeOtherApp, config_path='my/other/path', values_source_list=[{"a": 1}, {"b": 2}] ) args = cm.call_args_list args, kwargs = args[0] assert isinstance(args[0], Namespace) assert kwargs['app_name'] == SomeOtherApp.app_name assert kwargs['app_version'] == SomeOtherApp.app_version assert kwargs['app_description'] == SomeOtherApp.app_description assert kwargs['config_pathname'] == 'my/other/path' assert isinstance(kwargs['values_source_list'], list) assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy) assert kwargs['values_source_list'][1] == {"a": 1} assert kwargs['values_source_list'][2] == {"b": 2} assert result == 17
def test_do_run_with_alternate_class_path(self): config = DotDict() with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm: cm.return_value.context.return_value = mock.MagicMock() with mock.patch('socorro.app.socorro_app.signal') as s: class SomeOtherApp(SocorroApp): app_name = 'SomeOtherApp' app_verision = '1.2.3' app_description = 'a silly app' def main(self): ok_(self.config is cm.return_value.context. return_value.__enter__.return_value) return 17 result = main(SomeOtherApp, 'my/other/path') args = cm.call_args_list args, kwargs = args[0] ok_(isinstance(args[0], Namespace)) ok_(isinstance(kwargs['values_source_list'], list)) eq_(kwargs['app_name'], SomeOtherApp.app_name) eq_(kwargs['app_version'], SomeOtherApp.app_version) eq_(kwargs['app_description'], SomeOtherApp.app_description) eq_(kwargs['config_pathname'], 'my/other/path') ok_(kwargs['values_source_list'][-1], command_line) ok_(isinstance(kwargs['values_source_list'][-2], DotDict)) ok_(kwargs['values_source_list'][-3] is ConfigFileFutureProxy) ok_( isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy)) eq_(result, 17)
def test_do_run(self): config = DotDict() with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm: cm.return_value.context.return_value = mock.MagicMock() with mock.patch('socorro.app.socorro_app.signal') as s: class SomeOtherApp(SocorroApp): app_name='SomeOtherApp' app_verision='1.2.3' app_description='a silly app' def main(self): ok_( self.config is cm.return_value.context.return_value.__enter__ .return_value ) return 17 result = main(SomeOtherApp) args = cm.call_args_list args, kwargs = args[0] ok_(isinstance(args[0], Namespace)) ok_(isinstance(kwargs['values_source_list'], list)) eq_(kwargs['app_name'], SomeOtherApp.app_name) eq_(kwargs['app_version'], SomeOtherApp.app_version) eq_(kwargs['app_description'], SomeOtherApp.app_description) eq_(kwargs['config_pathname'], './config') ok_(kwargs['values_source_list'][-1], command_line) ok_(isinstance(kwargs['values_source_list'][-2], DotDict)) ok_(kwargs['values_source_list'][-3] is ConfigFileFutureProxy) ok_(isinstance( kwargs['values_source_list'][0], ApplicationDefaultsProxy )) eq_(result, 17)
def test_do_run_with_alternate_values_source_list(self): config = DotDict() with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm: cm.return_value.context.return_value = mock.MagicMock() with mock.patch('socorro.app.socorro_app.signal') as s: class SomeOtherApp(SocorroApp): app_name='SomeOtherApp' app_verision='1.2.3' app_description='a silly app' def main(self): expected = cm.return_value.context.return_value.__enter__.return_value assert self.config is expected return 17 result = main( SomeOtherApp, config_path='my/other/path', values_source_list=[{"a": 1}, {"b": 2}] ) args = cm.call_args_list args, kwargs = args[0] assert isinstance(args[0], Namespace) assert kwargs['app_name'] == SomeOtherApp.app_name assert kwargs['app_version'] == SomeOtherApp.app_version assert kwargs['app_description'] == SomeOtherApp.app_description assert kwargs['config_pathname'] == 'my/other/path' assert isinstance(kwargs['values_source_list'], list) assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy) assert kwargs['values_source_list'][1] == {"a": 1} assert kwargs['values_source_list'][2] == {"b": 2} assert result == 17
if not self.config.get('no_staticdata'): self.import_staticdata(db) if self.config['fakedata']: self.generate_fakedata(db, self.config['fakedata_days']) db.commit() command.stamp(alembic_cfg, "heads") db.session.close() # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # database owner section # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - self.config.logger.info( 'database extensions section with %s', superuser_normaldb_pg_url ) with PostgreSQLAlchemyManager( superuser_normaldb_pg_url, self.config.logger, autocommit=False, on_heroku=self.config.on_heroku ) as db: db.set_table_owner(self.config.database_username) db.set_default_owner(database_name, self.config.database_username) db.set_grants(self.config) # config has user lists return 0 if __name__ == "__main__": sys.exit(main(SocorroDBApp))
""" required_config = Namespace() required_config.add_option('date', default=datetime.datetime.utcnow().date(), doc='Date to run for', from_string_converter=string_to_datetime) required_config.add_option( 'crontabber_job_class', default='socorro.cron.jobs.ftpscraper.FTPScraperCronApp', doc='bla', from_string_converter=class_converter, ) @staticmethod def get_application_defaults(): return {'database.database_class': mock.MagicMock()} def __init__(self, config): self.config = config self.config.dry_run = True self.ftpscraper = config.crontabber_job_class(config, {}) def main(self): assert self.config.dry_run self.ftpscraper.run(self.config.date) if __name__ == '__main__': # pragma: no cover sys.exit(main(FTPScraperCronAppDryRunner))
from configman import Namespace class ExampleApp(App): app_name = 'example' app_version = '0.1' app_description = __doc__ # in this section, define any configuration requirements required_config = Namespace() required_config.add_option('name', default='Wilma', doc='a name to echo') required_config.add_option('time', default=datetime.datetime.now(), doc='the time of day') # implementing this constructor is only necessary when there is more # initialization to be done before main can be called # def __init__(self, config): # super(ExampleApp,self).__init__(config) def main(self): # this is where we'd implement the app # the configuraton is already setup as self.config print('hello, %s. The time is: %s' % (self.config.name, self.config.time)) if __name__ == '__main__': main(ExampleApp)
self.config.processor_name = self.app_instance_name # this function will be called by the MainThread periodically # while the threaded_task_manager processes crashes. self.waiting_func = None self.processor = self.config.processor.processor_class( self.config.processor, self.quit_check ) def close(self): """when the processor shutsdown, this function cleans up""" try: self.companion_process.close() except AttributeError: # there is either no companion or it doesn't have a close method # we can skip on pass try: self.processor.close() except AttributeError: # the processor implementation does not have a close method # we can blithely skip on pass if __name__ == '__main__': main(ProcessorApp)
#-------------------------------------------------------------------------- def find_method(self, a_method_name): try: return getattr(self, a_method_name + '_command') except AttributeError: return partial(self.method_not_found, a_method_name) #-------------------------------------------------------------------------- def main(self): self.crash_store = self.config.crashstorage_class(self.config) command = self.find_method(self.config.command) result = command() if result is not None: print result commands = [] for a_symbol in dir(FetchApp): if a_symbol.endswith('_command'): commands.append(a_symbol.replace('_command', '')) FetchApp.required_config.command.doc = ( FetchApp.required_config.command.doc % ', '.join(commands) ) if __name__ == '__main__': main(FetchApp)
class MissingSymbolsCronAppDryRunner(App): # pragma: no cover """App to test running missing-symbols right here right now. To run it, simply execute this file: $ python socorro/cron/jobs/missingsymbols.py """ required_config = Namespace() required_config.add_option( 'crontabber_job_class', default='socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp', doc='bla', from_string_converter=class_converter, ) def __init__(self, config): self.config = config self.app = config.crontabber_job_class(config, {}) def main(self): self.app.run() if __name__ == '__main__': # pragma: no cover import sys sys.exit(main(MissingSymbolsCronAppDryRunner))
#-------------------------------------------------------------------------- def find_method(self, a_method_name): try: return getattr(self, a_method_name + '_command') except AttributeError: return partial(self.method_not_found, a_method_name) #-------------------------------------------------------------------------- def main(self): self.crash_store = self.config.crashstorage_class(self.config) command = self.find_method(self.config.command) result = command() if result is not None: print result commands = [] for a_symbol in dir(FetchApp): if a_symbol.endswith('_command'): commands.append(a_symbol.replace('_command', '')) FetchApp.required_config.command.doc = ( FetchApp.required_config.command.doc % ', '.join(commands) ) if __name__ == '__main__': main(FetchApp)
class MissingSymbolsCronAppDryRunner(App): # pragma: no cover """App to test running missing-symbols right here right now. To run it, simply execute this file: $ python socorro/cron/jobs/missingsymbols.py """ required_config = Namespace() required_config.add_option( 'crontabber_job_class', default='socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp', doc='bla', from_string_converter=class_converter, ) def __init__(self, config): self.config = config self.app = config.crontabber_job_class(config, {}) def main(self): self.app.run() if __name__ == '__main__': # pragma: no cover import sys sys.exit(main(MissingSymbolsCronAppDryRunner))
import os from socorro.app.socorro_app import main from socorro.middleware.middleware_app import MiddlewareApp from socorro.webapi.servers import WSGIServer import socorro.middleware.middleware_app from configman import ( ConfigFileFutureProxy, environment ) if os.path.isfile('/etc/socorro/middleware.ini'): config_path = '/etc/socorro' else: config_path = WSGIServer.get_socorro_config_path(__file__) # invoke the socorro main function to create the configman app class and which # will then create the wsgi app object. main( MiddlewareApp, # the socorro app class config_path=config_path, values_source_list=[ ConfigFileFutureProxy, environment ] ) application = socorro.middleware.middleware_app.application
alembic_cfg = Config(self.config.alembic_config) alembic_cfg.set_main_option("sqlalchemy.url", sa_url) with PostgreSQLAlchemyManager(sa_url, self.config.logger) as db: connection = db.engine.connect() db.setup_admin() if self.no_schema: db.commit() return 0 db.create_types() db.create_procs() db.set_sequence_owner('breakpad_rw') db.commit() db.create_tables() db.set_table_owner('breakpad_rw') db.create_views() db.commit() db.set_grants(self.config) # config has user lists if not self.config['no_staticdata']: self.import_staticdata(db) if self.config['fakedata']: self.generate_fakedata(db, self.config['fakedata_days']) db.commit() command.stamp(alembic_cfg, "head") db.set_default_owner(self.database_name) db.session.close() return 0 if __name__ == "__main__": sys.exit(main(SocorroDBApp))
'date', default=datetime.datetime.utcnow().date(), doc='Date to run for', from_string_converter=string_to_datetime ) required_config.add_option( 'crontabber_job_class', default='socorro.cron.jobs.ftpscraper.FTPScraperCronApp', doc='bla', from_string_converter=class_converter, ) @staticmethod def get_application_defaults(): return { 'database.database_class': mock.MagicMock() } def __init__(self, config): self.config = config self.config.dry_run = True self.ftpscraper = config.crontabber_job_class(config, {}) def main(self): assert self.config.dry_run self.ftpscraper.run(self.config.date) if __name__ == '__main__': # pragma: no cover sys.exit(main(FTPScraperCronAppDryRunner))
main ) from socorro.webapi.servers import WSGIServer import socorro.collector.collector_app from configman import ( ConfigFileFutureProxy, environment ) if os.path.isfile('/etc/socorro/collector.ini'): config_path = '/etc/socorro' else: config_path = WSGIServer.get_socorro_config_path(__file__) # invoke the generic main function to create the configman app class and which # will then create the wsgi app object. main( # we use the generic Socorro App class. We'll rely on configuration to set # the 'application' class object to the appropriate collector_app class # for example, it could be "CollectorApp" or "Collector2015App" SocorroWelcomeApp, config_path=config_path, values_source_list=[ ConfigFileFutureProxy, environment ] ) application = socorro.collector.collector_app.application
host=config.host, port=config.port, virtual_host=config.virtual_host, credentials=pika.credentials.PlainCredentials( config.rabbitmq_user, config.rabbitmq_password))) except Exception: logging.error("Failed to connect") raise self.connection = connection def main(self): self.connect() channel = self.connection.channel() channel.queue_declare(queue='socorro.reprocessing', durable=True) with open(self.config.reprocesscrashlist.crashes, 'r') as file: for uuid in file.read().splitlines(): channel.basic_publish( exchange='', routing_key="socorro.reprocessing", body=uuid, properties=pika.BasicProperties(delivery_mode=2)) logging.debug('submitted %s' % uuid) self.connection.close() if __name__ == '__main__': main(ReprocessCrashlistApp)
def _action_after_iteration_completes(self): self.config.logger.info( 'the queuing iterator is exhausted - waiting to quit') self.task_manager.wait_for_empty_queue( 5, "waiting for the queue to drain before quitting") time.sleep(self.config.producer_consumer.number_of_threads * 2) def _filter_disallowed_values(self, current_value): """In this base class there are no disallowed values coming from the iterators. Other users of these iterator may have some standards and can detect and reject them here """ return current_value is None def _transform(self, crash_id): """Transfers raw data from the source to the destination without changing the data """ if self.config.submitter.dry_run: print(crash_id) else: raw_crash = self.source.get_raw_crash(crash_id) dumps = self.source.get_raw_dumps_as_files(crash_id) self.destination.save_raw_crash_with_file_dumps( raw_crash, dumps, crash_id) if __name__ == '__main__': main(SubmitterApp)