def test_do_run(self):
config = DotDict()
with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
cm.return_value.context.return_value = mock.MagicMock()
with mock.patch('socorro.app.socorro_app.signal') as s:
class SomeOtherApp(SocorroApp):
app_name = 'SomeOtherApp'
app_verision = '1.2.3'
app_description = 'a silly app'
def main(self):
expected = cm.return_value.context.return_value.__enter__.return_value
assert self.config is expected
return 17
result = main(SomeOtherApp)
args = cm.call_args_list
args, kwargs = args[0]
assert isinstance(args[0], Namespace)
assert isinstance(kwargs['values_source_list'], list)
assert kwargs['app_name'] == SomeOtherApp.app_name
assert kwargs['app_version'] == SomeOtherApp.app_version
assert kwargs[
'app_description'] == SomeOtherApp.app_description
assert kwargs['config_pathname'] == './config'
assert kwargs['values_source_list'][-1] == command_line
assert isinstance(kwargs['values_source_list'][-2], DotDict)
assert kwargs['values_source_list'][-3] is ConfigFileFutureProxy
assert isinstance(kwargs['values_source_list'][0],
ApplicationDefaultsProxy)
assert result == 17
def test_do_run_with_alternate_values_source_list(self):
with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
cm.return_value.context.return_value = mock.MagicMock()
with mock.patch('socorro.app.socorro_app.signal'):
class SomeOtherApp(SocorroApp):
app_name = 'SomeOtherApp'
app_verision = '1.2.3'
app_description = 'a silly app'
def main(self):
expected = cm.return_value.context.return_value.__enter__.return_value
assert self.config is expected
return 17
result = main(
SomeOtherApp,
config_path='my/other/path',
values_source_list=[{"a": 1}, {"b": 2}]
)
args = cm.call_args_list
args, kwargs = args[0]
assert isinstance(args[0], Namespace)
assert kwargs['app_name'] == SomeOtherApp.app_name
assert kwargs['app_version'] == SomeOtherApp.app_version
assert kwargs['app_description'] == SomeOtherApp.app_description
assert kwargs['config_pathname'] == 'my/other/path'
assert isinstance(kwargs['values_source_list'], list)
assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy)
assert kwargs['values_source_list'][1] == {"a": 1}
assert kwargs['values_source_list'][2] == {"b": 2}
assert result == 17
def test_do_run_with_alternate_class_path(self):
config = DotDict()
with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
cm.return_value.context.return_value = mock.MagicMock()
with mock.patch('socorro.app.socorro_app.signal') as s:
class SomeOtherApp(SocorroApp):
app_name = 'SomeOtherApp'
app_verision = '1.2.3'
app_description = 'a silly app'
def main(self):
ok_(self.config is cm.return_value.context.
return_value.__enter__.return_value)
return 17
result = main(SomeOtherApp, 'my/other/path')
args = cm.call_args_list
args, kwargs = args[0]
ok_(isinstance(args[0], Namespace))
ok_(isinstance(kwargs['values_source_list'], list))
eq_(kwargs['app_name'], SomeOtherApp.app_name)
eq_(kwargs['app_version'], SomeOtherApp.app_version)
eq_(kwargs['app_description'], SomeOtherApp.app_description)
eq_(kwargs['config_pathname'], 'my/other/path')
ok_(kwargs['values_source_list'][-1], command_line)
ok_(isinstance(kwargs['values_source_list'][-2], DotDict))
ok_(kwargs['values_source_list'][-3] is ConfigFileFutureProxy)
ok_(
isinstance(kwargs['values_source_list'][0],
ApplicationDefaultsProxy))
eq_(result, 17)
def test_do_run(self):
config = DotDict()
with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
cm.return_value.context.return_value = mock.MagicMock()
with mock.patch('socorro.app.socorro_app.signal') as s:
class SomeOtherApp(SocorroApp):
app_name='SomeOtherApp'
app_verision='1.2.3'
app_description='a silly app'
def main(self):
ok_(
self.config
is cm.return_value.context.return_value.__enter__
.return_value
)
return 17
result = main(SomeOtherApp)
args = cm.call_args_list
args, kwargs = args[0]
ok_(isinstance(args[0], Namespace))
ok_(isinstance(kwargs['values_source_list'], list))
eq_(kwargs['app_name'], SomeOtherApp.app_name)
eq_(kwargs['app_version'], SomeOtherApp.app_version)
eq_(kwargs['app_description'], SomeOtherApp.app_description)
eq_(kwargs['config_pathname'], './config')
ok_(kwargs['values_source_list'][-1], command_line)
ok_(isinstance(kwargs['values_source_list'][-2], DotDict))
ok_(kwargs['values_source_list'][-3] is ConfigFileFutureProxy)
ok_(isinstance(
kwargs['values_source_list'][0],
ApplicationDefaultsProxy
))
eq_(result, 17)
def test_do_run_with_alternate_values_source_list(self):
config = DotDict()
with mock.patch('socorro.app.socorro_app.ConfigurationManager') as cm:
cm.return_value.context.return_value = mock.MagicMock()
with mock.patch('socorro.app.socorro_app.signal') as s:
class SomeOtherApp(SocorroApp):
app_name='SomeOtherApp'
app_verision='1.2.3'
app_description='a silly app'
def main(self):
expected = cm.return_value.context.return_value.__enter__.return_value
assert self.config is expected
return 17
result = main(
SomeOtherApp,
config_path='my/other/path',
values_source_list=[{"a": 1}, {"b": 2}]
)
args = cm.call_args_list
args, kwargs = args[0]
assert isinstance(args[0], Namespace)
assert kwargs['app_name'] == SomeOtherApp.app_name
assert kwargs['app_version'] == SomeOtherApp.app_version
assert kwargs['app_description'] == SomeOtherApp.app_description
assert kwargs['config_pathname'] == 'my/other/path'
assert isinstance(kwargs['values_source_list'], list)
assert isinstance(kwargs['values_source_list'][0], ApplicationDefaultsProxy)
assert kwargs['values_source_list'][1] == {"a": 1}
assert kwargs['values_source_list'][2] == {"b": 2}
assert result == 17
if not self.config.get('no_staticdata'):
self.import_staticdata(db)
if self.config['fakedata']:
self.generate_fakedata(db, self.config['fakedata_days'])
db.commit()
command.stamp(alembic_cfg, "heads")
db.session.close()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database owner section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'database extensions section with %s',
superuser_normaldb_pg_url
)
with PostgreSQLAlchemyManager(
superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
db.set_table_owner(self.config.database_username)
db.set_default_owner(database_name, self.config.database_username)
db.set_grants(self.config) # config has user lists
return 0
if __name__ == "__main__":
sys.exit(main(SocorroDBApp))
"""
required_config = Namespace()
required_config.add_option('date',
default=datetime.datetime.utcnow().date(),
doc='Date to run for',
from_string_converter=string_to_datetime)
required_config.add_option(
'crontabber_job_class',
default='socorro.cron.jobs.ftpscraper.FTPScraperCronApp',
doc='bla',
from_string_converter=class_converter,
)
@staticmethod
def get_application_defaults():
return {'database.database_class': mock.MagicMock()}
def __init__(self, config):
self.config = config
self.config.dry_run = True
self.ftpscraper = config.crontabber_job_class(config, {})
def main(self):
assert self.config.dry_run
self.ftpscraper.run(self.config.date)
if __name__ == '__main__': # pragma: no cover
sys.exit(main(FTPScraperCronAppDryRunner))
from configman import Namespace
class ExampleApp(App):
app_name = 'example'
app_version = '0.1'
app_description = __doc__
# in this section, define any configuration requirements
required_config = Namespace()
required_config.add_option('name', default='Wilma', doc='a name to echo')
required_config.add_option('time',
default=datetime.datetime.now(),
doc='the time of day')
# implementing this constructor is only necessary when there is more
# initialization to be done before main can be called
# def __init__(self, config):
# super(ExampleApp,self).__init__(config)
def main(self):
# this is where we'd implement the app
# the configuraton is already setup as self.config
print('hello, %s. The time is: %s' %
(self.config.name, self.config.time))
if __name__ == '__main__':
main(ExampleApp)
self.config.processor_name = self.app_instance_name
# this function will be called by the MainThread periodically
# while the threaded_task_manager processes crashes.
self.waiting_func = None
self.processor = self.config.processor.processor_class(
self.config.processor,
self.quit_check
)
def close(self):
"""when the processor shutsdown, this function cleans up"""
try:
self.companion_process.close()
except AttributeError:
# there is either no companion or it doesn't have a close method
# we can skip on
pass
try:
self.processor.close()
except AttributeError:
# the processor implementation does not have a close method
# we can blithely skip on
pass
if __name__ == '__main__':
main(ProcessorApp)
#--------------------------------------------------------------------------
def find_method(self, a_method_name):
try:
return getattr(self, a_method_name + '_command')
except AttributeError:
return partial(self.method_not_found, a_method_name)
#--------------------------------------------------------------------------
def main(self):
self.crash_store = self.config.crashstorage_class(self.config)
command = self.find_method(self.config.command)
result = command()
if result is not None:
print result
commands = []
for a_symbol in dir(FetchApp):
if a_symbol.endswith('_command'):
commands.append(a_symbol.replace('_command', ''))
FetchApp.required_config.command.doc = (
FetchApp.required_config.command.doc % ', '.join(commands)
)
if __name__ == '__main__':
main(FetchApp)
class MissingSymbolsCronAppDryRunner(App): # pragma: no cover
"""App to test running missing-symbols right here right now.
To run it, simply execute this file:
$ python socorro/cron/jobs/missingsymbols.py
"""
required_config = Namespace()
required_config.add_option(
'crontabber_job_class',
default='socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp',
doc='bla',
from_string_converter=class_converter,
)
def __init__(self, config):
self.config = config
self.app = config.crontabber_job_class(config, {})
def main(self):
self.app.run()
if __name__ == '__main__': # pragma: no cover
import sys
sys.exit(main(MissingSymbolsCronAppDryRunner))
#--------------------------------------------------------------------------
def find_method(self, a_method_name):
try:
return getattr(self, a_method_name + '_command')
except AttributeError:
return partial(self.method_not_found, a_method_name)
#--------------------------------------------------------------------------
def main(self):
self.crash_store = self.config.crashstorage_class(self.config)
command = self.find_method(self.config.command)
result = command()
if result is not None:
print result
commands = []
for a_symbol in dir(FetchApp):
if a_symbol.endswith('_command'):
commands.append(a_symbol.replace('_command', ''))
FetchApp.required_config.command.doc = (
FetchApp.required_config.command.doc % ', '.join(commands)
)
if __name__ == '__main__':
main(FetchApp)
class MissingSymbolsCronAppDryRunner(App): # pragma: no cover
"""App to test running missing-symbols right here right now.
To run it, simply execute this file:
$ python socorro/cron/jobs/missingsymbols.py
"""
required_config = Namespace()
required_config.add_option(
'crontabber_job_class',
default='socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp',
doc='bla',
from_string_converter=class_converter,
)
def __init__(self, config):
self.config = config
self.app = config.crontabber_job_class(config, {})
def main(self):
self.app.run()
if __name__ == '__main__': # pragma: no cover
import sys
sys.exit(main(MissingSymbolsCronAppDryRunner))
import os
from socorro.app.socorro_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import WSGIServer
import socorro.middleware.middleware_app
from configman import (
ConfigFileFutureProxy,
environment
)
if os.path.isfile('/etc/socorro/middleware.ini'):
config_path = '/etc/socorro'
else:
config_path = WSGIServer.get_socorro_config_path(__file__)
# invoke the socorro main function to create the configman app class and which
# will then create the wsgi app object.
main(
MiddlewareApp, # the socorro app class
config_path=config_path,
values_source_list=[
ConfigFileFutureProxy,
environment
]
)
application = socorro.middleware.middleware_app.application
alembic_cfg = Config(self.config.alembic_config)
alembic_cfg.set_main_option("sqlalchemy.url", sa_url)
with PostgreSQLAlchemyManager(sa_url, self.config.logger) as db:
connection = db.engine.connect()
db.setup_admin()
if self.no_schema:
db.commit()
return 0
db.create_types()
db.create_procs()
db.set_sequence_owner('breakpad_rw')
db.commit()
db.create_tables()
db.set_table_owner('breakpad_rw')
db.create_views()
db.commit()
db.set_grants(self.config) # config has user lists
if not self.config['no_staticdata']:
self.import_staticdata(db)
if self.config['fakedata']:
self.generate_fakedata(db, self.config['fakedata_days'])
db.commit()
command.stamp(alembic_cfg, "head")
db.set_default_owner(self.database_name)
db.session.close()
return 0
if __name__ == "__main__":
sys.exit(main(SocorroDBApp))
'date',
default=datetime.datetime.utcnow().date(),
doc='Date to run for',
from_string_converter=string_to_datetime
)
required_config.add_option(
'crontabber_job_class',
default='socorro.cron.jobs.ftpscraper.FTPScraperCronApp',
doc='bla',
from_string_converter=class_converter,
)
@staticmethod
def get_application_defaults():
return {
'database.database_class': mock.MagicMock()
}
def __init__(self, config):
self.config = config
self.config.dry_run = True
self.ftpscraper = config.crontabber_job_class(config, {})
def main(self):
assert self.config.dry_run
self.ftpscraper.run(self.config.date)
if __name__ == '__main__': # pragma: no cover
sys.exit(main(FTPScraperCronAppDryRunner))
main
)
from socorro.webapi.servers import WSGIServer
import socorro.collector.collector_app
from configman import (
ConfigFileFutureProxy,
environment
)
if os.path.isfile('/etc/socorro/collector.ini'):
config_path = '/etc/socorro'
else:
config_path = WSGIServer.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
# we use the generic Socorro App class. We'll rely on configuration to set
# the 'application' class object to the appropriate collector_app class
# for example, it could be "CollectorApp" or "Collector2015App"
SocorroWelcomeApp,
config_path=config_path,
values_source_list=[
ConfigFileFutureProxy,
environment
]
)
application = socorro.collector.collector_app.application
host=config.host,
port=config.port,
virtual_host=config.virtual_host,
credentials=pika.credentials.PlainCredentials(
config.rabbitmq_user, config.rabbitmq_password)))
except Exception:
logging.error("Failed to connect")
raise
self.connection = connection
def main(self):
self.connect()
channel = self.connection.channel()
channel.queue_declare(queue='socorro.reprocessing', durable=True)
with open(self.config.reprocesscrashlist.crashes, 'r') as file:
for uuid in file.read().splitlines():
channel.basic_publish(
exchange='',
routing_key="socorro.reprocessing",
body=uuid,
properties=pika.BasicProperties(delivery_mode=2))
logging.debug('submitted %s' % uuid)
self.connection.close()
if __name__ == '__main__':
main(ReprocessCrashlistApp)
def _action_after_iteration_completes(self):
self.config.logger.info(
'the queuing iterator is exhausted - waiting to quit')
self.task_manager.wait_for_empty_queue(
5, "waiting for the queue to drain before quitting")
time.sleep(self.config.producer_consumer.number_of_threads * 2)
def _filter_disallowed_values(self, current_value):
"""In this base class there are no disallowed values coming from the
iterators. Other users of these iterator may have some standards and
can detect and reject them here
"""
return current_value is None
def _transform(self, crash_id):
"""Transfers raw data from the source to the destination without changing the data
"""
if self.config.submitter.dry_run:
print(crash_id)
else:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
self.destination.save_raw_crash_with_file_dumps(
raw_crash, dumps, crash_id)
if __name__ == '__main__':
main(SubmitterApp)
