def test_exit_codes(self, logging):
vsl = (ConfigFileFutureProxy, {'exit_code': 123})
exit_code = main(ExitingApp, values_source_list=vsl)
self.assertEqual(exit_code, 123)
vsl = (ConfigFileFutureProxy, {'exit_code': 0})
exit_code = main(ExitingApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
vsl = (ConfigFileFutureProxy, {'exit_code': None})
exit_code = main(ExitingApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
def test_exit_codes(self, logging):
vsl = (ConfigFileFutureProxy, {'exit_code': 123})
exit_code = main(ExitingApp, values_source_list=vsl)
self.assertEqual(exit_code, 123)
vsl = (ConfigFileFutureProxy, {'exit_code': 0})
exit_code = main(ExitingApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
vsl = (ConfigFileFutureProxy, {'exit_code': None})
exit_code = main(ExitingApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
def test_run_weeklyReportsPartitions(self):
"""Create a mock function named exactly like the stored procedure in
the cron script we want to functionally test.
That way we can assert that it was run.
"""
# provide values for the config to pick up
# be explicit about the values_source_list to
# avoid configman picking up nosetests arguments
main(WeeklyReportsPartitions, values_source_list=[DSN])
# check that something was written to the mock_bucket
cursor = self.conn.cursor()
cursor.execute('select count(*) from mock_bucket;')
self.assertTrue(cursor.fetchone())
def test_overriding_config_path(self, logging):
vsl = (ConfigFileFutureProxy, )
exit_code = main(MyApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = '/foo/bar'
self.assertRaises(IOError, main, (MyApp, ), values_source_list=vsl)
os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = self.tempdir
exit_code = main(MyApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
logging.getLogger().error.assert_called_with('colour')
_ini_file = os.path.join(self.tempdir, 'myapp.ini')
with open(_ini_file, 'w') as f:
f.write('color_or_colour=color\n')
exit_code = main(MyApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
logging.getLogger().error.assert_called_with('color')
def test_overriding_config_path(self, logging):
vsl = (ConfigFileFutureProxy,)
exit_code = main(MyApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = '/foo/bar'
self.assertRaises(IOError, main, (MyApp,), values_source_list=vsl)
os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = self.tempdir
exit_code = main(MyApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
logging.getLogger().error.assert_called_with('colour')
_ini_file = os.path.join(self.tempdir, 'myapp.ini')
with open(_ini_file, 'w') as f:
f.write('color_or_colour=color\n')
exit_code = main(MyApp, values_source_list=vsl)
self.assertEqual(exit_code, 0)
logging.getLogger().error.assert_called_with('color')
app_name = 'example'
app_version = '0.1'
app_description = __doc__
#--------------------------------------------------------------------------
# in this section, define any configuration requirements
required_config = Namespace()
required_config.add_option('name',
default='Wilma',
doc='a name to echo')
required_config.add_option('time',
default=datetime.datetime.now(),
doc='the time of day')
#--------------------------------------------------------------------------
# implementing this constructor is only necessary when there is more
# initialization to be done before main can be called
#def __init__(self, config):
#super(ExampleApp,self).__init__(config)
#--------------------------------------------------------------------------
def main(self):
# this is where we'd implement the app
# the configuraton is already setup as self.config
print 'hello, %s. The time is: %s' % (self.config.name,
self.config.time)
if __name__ == '__main__':
main(ExampleApp)
def processor2012(config):
from configman.dotdict import DotDict
# the following section is a translation of old-style configuration into
# the new style configuration. It lets a new style app run without leaving
# the old configuration system.
trans_config = DotDict()
#--------------------------------------------------------------------------
# destination -
trans_config.destination = DotDict()
# name: destination.crashstorage_class
# doc: the destination storage class
# converter: configman.converters.class_converter
import socorro.external.crashstorage_base
trans_config.destination.crashstorage_class = \
socorro.external.crashstorage_base.PolyCrashStorage
# name: destination.storage_classes
# doc: a comma delimited list of storage classes
# converter: configman.converters.class_list_converter
import socorro.external.postgresql.crashstorage
import socorro.external.hbase.crashstorage
import socorro.external.elasticsearch.crashstorage
trans_config.destination.storage_classes = '''
socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage,
socorro.external.hbase.crashstorage.HBaseCrashStorage,
socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage
'''
#--------------------------------------------------------------------------
# storage0 -
trans_config.destination.storage0 = DotDict()
# name: destination.storage0.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.destination.storage0.backoff_delays = [
10, 30, 60, 120, 300, 300, 300, 300, 300, 300
]
# name: destination.storage0.crashstorage_class
# doc: None
# converter: configman.converters.class_converter
trans_config.destination.storage0.crashstorage_class = \
socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage
# name: destination.storage0.database_class
# doc: the class responsible for connecting toPostgres
# converter: configman.converters.class_converter
import socorro.external.postgresql.connection_context
trans_config.destination.storage0.database_class = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: destination.storage0.database_host
# doc: the hostname of the database
# converter: str
trans_config.destination.storage0.database_host = config.databaseHost
# name: destination.storage0.database_name
# doc: the name of the database
# converter: str
trans_config.destination.storage0.database_name = config.databaseName
# name: destination.storage0.database_password
# doc: the user's database password
# converter: str
trans_config.destination.storage0.database_password = \
config.databasePassword
# name: destination.storage0.database_port
# doc: the port for the database
# converter: int
trans_config.destination.storage0.database_port = config.databasePort
# name: destination.storage0.database_user
# doc: the name of the user within the database
# converter: str
trans_config.destination.storage0.database_user = config.databaseUserName
# name: destination.storage0.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
import socorro.database.transaction_executor
trans_config.destination.storage0.transaction_executor_class = (
socorro.database.transaction_executor.
TransactionExecutorWithLimitedBackoff)
# name: destination.storage0.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.destination.storage0.wait_log_interval = 5
#--------------------------------------------------------------------------
# storage1 -
trans_config.destination.storage1 = DotDict()
# name: destination.storage1.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.destination.storage1.backoff_delays = [
10, 30, 60, 120, 300, 300, 300, 300, 300, 300
]
# name: destination.storage1.crashstorage_class
# doc: None
# converter: configman.converters.class_converter
trans_config.destination.storage1.crashstorage_class = \
socorro.external.hbase.crashstorage.HBaseCrashStorage
# name: destination.storage1.forbidden_keys
# doc: a comma delimited list of keys banned from the processed crash in
# HBase
# converter: socorro.external.hbase.crashstorage.<lambda>
trans_config.destination.storage1.forbidden_keys = \
['email', 'url', 'user_id', 'exploitability']
# name: destination.storage1.hbase_host
# doc: Host to HBase server
# converter: str
trans_config.destination.storage1.hbase_host = config.hbaseHost
# name: destination.storage1.hbase_port
# doc: Port to HBase server
# converter: int
trans_config.destination.storage1.hbase_port = config.hbasePort
# name: destination.storage1.hbase_timeout
# doc: timeout in milliseconds for an HBase connection
# converter: int
trans_config.destination.storage1.hbase_timeout = config.hbaseTimeout
# name: destination.storage1.number_of_retries
# doc: Max. number of retries when fetching from hbaseClient
# converter: int
trans_config.destination.storage1.number_of_retries = 2
# name: destination.storage1.transaction_executor_class
# doc: a class that will execute transactions
# converter: configman.converters.class_converter
trans_config.destination.storage1.transaction_executor_class = (
socorro.database.transaction_executor.
TransactionExecutorWithLimitedBackoff)
# name: destination.storage1.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.destination.storage1.wait_log_interval = 5
# name: destination.storage1.temporary_file_system_storage_path
# doc: a local filesystem path dumps can be temporarily written
# for processing
# converter: str
trans_config.destination.storage1.temporary_file_system_storage_path = \
config.temporaryFileSystemStoragePath
# name: destination.storage1.dump_file_suffix
# doc: the suffix used to identify a dump file (for use in temp files)
# converter: str
trans_config.destination.storage1.dump_file_suffix = \
config.dumpFileSuffix
#--------------------------------------------------------------------------
# storage2 -
trans_config.destination.storage2 = DotDict()
# name: destination.storage2.crashstorage_class
# doc: None
# converter: configman.converters.class_converter
trans_config.destination.storage2.crashstorage_class = \
socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage
# name: destination.storage2.elasticsearch_urls
# doc: the urls for the elasticsearch instances (leave blank to disable)
# converter: str
trans_config.destination.storage2.elasticsearch_urls = \
config.elasticsearch_urls
# name: destination.storage2.elasticsearch_index
# doc: an index to insert crashes in elasticsearch (use datetime's strftime
# format to have daily, weekly or monthly indexes)
# converter: str
trans_config.destination.storage2.elasticsearch_index = \
config.elasticsearch_index
# name: destination.storage2.elasticsearch_doctype
# doc: a type to insert crashes in elasticsearch
# converter: str
trans_config.destination.storage2.elasticsearch_doctype = \
config.elasticsearch_doctype
# name: destination.storage2.elasticsearch_index_settings
# doc: the mapping of crash reports to insert
# converter: str
trans_config.destination.storage2.elasticsearch_index_settings = \
config.elasticsearch_index_settings
# name: destination.storage2.timeout
# doc: how long to wait in seconds for confirmation of a submission
# converter: int
trans_config.destination.storage2.timeout = 2
# name: destination.storage2.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
import socorro.database.transaction_executor
trans_config.destination.storage2.transaction_executor_class = \
socorro.database.transaction_executor.TransactionExecutor
#--------------------------------------------------------------------------
# logging -
trans_config.logging = DotDict()
# name: logging.stderr_error_logging_level
# doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 -
# WARNING, 40 - ERROR, 50 - CRITICAL)
# converter: int
trans_config.logging.stderr_error_logging_level = \
config.stderrErrorLoggingLevel
# name: logging.stderr_line_format_string
# doc: python logging system format for logging to stderr
# converter: str
trans_config.logging.stderr_line_format_string = \
config.stderrLineFormatString
# name: logging.syslog_error_logging_level
# doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING,
# 40 - ERROR, 50 - CRITICAL)
# converter: int
trans_config.logging.syslog_error_logging_level = \
config.syslogErrorLoggingLevel
# name: logging.syslog_facility_string
# doc: syslog facility string ("user", "local0", etc)
# converter: str
trans_config.logging.syslog_facility_string = config.syslogFacilityString
# name: logging.syslog_host
# doc: syslog hostname
# converter: str
trans_config.logging.syslog_host = config.syslogHost
# name: logging.syslog_line_format_string
# doc: python logging system format for syslog entries
# converter: str
# NOTE: the old and new systems use very differet formats for
# the logging template. In the entire history of the project
# the template has not changed. Rather than spending the effort
# to translate the format, this hard coding will do. Since this
# chimera processor is transintional, this will go away in the
# not too distant future.
trans_config.logging.syslog_line_format_string = \
'{app_name} (pid {process}): ' \
'{asctime} {levelname} - {threadName} - ' \
'{message}'
# name: logging.syslog_port
# doc: syslog port
# converter: int
trans_config.logging.syslog_port = config.syslogPort
#--------------------------------------------------------------------------
# new_crash_source -
trans_config.new_crash_source = DotDict()
# name: new_crash_source.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.new_crash_source.backoff_delays = [
10, 30, 60, 120, 300, 300, 300, 300, 300, 300
]
# name: new_crash_source.batchJobLimit
# doc: the number of jobs to pull in a time
# converter: int
trans_config.new_crash_source.batchJobLimit = config.batchJobLimit
# name: new_crash_source.database_class
# doc: the class of the database
# converter: configman.converters.class_converter
trans_config.new_crash_source.database_class = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: new_crash_source.database_host
# doc: the hostname of the database
# converter: str
trans_config.new_crash_source.database_host = config.databaseHost
# name: new_crash_source.database_name
# doc: the name of the database
# converter: str
trans_config.new_crash_source.database_name = config.databaseName
# name: new_crash_source.database_password
# doc: the user's database password
# converter: str
trans_config.new_crash_source.database_password = config.databasePassword
# name: new_crash_source.database_port
# doc: the port for the database
# converter: int
trans_config.new_crash_source.database_port = config.databasePort
# name: new_crash_source.database_user
# doc: the name of the user within the database
# converter: str
trans_config.new_crash_source.database_user = config.databaseUserName
# name: new_crash_source.new_crash_source_class
# doc: an iterable that will stream crash_ids needing processing
# converter: configman.converters.class_converter
import socorro.processor.legacy_new_crash_source
trans_config.new_crash_source.new_crash_source_class = \
socorro.processor.legacy_new_crash_source.LegacyNewCrashSource
# name: new_crash_source.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
trans_config.new_crash_source.transaction_executor_class = (
socorro.database.transaction_executor.
TransactionExecutorWithLimitedBackoff)
# name: new_crash_source.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.new_crash_source.wait_log_interval = 5
#--------------------------------------------------------------------------
# processor -
trans_config.processor = DotDict()
# name: processor.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.processor.backoff_delays = [
10, 30, 60, 120, 300, 300, 300, 300, 300, 300
]
# name: processor.collect_addon
# doc: boolean indictating if information about add-ons should be collected
# converter: configman.converters.boolean_converter
trans_config.processor.collect_addon = config.collectAddon
# name: processor.collect_crash_process
# doc: boolean indictating if information about process type should be
# collected
# converter: configman.converters.boolean_converter
trans_config.processor.collect_crash_process = config.collectCrashProcess
# name: processor.crashing_thread_frame_threshold
# doc: the number of frames to keep in the raw dump for the crashing thread
# converter: int
trans_config.processor.crashing_thread_frame_threshold = \
config.crashingThreadFrameThreshold
# name: processor.crashing_thread_tail_frame_threshold
# doc: the number of frames to keep in the raw dump at the tail of the
# frame
# list
# converter: int
trans_config.processor.crashing_thread_tail_frame_threshold = \
config.crashingThreadTailFrameThreshold
# name: processor.database_class
# doc: the class of the database
# converter: configman.converters.class_converter
trans_config.processor.database_class = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: processor.database_host
# doc: the hostname of the database
# converter: str
trans_config.processor.database_host = config.databaseHost
# name: processor.database_name
# doc: the name of the database
# converter: str
trans_config.processor.database_name = config.databaseName
# name: processor.database_password
# doc: the user's database password
# converter: str
trans_config.processor.database_password = config.databasePassword
# name: processor.database_port
# doc: the port for the database
# converter: int
trans_config.processor.database_port = config.databasePort
# name: processor.database_user
# doc: the name of the user within the database
# converter: str
trans_config.processor.database_user = config.databaseUserName
# name: processor.known_flash_identifiers
# doc: A subset of the known "debug identifiers" for flash versions,
# associated to the version
# converter: json.loads
trans_config.processor.known_flash_identifiers = \
config.knownFlashIdentifiers
# name: processor.minidump_stackwalk_pathname
# doc: the full pathname of the extern program minidump_stackwalk (quote
# path with embedded spaces)
# converter: str
trans_config.processor.minidump_stackwalk_pathname = \
config.minidump_stackwalkPathname
# name: processor.processor_class
# doc: the class that transforms raw crashes into processed crashes
# converter: configman.converters.class_converter
import socorro.processor.legacy_processor
trans_config.processor.processor_class = \
socorro.processor.legacy_processor.LegacyCrashProcessor
# name: processor.processor_symbols_pathname_list
# doc: comma or space separated list of symbol files for minidump_stackwalk
# (quote paths with embedded spaces)
# converter: socorro.processor.legacy_processor.create_symbol_path_str
trans_config.processor.processor_symbols_pathname_list = \
config.processorSymbolsPathnameList
# name: processor.stackwalk_command_line
# doc: the template for the command to invoke minidump_stackwalk
# converter: str
trans_config.processor.stackwalk_command_line = \
config.stackwalkCommandLine.replace(
'minidump_stackwalkPathname',
'minidump_stackwalk_pathname'
).replace(
'processorSymbolsPathnameList',
'processor_symbols_pathname_list'
)
# name: exploitability_tool_command_line
# doc: the template for the command to invoke the exploitability tool
# converter: str
trans_config.processor.exploitability_tool_command_line = \
config.exploitability_tool_command_line
# name: exploitability_tool_pathname
# doc: the full pathname of the extern program exploitability tool
# (quote path with embedded spaces)
# converter: str
trans_config.processor.exploitability_tool_pathname = \
config.exploitability_tool_pathname
# name: processor.symbol_cache_path
# doc: the path where the symbol cache is found (quote path with embedded
# spaces)
# converter: str
trans_config.processor.symbol_cache_path = config.symbolCachePath
# name: processor.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
trans_config.processor.transaction_executor_class = (
socorro.database.transaction_executor.
TransactionExecutorWithLimitedBackoff)
# name: processor.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.processor.wait_log_interval = 5
#--------------------------------------------------------------------------
# c_signature -
trans_config.processor.c_signature = DotDict()
# name: processor.c_signature.c_signature_tool_class
# doc: the class that can generate a C signature
# converter: configman.converters.class_converter
import socorro.processor.signature_utilities
trans_config.processor.c_signature.c_signature_tool_class = \
socorro.processor.signature_utilities.CSignatureTool
# name: processor.c_signature.irrelevant_signature_re
# doc: a regular expression matching frame signatures that should be
# ignored when generating an overall signature
# converter: str
trans_config.processor.c_signature.irrelevant_signature_re = \
config.irrelevantSignatureRegEx
# name: processor.c_signature.prefix_signature_re
# doc: a regular expression matching frame signatures that should always be
# coupled with the following frame signature when generating an
# overall signature
# converter: str
trans_config.processor.c_signature.prefix_signature_re = \
config.prefixSignatureRegEx
# name: processor.c_signature.signature_sentinels
# doc: a list of frame signatures that should always be considered top of
# the stack if present in the stack
# converter: eval
trans_config.processor.c_signature.signature_sentinels = \
config.signatureSentinels
# name: processor.c_signature.signatures_with_line_numbers_re
# doc: any signatures that match this list should be combined with their
# associated source code line numbers
# converter: str
trans_config.processor.c_signature.signatures_with_line_numbers_re = \
config.signaturesWithLineNumbersRegEx
#--------------------------------------------------------------------------
# java_signature -
trans_config.processor.java_signature = DotDict()
# name: processor.java_signature.java_signature_tool_class
# doc: the class that can generate a Java signature
# converter: configman.converters.class_converter
trans_config.processor.java_signature.java_signature_tool_class = \
socorro.processor.signature_utilities.JavaSignatureTool
#--------------------------------------------------------------------------
# producer_consumer -
trans_config.producer_consumer = DotDict()
# name: producer_consumer.idle_delay
# doc: the delay in seconds if no job is found
# converter: int
trans_config.producer_consumer.idle_delay = \
config.processorLoopTime.seconds
# name: producer_consumer.maximum_queue_size
# doc: the maximum size of the internal queue
# converter: int
trans_config.producer_consumer.maximum_queue_size = \
config.numberOfThreads * 2
# name: producer_consumer.number_of_threads
# doc: the number of threads
# converter: int
trans_config.producer_consumer.number_of_threads = config.numberOfThreads
# name: producer_consumer.producer_consumer_class
# doc: the class implements a threaded producer consumer queue
# converter: configman.converters.class_converter
import socorro.lib.threaded_task_manager
trans_config.producer_consumer.producer_consumer_class = \
socorro.lib.threaded_task_manager.ThreadedTaskManager
#--------------------------------------------------------------------------
# registrar -
trans_config.registrar = DotDict()
# name: registrar.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.registrar.backoff_delays = [
10, 30, 60, 120, 300, 300, 300, 300, 300, 300
]
# name: registrar.check_in_frequency
# doc: how often the processor is required to reregister (hh:mm:ss)
# converter: configman.converters.timedelta_converter
trans_config.registrar.check_in_frequency = \
config.processorCheckInFrequency
# name: registrar.database
# doc: the class of the registrar's database
# converter: configman.converters.class_converter
trans_config.registrar.database = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: registrar.database_host
# doc: the hostname of the database
# converter: str
trans_config.registrar.database_host = config.databaseHost
# name: registrar.database_name
# doc: the name of the database
# converter: str
trans_config.registrar.database_name = config.databaseName
# name: registrar.database_password
# doc: the user's database password
# converter: str
trans_config.registrar.database_password = config.databasePassword
# name: registrar.database_port
# doc: the port for the database
# converter: int
trans_config.registrar.database_port = config.databasePort
# name: registrar.database_user
# doc: the name of the user within the database
# converter: str
trans_config.registrar.database_user = config.databaseUserName
# name: registrar.processor_id
# doc: the id number for the processor (must already exist) (0 for create
# new Id, "auto" for autodetection, "host" for same host)
# converter: str
trans_config.registrar.processor_id = config.processorId
# name: registrar.registrar_class
# doc: the class that registers and tracks processors
# converter: configman.converters.class_converter
import socorro.processor.registration_client
trans_config.registrar.registrar_class = \
socorro.processor.registration_client.ProcessorAppRegistrationClient
# name: registrar.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
trans_config.registrar.transaction_executor_class = (
socorro.database.transaction_executor.
TransactionExecutorWithLimitedBackoff)
# name: registrar.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.registrar.wait_log_interval = 5
#--------------------------------------------------------------------------
# source -
trans_config.source = DotDict()
# name: source.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.source.backoff_delays = [
10, 30, 60, 120, 300, 300, 300, 300, 300, 300
]
# name: source.crashstorage_class
# doc: the source storage class
# converter: configman.converters.class_converter
trans_config.source.crashstorage_class = \
socorro.external.hbase.crashstorage.HBaseCrashStorage
# name: source.forbidden_keys
# doc: a comma delimited list of keys banned from the processed crash in
# HBase
# converter: socorro.external.hbase.crashstorage.<lambda>
trans_config.source.forbidden_keys = \
['email', 'url', 'user_id', 'exploitability']
# name: source.hbase_host
# doc: Host to HBase server
# converter: str
trans_config.source.hbase_host = config.hbaseHost
# name: source.hbase_port
# doc: Port to HBase server
# converter: int
trans_config.source.hbase_port = config.hbasePort
# name: source.hbase_timeout
# doc: timeout in milliseconds for an HBase connection
# converter: int
trans_config.source.hbase_timeout = config.hbaseTimeout
# name: source.number_of_retries
# doc: Max. number of retries when fetching from hbaseClient
# converter: int
trans_config.source.number_of_retries = 2
# name: source.transaction_executor_class
# doc: a class that will execute transactions
# converter: configman.converters.class_converter
trans_config.source.transaction_executor_class = (
socorro.database.transaction_executor.
TransactionExecutorWithLimitedBackoff)
# name: source.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.source.wait_log_interval = 5
# name: destination.storage1.temporary_file_system_storage_path
# doc: a local filesystem path dumps can be temporarily written
# for processing
# converter: str
trans_config.source.temporary_file_system_storage_path = \
config.temporaryFileSystemStoragePath
# name: destination.storage1.dump_file_suffix
# doc: the suffix used to identify a dump file (for use in temp files)
# converter: str
trans_config.source.dump_file_suffix = config.dumpFileSuffix
#--------------------------------------------------------------------------
from socorro.app.generic_app import main
from socorro.processor.processor_app import ProcessorApp
main(ProcessorApp, [trans_config])
#--------------------------------------------------------------------------
def main(self):
"""this function is run by the main thread. It just starts the
subordinate threads and then waits for them to complete."""
standard_job_thread = threading.Thread(
name="standard_job_thread",
target=self._standard_job_thread
)
standard_job_thread.start()
priority_job_thread = threading.Thread(
name="priority_job_thread",
target=self._priority_job_thread
)
priority_job_thread.start()
job_cleanup_thread = threading.Thread(
name="job_cleanup_thread",
target=self._job_cleanup_thread
)
job_cleanup_thread.start()
self.config.logger.debug("waiting to join.")
self._responsive_join(job_cleanup_thread)
self._responsive_join(priority_job_thread)
self._responsive_join(standard_job_thread)
if __name__ == '__main__':
main(MonitorApp)
id_field='name',
)
# Verify data was correctly inserted.
es_connection.refresh()
total_indexed = es_connection.count(
'*',
index='socorro',
doc_type='supersearch_fields',
)['count']
total_expected = len(all_fields)
if total_expected != total_indexed:
indexed_fields = es_connection.search(
'*',
index='socorro',
doc_type='supersearch_fields',
size=total_indexed,
)
indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]
self.config.logger.error(
'The SuperSearch fields data was not correctly indexed, '
'%s fields are missing from the database. Missing fields: %s',
total_expected - total_indexed,
list(set(all_fields.keys()) - set(indexed_fields)))
if __name__ == '__main__':
generic_app.main(SetupSuperSearchApp)
def processor2012(config):
from configman.dotdict import DotDict
# the following section is a translation of old-style configuration into
# the new style configuration. It lets a new style app run without leaving
# the old configuration system.
trans_config = DotDict()
#--------------------------------------------------------------------------
# destination -
trans_config.destination = DotDict()
# name: destination.crashstorage_class
# doc: the destination storage class
# converter: configman.converters.class_converter
import socorro.external.crashstorage_base
trans_config.destination.crashstorage_class = \
socorro.external.crashstorage_base.PolyCrashStorage
# name: destination.storage_classes
# doc: a comma delimited list of storage classes
# converter: configman.converters.class_list_converter
import socorro.external.postgresql.crashstorage
import socorro.external.hbase.crashstorage
import socorro.external.elasticsearch.crashstorage
trans_config.destination.storage_classes = '''
socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage,
socorro.external.hbase.crashstorage.HBaseCrashStorage,
socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage
'''
#--------------------------------------------------------------------------
# storage0 -
trans_config.destination.storage0 = DotDict()
# name: destination.storage0.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.destination.storage0.backoff_delays = [10, 30, 60, 120, 300,
300, 300, 300, 300,
300]
# name: destination.storage0.crashstorage_class
# doc: None
# converter: configman.converters.class_converter
trans_config.destination.storage0.crashstorage_class = \
socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage
# name: destination.storage0.database_class
# doc: the class responsible for connecting toPostgres
# converter: configman.converters.class_converter
import socorro.external.postgresql.connection_context
trans_config.destination.storage0.database_class = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: destination.storage0.database_host
# doc: the hostname of the database
# converter: str
trans_config.destination.storage0.database_host = config.databaseHost
# name: destination.storage0.database_name
# doc: the name of the database
# converter: str
trans_config.destination.storage0.database_name = config.databaseName
# name: destination.storage0.database_password
# doc: the user's database password
# converter: str
trans_config.destination.storage0.database_password = \
config.databasePassword
# name: destination.storage0.database_port
# doc: the port for the database
# converter: int
trans_config.destination.storage0.database_port = config.databasePort
# name: destination.storage0.database_user
# doc: the name of the user within the database
# converter: str
trans_config.destination.storage0.database_user = config.databaseUserName
# name: destination.storage0.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
import socorro.database.transaction_executor
trans_config.destination.storage0.transaction_executor_class = (
socorro.database.transaction_executor
.TransactionExecutorWithLimitedBackoff
)
# name: destination.storage0.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.destination.storage0.wait_log_interval = 5
#--------------------------------------------------------------------------
# storage1 -
trans_config.destination.storage1 = DotDict()
# name: destination.storage1.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.destination.storage1.backoff_delays = [10, 30, 60, 120, 300,
300, 300, 300, 300,
300]
# name: destination.storage1.crashstorage_class
# doc: None
# converter: configman.converters.class_converter
trans_config.destination.storage1.crashstorage_class = \
socorro.external.hbase.crashstorage.HBaseCrashStorage
# name: destination.storage1.forbidden_keys
# doc: a comma delimited list of keys banned from the processed crash in
# HBase
# converter: socorro.external.hbase.crashstorage.<lambda>
trans_config.destination.storage1.forbidden_keys = \
['email', 'url', 'user_id', 'exploitability']
# name: destination.storage1.hbase_host
# doc: Host to HBase server
# converter: str
trans_config.destination.storage1.hbase_host = config.hbaseHost
# name: destination.storage1.hbase_port
# doc: Port to HBase server
# converter: int
trans_config.destination.storage1.hbase_port = config.hbasePort
# name: destination.storage1.hbase_timeout
# doc: timeout in milliseconds for an HBase connection
# converter: int
trans_config.destination.storage1.hbase_timeout = config.hbaseTimeout
# name: destination.storage1.number_of_retries
# doc: Max. number of retries when fetching from hbaseClient
# converter: int
trans_config.destination.storage1.number_of_retries = 2
# name: destination.storage1.transaction_executor_class
# doc: a class that will execute transactions
# converter: configman.converters.class_converter
trans_config.destination.storage1.transaction_executor_class = (
socorro.database.transaction_executor
.TransactionExecutorWithLimitedBackoff
)
# name: destination.storage1.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.destination.storage1.wait_log_interval = 5
# name: destination.storage1.temporary_file_system_storage_path
# doc: a local filesystem path dumps can be temporarily written
# for processing
# converter: str
trans_config.destination.storage1.temporary_file_system_storage_path = \
config.temporaryFileSystemStoragePath
# name: destination.storage1.dump_file_suffix
# doc: the suffix used to identify a dump file (for use in temp files)
# converter: str
trans_config.destination.storage1.dump_file_suffix = \
config.dumpFileSuffix
#--------------------------------------------------------------------------
# storage2 -
trans_config.destination.storage2 = DotDict()
# name: destination.storage2.crashstorage_class
# doc: None
# converter: configman.converters.class_converter
trans_config.destination.storage2.crashstorage_class = \
socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage
# name: destination.storage2.elasticsearch_urls
# doc: the urls for the elasticsearch instances (leave blank to disable)
# converter: str
trans_config.destination.storage2.elasticsearch_urls = \
config.elasticsearch_urls
# name: destination.storage2.elasticsearch_index
# doc: an index to insert crashes in elasticsearch (use datetime's strftime
# format to have daily, weekly or monthly indexes)
# converter: str
trans_config.destination.storage2.elasticsearch_index = \
config.elasticsearch_index
# name: destination.storage2.elasticsearch_doctype
# doc: a type to insert crashes in elasticsearch
# converter: str
trans_config.destination.storage2.elasticsearch_doctype = \
config.elasticsearch_doctype
# name: destination.storage2.elasticsearch_index_settings
# doc: the mapping of crash reports to insert
# converter: str
trans_config.destination.storage2.elasticsearch_index_settings = \
config.elasticsearch_index_settings
# name: destination.storage2.timeout
# doc: how long to wait in seconds for confirmation of a submission
# converter: int
trans_config.destination.storage2.timeout = 2
# name: destination.storage2.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
import socorro.database.transaction_executor
trans_config.destination.storage2.transaction_executor_class = \
socorro.database.transaction_executor.TransactionExecutor
#--------------------------------------------------------------------------
# logging -
trans_config.logging = DotDict()
# name: logging.stderr_error_logging_level
# doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 -
# WARNING, 40 - ERROR, 50 - CRITICAL)
# converter: int
trans_config.logging.stderr_error_logging_level = \
config.stderrErrorLoggingLevel
# name: logging.stderr_line_format_string
# doc: python logging system format for logging to stderr
# converter: str
trans_config.logging.stderr_line_format_string = \
config.stderrLineFormatString
# name: logging.syslog_error_logging_level
# doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING,
# 40 - ERROR, 50 - CRITICAL)
# converter: int
trans_config.logging.syslog_error_logging_level = \
config.syslogErrorLoggingLevel
# name: logging.syslog_facility_string
# doc: syslog facility string ("user", "local0", etc)
# converter: str
trans_config.logging.syslog_facility_string = config.syslogFacilityString
# name: logging.syslog_host
# doc: syslog hostname
# converter: str
trans_config.logging.syslog_host = config.syslogHost
# name: logging.syslog_line_format_string
# doc: python logging system format for syslog entries
# converter: str
# NOTE: the old and new systems use very differet formats for
# the logging template. In the entire history of the project
# the template has not changed. Rather than spending the effort
# to translate the format, this hard coding will do. Since this
# chimera processor is transintional, this will go away in the
# not too distant future.
trans_config.logging.syslog_line_format_string = \
'{app_name} (pid {process}): ' \
'{asctime} {levelname} - {threadName} - ' \
'{message}'
# name: logging.syslog_port
# doc: syslog port
# converter: int
trans_config.logging.syslog_port = config.syslogPort
#--------------------------------------------------------------------------
# new_crash_source -
trans_config.new_crash_source = DotDict()
# name: new_crash_source.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.new_crash_source.backoff_delays = [10, 30, 60, 120, 300, 300,
300, 300, 300, 300]
# name: new_crash_source.batchJobLimit
# doc: the number of jobs to pull in a time
# converter: int
trans_config.new_crash_source.batchJobLimit = config.batchJobLimit
# name: new_crash_source.database_class
# doc: the class of the database
# converter: configman.converters.class_converter
trans_config.new_crash_source.database_class = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: new_crash_source.database_host
# doc: the hostname of the database
# converter: str
trans_config.new_crash_source.database_host = config.databaseHost
# name: new_crash_source.database_name
# doc: the name of the database
# converter: str
trans_config.new_crash_source.database_name = config.databaseName
# name: new_crash_source.database_password
# doc: the user's database password
# converter: str
trans_config.new_crash_source.database_password = config.databasePassword
# name: new_crash_source.database_port
# doc: the port for the database
# converter: int
trans_config.new_crash_source.database_port = config.databasePort
# name: new_crash_source.database_user
# doc: the name of the user within the database
# converter: str
trans_config.new_crash_source.database_user = config.databaseUserName
# name: new_crash_source.new_crash_source_class
# doc: an iterable that will stream crash_ids needing processing
# converter: configman.converters.class_converter
import socorro.processor.legacy_new_crash_source
trans_config.new_crash_source.new_crash_source_class = \
socorro.processor.legacy_new_crash_source.LegacyNewCrashSource
# name: new_crash_source.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
trans_config.new_crash_source.transaction_executor_class = (
socorro.database.transaction_executor
.TransactionExecutorWithLimitedBackoff
)
# name: new_crash_source.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.new_crash_source.wait_log_interval = 5
#--------------------------------------------------------------------------
# processor -
trans_config.processor = DotDict()
# name: processor.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.processor.backoff_delays = [10, 30, 60, 120, 300, 300, 300,
300, 300, 300]
# name: processor.collect_addon
# doc: boolean indictating if information about add-ons should be collected
# converter: configman.converters.boolean_converter
trans_config.processor.collect_addon = config.collectAddon
# name: processor.collect_crash_process
# doc: boolean indictating if information about process type should be
# collected
# converter: configman.converters.boolean_converter
trans_config.processor.collect_crash_process = config.collectCrashProcess
# name: processor.crashing_thread_frame_threshold
# doc: the number of frames to keep in the raw dump for the crashing thread
# converter: int
trans_config.processor.crashing_thread_frame_threshold = \
config.crashingThreadFrameThreshold
# name: processor.crashing_thread_tail_frame_threshold
# doc: the number of frames to keep in the raw dump at the tail of the
# frame
# list
# converter: int
trans_config.processor.crashing_thread_tail_frame_threshold = \
config.crashingThreadTailFrameThreshold
# name: processor.database_class
# doc: the class of the database
# converter: configman.converters.class_converter
trans_config.processor.database_class = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: processor.database_host
# doc: the hostname of the database
# converter: str
trans_config.processor.database_host = config.databaseHost
# name: processor.database_name
# doc: the name of the database
# converter: str
trans_config.processor.database_name = config.databaseName
# name: processor.database_password
# doc: the user's database password
# converter: str
trans_config.processor.database_password = config.databasePassword
# name: processor.database_port
# doc: the port for the database
# converter: int
trans_config.processor.database_port = config.databasePort
# name: processor.database_user
# doc: the name of the user within the database
# converter: str
trans_config.processor.database_user = config.databaseUserName
# name: processor.known_flash_identifiers
# doc: A subset of the known "debug identifiers" for flash versions,
# associated to the version
# converter: json.loads
trans_config.processor.known_flash_identifiers = \
config.knownFlashIdentifiers
# name: processor.minidump_stackwalk_pathname
# doc: the full pathname of the extern program minidump_stackwalk (quote
# path with embedded spaces)
# converter: str
trans_config.processor.minidump_stackwalk_pathname = \
config.minidump_stackwalkPathname
# name: processor.processor_class
# doc: the class that transforms raw crashes into processed crashes
# converter: configman.converters.class_converter
import socorro.processor.legacy_processor
trans_config.processor.processor_class = \
socorro.processor.legacy_processor.LegacyCrashProcessor
# name: processor.processor_symbols_pathname_list
# doc: comma or space separated list of symbol files for minidump_stackwalk
# (quote paths with embedded spaces)
# converter: socorro.processor.legacy_processor.create_symbol_path_str
trans_config.processor.processor_symbols_pathname_list = \
config.processorSymbolsPathnameList
# name: processor.stackwalk_command_line
# doc: the template for the command to invoke minidump_stackwalk
# converter: str
trans_config.processor.stackwalk_command_line = \
config.stackwalkCommandLine.replace(
'minidump_stackwalkPathname',
'minidump_stackwalk_pathname'
).replace(
'processorSymbolsPathnameList',
'processor_symbols_pathname_list'
)
# name: exploitability_tool_command_line
# doc: the template for the command to invoke the exploitability tool
# converter: str
trans_config.processor.exploitability_tool_command_line = \
config.exploitability_tool_command_line
# name: exploitability_tool_pathname
# doc: the full pathname of the extern program exploitability tool
# (quote path with embedded spaces)
# converter: str
trans_config.processor.exploitability_tool_pathname = \
config.exploitability_tool_pathname
# name: processor.symbol_cache_path
# doc: the path where the symbol cache is found (quote path with embedded
# spaces)
# converter: str
trans_config.processor.symbol_cache_path = config.symbolCachePath
# name: processor.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
trans_config.processor.transaction_executor_class = (
socorro.database.transaction_executor
.TransactionExecutorWithLimitedBackoff
)
# name: processor.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.processor.wait_log_interval = 5
#--------------------------------------------------------------------------
# c_signature -
trans_config.processor.c_signature = DotDict()
# name: processor.c_signature.c_signature_tool_class
# doc: the class that can generate a C signature
# converter: configman.converters.class_converter
import socorro.processor.signature_utilities
trans_config.processor.c_signature.c_signature_tool_class = \
socorro.processor.signature_utilities.CSignatureTool
# name: processor.c_signature.irrelevant_signature_re
# doc: a regular expression matching frame signatures that should be
# ignored when generating an overall signature
# converter: str
trans_config.processor.c_signature.irrelevant_signature_re = \
config.irrelevantSignatureRegEx
# name: processor.c_signature.prefix_signature_re
# doc: a regular expression matching frame signatures that should always be
# coupled with the following frame signature when generating an
# overall signature
# converter: str
trans_config.processor.c_signature.prefix_signature_re = \
config.prefixSignatureRegEx
# name: processor.c_signature.signature_sentinels
# doc: a list of frame signatures that should always be considered top of
# the stack if present in the stack
# converter: eval
trans_config.processor.c_signature.signature_sentinels = \
config.signatureSentinels
# name: processor.c_signature.signatures_with_line_numbers_re
# doc: any signatures that match this list should be combined with their
# associated source code line numbers
# converter: str
trans_config.processor.c_signature.signatures_with_line_numbers_re = \
config.signaturesWithLineNumbersRegEx
#--------------------------------------------------------------------------
# java_signature -
trans_config.processor.java_signature = DotDict()
# name: processor.java_signature.java_signature_tool_class
# doc: the class that can generate a Java signature
# converter: configman.converters.class_converter
trans_config.processor.java_signature.java_signature_tool_class = \
socorro.processor.signature_utilities.JavaSignatureTool
#--------------------------------------------------------------------------
# producer_consumer -
trans_config.producer_consumer = DotDict()
# name: producer_consumer.idle_delay
# doc: the delay in seconds if no job is found
# converter: int
trans_config.producer_consumer.idle_delay = \
config.processorLoopTime.seconds
# name: producer_consumer.maximum_queue_size
# doc: the maximum size of the internal queue
# converter: int
trans_config.producer_consumer.maximum_queue_size = \
config.numberOfThreads * 2
# name: producer_consumer.number_of_threads
# doc: the number of threads
# converter: int
trans_config.producer_consumer.number_of_threads = config.numberOfThreads
# name: producer_consumer.producer_consumer_class
# doc: the class implements a threaded producer consumer queue
# converter: configman.converters.class_converter
import socorro.lib.threaded_task_manager
trans_config.producer_consumer.producer_consumer_class = \
socorro.lib.threaded_task_manager.ThreadedTaskManager
#--------------------------------------------------------------------------
# registrar -
trans_config.registrar = DotDict()
# name: registrar.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.registrar.backoff_delays = [10, 30, 60, 120, 300, 300, 300,
300, 300, 300]
# name: registrar.check_in_frequency
# doc: how often the processor is required to reregister (hh:mm:ss)
# converter: configman.converters.timedelta_converter
trans_config.registrar.check_in_frequency = \
config.processorCheckInFrequency
# name: registrar.database
# doc: the class of the registrar's database
# converter: configman.converters.class_converter
trans_config.registrar.database = \
socorro.external.postgresql.connection_context.ConnectionContext
# name: registrar.database_host
# doc: the hostname of the database
# converter: str
trans_config.registrar.database_host = config.databaseHost
# name: registrar.database_name
# doc: the name of the database
# converter: str
trans_config.registrar.database_name = config.databaseName
# name: registrar.database_password
# doc: the user's database password
# converter: str
trans_config.registrar.database_password = config.databasePassword
# name: registrar.database_port
# doc: the port for the database
# converter: int
trans_config.registrar.database_port = config.databasePort
# name: registrar.database_user
# doc: the name of the user within the database
# converter: str
trans_config.registrar.database_user = config.databaseUserName
# name: registrar.processor_id
# doc: the id number for the processor (must already exist) (0 for create
# new Id, "auto" for autodetection, "host" for same host)
# converter: str
trans_config.registrar.processor_id = config.processorId
# name: registrar.registrar_class
# doc: the class that registers and tracks processors
# converter: configman.converters.class_converter
import socorro.processor.registration_client
trans_config.registrar.registrar_class = \
socorro.processor.registration_client.ProcessorAppRegistrationClient
# name: registrar.transaction_executor_class
# doc: a class that will manage transactions
# converter: configman.converters.class_converter
trans_config.registrar.transaction_executor_class = (
socorro.database.transaction_executor
.TransactionExecutorWithLimitedBackoff
)
# name: registrar.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.registrar.wait_log_interval = 5
#--------------------------------------------------------------------------
# source -
trans_config.source = DotDict()
# name: source.backoff_delays
# doc: delays in seconds between retries
# converter: eval
trans_config.source.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300,
300, 300]
# name: source.crashstorage_class
# doc: the source storage class
# converter: configman.converters.class_converter
trans_config.source.crashstorage_class = \
socorro.external.hbase.crashstorage.HBaseCrashStorage
# name: source.forbidden_keys
# doc: a comma delimited list of keys banned from the processed crash in
# HBase
# converter: socorro.external.hbase.crashstorage.<lambda>
trans_config.source.forbidden_keys = \
['email', 'url', 'user_id', 'exploitability']
# name: source.hbase_host
# doc: Host to HBase server
# converter: str
trans_config.source.hbase_host = config.hbaseHost
# name: source.hbase_port
# doc: Port to HBase server
# converter: int
trans_config.source.hbase_port = config.hbasePort
# name: source.hbase_timeout
# doc: timeout in milliseconds for an HBase connection
# converter: int
trans_config.source.hbase_timeout = config.hbaseTimeout
# name: source.number_of_retries
# doc: Max. number of retries when fetching from hbaseClient
# converter: int
trans_config.source.number_of_retries = 2
# name: source.transaction_executor_class
# doc: a class that will execute transactions
# converter: configman.converters.class_converter
trans_config.source.transaction_executor_class = (
socorro.database.transaction_executor
.TransactionExecutorWithLimitedBackoff
)
# name: source.wait_log_interval
# doc: seconds between log during retries
# converter: int
trans_config.source.wait_log_interval = 5
# name: destination.storage1.temporary_file_system_storage_path
# doc: a local filesystem path dumps can be temporarily written
# for processing
# converter: str
trans_config.source.temporary_file_system_storage_path = \
config.temporaryFileSystemStoragePath
# name: destination.storage1.dump_file_suffix
# doc: the suffix used to identify a dump file (for use in temp files)
# converter: str
trans_config.source.dump_file_suffix = config.dumpFileSuffix
#--------------------------------------------------------------------------
from socorro.app.generic_app import main
from socorro.processor.processor_app import ProcessorApp
main(ProcessorApp, [trans_config])
default=ElasticSearchCrashStorage,
doc='The class to use to store crash reports in elasticsearch.')
required_config.add_option('processed_crash_file',
default='./testcrash/processed_crash.json',
doc='The file containing the processed crash.')
def main(self):
storage = self.config.elasticsearch_storage_class(self.config)
crash_file = open(self.config.processed_crash_file)
processed_crash = json.load(crash_file)
es_index = storage.get_index_for_crash(processed_crash)
es_doctype = self.config.elasticsearch_doctype
crash_id = processed_crash['uuid']
storage.save_processed(processed_crash)
# Verify the crash has been inserted
es = pyelasticsearch.ElasticSearch(self.config.elasticsearch_urls)
crash = es.get(es_index, es_doctype, crash_id)
assert crash['exists']
print 'Success - %s/%s/%s' % (es_index, es_doctype, crash_id)
if __name__ == '__main__':
generic_app.main(IntegrationTestElasticsearchStorageApp)
print 'done'
class IntegrationTestAutomaticEmailsApp(generic_app.App):
app_name = 'test_automatic_emails'
app_version = '0.1'
app_description = __doc__
required_config = Namespace()
required_config.add_option(
'automatic_emails_class',
default=AutomaticEmailsCronApp,
doc='The class to use to send automatic emails.'
)
required_config.add_option(
'tester_email_address',
default='',
doc='Send the automatic email to this address.'
)
def main(self):
emailer = self.config.automatic_emails_class(self.config, '')
report = {
'email': self.config.tester_email_address,
}
print emailer.send_email(report)
if __name__ == '__main__':
generic_app.main(IntegrationTestAutomaticEmailsApp)
def connect(self):
print "connecting"
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self.config.test_rabbitmq.rabbitmq_host,
port=self.config.test_rabbitmq.rabbitmq_port,
virtual_host=self.config.test_rabbitmq.rabbitmq_vhost,
credentials=pika.credentials.PlainCredentials(
self.config.test_rabbitmq.rabbitmq_user,
self.config.test_rabbitmq.rabbitmq_password)))
except:
print "Failed to connect"
raise
self.connection = connection
def purge_queue(self, queue):
self.connect()
channel = self.connection.channel()
try:
channel.queue_delete(queue=queue)
except pika.exceptions.ChannelClosed, e:
print "Could not find queue %s" % queue
print "Deleted queue %s" % queue
if __name__ == '__main__':
main(TestRabbitMQApp)
import os
from socorro.app.generic_app import main
from collector.collector_app import CollectorApp
from socorro.webapi.servers import ApacheModWSGI
import collector.collector_app
from configman import ConfigFileFutureProxy
if os.path.isfile('/etc/socorro/collector.ini'):
config_path = '/etc/socorro'
else:
config_path = ApacheModWSGI.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
CollectorApp, # the socorro app class
config_path=config_path)
application = collector.collector_app.application
self.config.logger.critical("something's gone horribly wrong",
exc_info=True)
self.quit = True
finally:
self.config.logger.info("priorityLoop done.")
#--------------------------------------------------------------------------
def main(self):
"""this function is run by the main thread. It just starts the
subordinate threads and then waits for them to complete."""
standard_job_thread = threading.Thread(
name="standard_job_thread", target=self._standard_job_thread)
standard_job_thread.start()
priority_job_thread = threading.Thread(
name="priority_job_thread", target=self._priority_job_thread)
priority_job_thread.start()
job_cleanup_thread = threading.Thread(name="job_cleanup_thread",
target=self._job_cleanup_thread)
job_cleanup_thread.start()
self.config.logger.debug("waiting to join.")
self._responsive_join(job_cleanup_thread)
self._responsive_join(priority_job_thread)
self._responsive_join(standard_job_thread)
if __name__ == '__main__':
main(MonitorApp)
import os
from socorro.app.generic_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import ApacheModWSGI
import socorro.middleware.middleware_app
from configman import (
ConfigFileFutureProxy,
environment
)
if os.path.isfile('/etc/socorro/middleware.ini'):
config_path = '/etc/socorro'
else:
config_path = ApacheModWSGI.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
MiddlewareApp, # the socorro app class
config_path=config_path,
values_source_list=[
ConfigFileFutureProxy,
environment
]
)
application = socorro.middleware.middleware_app.application
import os
from socorro.app.generic_app import main
from socorro.collector.collector_app import CollectorApp
from socorro.webapi.servers import ApacheModWSGI
import socorro.collector.collector_app
from configman import ConfigFileFutureProxy
if os.path.isfile('/etc/socorro/collector.ini'):
config_path = '/etc/socorro'
else:
config_path = ApacheModWSGI.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
CollectorApp, # the socorro app class
config_path=config_path
)
application = socorro.collector.collector_app.application
from socorro.app import generic_app
from socorro.cron.jobs.automatic_emails import AutomaticEmailsCronApp
class IntegrationTestAutomaticEmailsApp(generic_app.App):
app_name = 'test_automatic_emails'
app_version = '0.1'
app_description = __doc__
required_config = Namespace()
required_config.add_option(
'automatic_emails_class',
default=AutomaticEmailsCronApp,
doc='The class to use to send automatic emails.')
required_config.add_option('tester_email_address',
default='',
doc='Send the automatic email to this address.')
def main(self):
emailer = self.config.automatic_emails_class(self.config, '')
report = {
'email': self.config.tester_email_address,
}
print emailer.send_email(report)
if __name__ == '__main__':
generic_app.main(IntegrationTestAutomaticEmailsApp)
'fields': es_fields,
'size': es_size,
'from': es_from
}
return self.es_storage.es.search(
es_query,
index=index,
)['hits']
def get_index_for_date(self, date, index_format):
"""return the elasticsearch index for a date"""
if not index_format:
return None
if '%' in index_format:
return date.strftime(index_format)
return index_format
def index_crashes(self, es_index, crashes_to_index):
self.es_storage.es.bulk_index(
es_index,
self.config.elasticsearch.elasticsearch_doctype,
crashes_to_index,
id_field='crash_id'
)
if __name__ == '__main__':
generic_app.main(ElasticsearchBackfillApp)
services_list = []
# populate the 'services_list' with the tuples that will define the
# urls and services offered by dataservice.
for impl_class_namespace in (
self.config.services.service_list.subordinate_namespace_names
):
impl_class = self.config.services[impl_class_namespace].cls
services_list.append(impl_class)
if not services_list:
raise RuntimeError(
"No services configured."
"See the [services].service_list setting in the config file"
)
self.web_server = self.config.web_server.wsgi_server_class(
self.config,
services_list
)
# for modwsgi the 'run' method returns the wsgi function that
# will use. For other webservers, the 'run' method actually starts
# the standalone web server.
application = self.web_server.run()
#==============================================================================
if __name__ == '__main__':
main(DataserviceApp)
db_version = db.version()
with open('sql/roles.sql') as f:
db.execute(f.read())
try:
db.execute('CREATE EXTENSION citext')
except ProgrammingError, e:
if re.match('type "citext" already exists',
e.pgerror.strip().split('ERROR: ')[1]):
# already done, no need to rerun
# pass ok
pass
if not self.no_schema:
with open('sql/schema.sql') as f:
db.execute(
f.read(),
['type "citext" already exists'],
)
db.execute('SELECT weekly_report_partitions()')
else:
db.execute('ALTER DATABASE %s OWNER TO breakpad_rw' %
self.database_name)
return 0
if __name__ == '__main__': # pragma: no cover
sys.exit(main(SocorroDB))
needs a database connection because instead of doing an insert
it just prints. However, it primes the connection by getting a cursor
out first (otherwise it'd have to do it every time in a loo[).
"""
def cursor(self):
pass
class FTPScraperCronAppRunner(FTPScraperCronApp): # pragma: no cover
required_config = Namespace()
required_config.add_option(
'date',
default=datetime.datetime.utcnow(),
doc='Date to run for',
from_string_converter='socorro.lib.datetimeutil.string_to_datetime'
)
def __init__(self, config):
self.config = config
self.config.dry_run = True
def main(self):
assert self.config.dry_run
self.run(_MockConnection(), self.config.date)
if __name__ == '__main__': # pragma: no cover
sys.exit(main(FTPScraperCronAppRunner))
needs a database connection because instead of doing an insert
it just prints. However, it primes the connection by getting a cursor
out first (otherwise it'd have to do it every time in a loo[).
"""
def cursor(self):
pass
class FTPScraperCronAppRunner(FTPScraperCronApp): # pragma: no cover
required_config = Namespace()
required_config.add_option(
'date',
default=datetime.datetime.utcnow(),
doc='Date to run for',
from_string_converter='socorro.lib.datetimeutil.string_to_datetime'
)
def __init__(self, config):
self.config = config
self.config.dry_run = True
def main(self):
assert self.config.dry_run
self.run(_MockConnection(), self.config.date)
if __name__ == '__main__': # pragma: no cover
sys.exit(main(FTPScraperCronAppRunner))
import os
from socorro.app.generic_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import WSGIServer
import socorro.middleware.middleware_app
from configman import (ConfigFileFutureProxy, environment)
if os.path.isfile('/etc/socorro/middleware.ini'):
config_path = '/etc/socorro'
else:
config_path = WSGIServer.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
MiddlewareApp, # the socorro app class
config_path=config_path,
values_source_list=[ConfigFileFutureProxy, environment])
application = socorro.middleware.middleware_app.application
#--------------------------------------------------------------------------
required_config.namespace('web_server')
required_config.web_server.add_option(
'wsgi_server_class',
doc='a class implementing a wsgi web server',
default='socorro.webapi.servers.WSGIServer',
from_string_converter=class_converter)
#--------------------------------------------------------------------------
def main(self):
# modwsgi requireds a module level name 'application'
global application
services_list = (self.config.collector.collector_class, )
self.config.crash_storage = self.config.storage.crashstorage_class(
self.config.storage)
self.config.throttler = self.config.throttler.throttler_class(
self.config.throttler)
self.web_server = self.config.web_server.wsgi_server_class(
self.config, # needs the whole config not the local namespace
services_list)
# for modwsgi the 'run' method returns the wsgi function that the web
# server will use. For other webservers, the 'run' method actually
# starts the standalone web server.
application = self.web_server.run()
if __name__ == '__main__':
main(CollectorApp)
params[args[i]] = args[i + 1]
except IndexError:
pass
for key, value in params.iteritems():
if value.count(terms_sep) and not DONT_TERM_SPLIT.match(value):
params[key] = value.split(terms_sep)
return params
@staticmethod
def decode_special_characters(value):
"""Return a decoded string or list of strings.
Because characters '/' and '+' are special in our URL scheme, we need
to double-encode them in the client. This function is to decode them
so our service can use them as expected.
"""
def convert(s):
return s.replace("%2B", "+").replace("%2F", "/")
if isinstance(value, (list, tuple)):
return [convert(x) for x in value]
assert isinstance(value, basestring)
return convert(value)
if __name__ == '__main__':
main(MiddlewareApp)
# Get data from PostgreSQL.
with pg_context() as pg_connection:
cursor = pg_connection.cursor()
sql = """
SELECT email, last_sending
FROM emails
WHERE last_sending >= %s
"""
cursor.execute(sql, (start_date, ))
results = cursor.fetchall()
if not results:
self.config.logger.info('No data in emails table in postgres')
return
for row in results:
# self.config.logger.info('putting %s into ES' % row[0])
if not row[0] or not row[0].strip():
continue
es_connection.index(
index=self.config.elasticsearch.elasticsearch_emails_index,
doc_type='emails',
doc={'last_sending': row[1]},
id=row[0])
if __name__ == '__main__':
generic_app.main(MoveEmailsApp)
required_config.add_option(
'hbase_crash_storage_class',
default=HappyBaseCrashStorage,
doc='the class responsible for proving an hbase connection',
from_string_converter=class_converter)
required_config.add_option(
'command',
default=help,
doc='command to use',
is_argument=True,
from_string_converter=lambda s: class_converter(__name__ + '.' + s))
required_config.add_option(
'json',
default=False,
short_form='j',
doc='json output instead of a pretty printed mapping',
)
def main(self):
self.storage = self.config.hbase_crash_storage_class(self.config)
self.config.command(self).run()
if __name__ == '__main__':
try:
generic_app.main(HBaseClientApp,
config_manager_cls=HBaseClientConfigurationManager)
except NotEnoughArguments as e:
print >> sys.stderr, "ERROR: was expecting another argument: " + e.arg
print >> sys.stderr, "Use the 'help' command to get help on commands."
self.config.services.services_controller.service_list
):
services_list.append(
# a tuple associating a URI with a service class
(
uri,
# a binding of the service class with the configuration
# namespace for that service class
class_with_partial_init(
self.config.services[namespace]
.service_implementation_class,
self.config.services[namespace],
self.config
)
)
)
# initialize the wsgi server with the list of URI/service impl tuples
self.web_server = self.config.web_server.wsgi_server_class(
self.config, # needs the whole config not the local namespace
services_list
)
# for modwsgi the 'run' method returns the wsgi function that the web
# server will use. For other webservers, the 'run' method actually
# starts the standalone web server.
application = self.web_server.run()
if __name__ == '__main__':
main(CollectorApp)
if not index:
return None
if '%' in index:
index = day.strftime(index)
return index
def format_dates_in_crash(self, processed_crash):
# HBase returns dates in a format that elasticsearch does not
# understand. To keep our elasticsearch mapping simple, we
# transform all dates to a recognized format.
for attr in processed_crash:
try:
processed_crash[attr] = datetimeutil.date_to_string(
datetimeutil.string_to_datetime(processed_crash[attr]))
except (ValueError, TypeError, ISO8601Error):
# the attribute is not a date
pass
return processed_crash
def index_crashes(self, es_index, crashes_to_index):
self.es_storage.es.bulk_index(es_index,
self.config.elasticsearch_doctype,
crashes_to_index,
id_field='uuid')
if __name__ == '__main__':
generic_app.main(ElasticsearchBackfillApp)
cursor = pg_connection.cursor()
sql = """
SELECT email, last_sending
FROM emails
WHERE last_sending >= %s
"""
cursor.execute(sql, (start_date,))
results = cursor.fetchall()
if not results:
self.config.logger.info('No data in emails table in postgres')
return
for row in results:
# self.config.logger.info('putting %s into ES' % row[0])
if not row[0] or not row[0].strip():
continue
es_connection.index(
index=self.config.elasticsearch.elasticsearch_emails_index,
doc_type='emails',
doc={
'last_sending': row[1]
},
id=row[0]
)
if __name__ == '__main__':
generic_app.main(MoveEmailsApp)
credentials=pika.credentials.PlainCredentials(
config.rabbitmq_user,
config.rabbitmq_password))
)
except:
logging.error("Failed to connect")
raise
self.connection = connection
def main(self):
self.connect()
channel = self.connection.channel()
channel.queue_declare(queue='socorro.reprocessing', durable=True)
with open(self.config.reprocesscrashlist.crashes, 'r') as file:
for uuid in file.read().splitlines():
channel.basic_publish(
exchange='',
routing_key="socorro.reprocessing",
body=uuid,
properties=pika.BasicProperties(delivery_mode=2)
)
logging.debug('submitted %s' % uuid)
self.connection.close()
if __name__ == '__main__':
main(ReprocessCrashlistApp)
#--------------------------------------------------------------------------
def main(self):
# modwsgi requireds a module level name 'application'
global application
services_list = []
# populate the 'services_list' with the tuples that will define the
# urls and services offered by dataservice.
for impl_class_namespace in (
self.config.services.service_list.subordinate_namespace_names):
impl_class = self.config.services[impl_class_namespace].cls
services_list.append(impl_class)
if not services_list:
raise RuntimeError(
"No services configured."
"See the [services].service_list setting in the config file")
self.web_server = self.config.web_server.wsgi_server_class(
self.config, services_list)
# for modwsgi the 'run' method returns the wsgi function that
# will use. For other webservers, the 'run' method actually starts
# the standalone web server.
application = self.web_server.run()
#==============================================================================
if __name__ == '__main__':
main(DataserviceApp)
channel.close()
def connect(self):
print "connecting"
try:
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=self.config.test_rabbitmq.rabbitmq_host,
port=self.config.test_rabbitmq.rabbitmq_port,
virtual_host=self.config.test_rabbitmq.rabbitmq_vhost,
credentials=pika.credentials.PlainCredentials(
self.config.test_rabbitmq.rabbitmq_user,
self.config.test_rabbitmq.rabbitmq_password)))
except:
print "Failed to connect"
raise
self.connection = connection
def purge_queue(self, queue):
self.connect()
channel = self.connection.channel()
try:
channel.queue_delete(queue=queue)
except pika.exceptions.ChannelClosed, e:
print "Could not find queue %s" % queue
print "Deleted queue %s" % queue
if __name__ == '__main__':
main(TestRabbitMQApp)
host=config.host,
port=config.port,
virtual_host=config.virtual_host,
credentials=pika.credentials.PlainCredentials(
config.rabbitmq_user, config.rabbitmq_password)))
except:
logging.error("Failed to connect")
raise
self.connection = connection
def main(self):
self.connect()
channel = self.connection.channel()
channel.queue_declare(queue='socorro.reprocessing', durable=True)
with open(self.config.reprocesscrashlist.crashes, 'r') as file:
for uuid in file.read().splitlines():
channel.basic_publish(
exchange='',
routing_key="socorro.reprocessing",
body=uuid,
properties=pika.BasicProperties(delivery_mode=2))
logging.debug('submitted %s' % uuid)
self.connection.close()
if __name__ == '__main__':
main(ReprocessCrashlistApp)
doc='The file containing the processed crash.'
)
def main(self):
storage = self.config.elasticsearch_storage_class(self.config)
crash_file = open(self.config.processed_crash_file)
processed_crash = json.load(crash_file)
es_index = storage.get_index_for_crash(processed_crash)
es_doctype = self.config.elasticsearch_doctype
crash_id = processed_crash['uuid']
storage.save_processed(processed_crash)
# Verify the crash has been inserted
es = pyelasticsearch.ElasticSearch(self.config.elasticsearch_urls)
crash = es.get(
es_index,
es_doctype,
crash_id
)
assert crash['exists']
print 'Success - %s/%s/%s' % (es_index, es_doctype, crash_id)
if __name__ == '__main__':
generic_app.main(IntegrationTestElasticsearchStorageApp)
print 'done'
raise
dsn = dsn_template % self.database_name
with PostgreSQLManager(dsn, self.config.logger) as db:
db_version = db.version()
with open('sql/roles.sql') as f:
db.execute(f.read())
if not self.no_schema:
with open('sql/schema.sql') as f:
db.execute(f.read(), ['schema "pgx_diag" already exists'])
db.execute('SELECT weekly_report_partitions()')
else:
db.execute(
'ALTER DATABASE %s OWNER TO breakpad_rw' %
self.database_name)
if re.match(r'9\.0[.*]', db_version):
with open(self.citext) as f:
db.execute(f.read())
elif re.match(r'9\.1[.*]', db_version):
db.execute('CREATE EXTENSION citext from unpackaged')
return 0
if __name__ == '__main__': # pragma: no cover
sys.exit(main(SocorroDB))
app_version = "0.1"
app_description = __doc__
required_config = Namespace()
required_config.add_option(
'hbase_crash_storage_class',
default=HBaseCrashStorage,
doc='the class responsible for proving an hbase connection',
from_string_converter=class_converter
)
required_config.add_option(
'command',
default=help,
doc='command to use',
from_string_converter=lambda s: class_converter(__name__ + '.' + s)
)
def main(self):
self.storage = self.config.hbase_crash_storage_class(self.config)
self.config.command(self).run()
if __name__ == '__main__':
try:
generic_app.main(HBaseClientApp,
config_manager_cls=HBaseClientConfigurationManager)
except NotEnoughArguments as e:
print >> sys.stderr, "ERROR: was expecting another argument: " + e.arg
print >> sys.stderr, "Use the 'help' command to get help on commands."
for class_name, __ in class_list:
class_config = self.config.crontabber['class-%s' % class_name]
if not self._configtest_one(class_config):
failed += 1
return not failed
def _configtest_one(self, config):
try:
seconds = convert_frequency(config.frequency)
time_ = config.time
if time_:
check_time(time_)
# if less than 1 day, it doesn't make sense to specify hour
if seconds < 60 * 60 * 24:
raise FrequencyDefinitionError(config.time)
return True
except (JobNotFoundError,
JobDescriptionError,
FrequencyDefinitionError,
TimeDefinitionError):
exc_type, exc_value, exc_tb = sys.exc_info()
print >>sys.stderr, "Error type:", exc_type
print >>sys.stderr, "Error value:", exc_value
print >>sys.stderr, ''.join(traceback.format_tb(exc_tb))
return False
if __name__ == '__main__': # pragma: no cover
sys.exit(main(CronTabber))
)
# Verify data was correctly inserted.
es_connection.refresh()
total_indexed = es_connection.count(
'*',
index='socorro',
doc_type='supersearch_fields',
)['count']
total_expected = len(all_fields)
if total_expected != total_indexed:
indexed_fields = es_connection.search(
'*',
index='socorro',
doc_type='supersearch_fields',
size=total_indexed,
)
indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]
self.config.logger.error(
'The SuperSearch fields data was not correctly indexed, '
'%s fields are missing from the database. Missing fields: %s',
total_expected - total_indexed,
list(set(all_fields.keys()) - set(indexed_fields))
)
if __name__ == '__main__':
generic_app.main(SetupSuperSearchApp)
params[args[i]] = args[i + 1]
except IndexError:
pass
for key, value in params.iteritems():
if value.count(terms_sep) and not DONT_TERM_SPLIT.match(value):
params[key] = value.split(terms_sep)
return params
@staticmethod
def decode_special_characters(value):
"""Return a decoded string or list of strings.
Because characters '/' and '+' are special in our URL scheme, we need
to double-encode them in the client. This function is to decode them
so our service can use them as expected.
"""
def convert(s):
return s.replace("%2B", "+").replace("%2F", "/")
if isinstance(value, (list, tuple)):
return [convert(x) for x in value]
assert isinstance(value, basestring)
return convert(value)
if __name__ == '__main__':
main(MiddlewareApp)
ekly-report-partitions
See https://bugzilla.mozilla.org/show_bug.cgi?id=701253
"""
class WeeklyReportsPartitions(App):
app_name = 'weekly_reports_partitions'
app_version = '0.1'
app_description = __doc__
required_config = Namespace()
required_config.add_option('transaction_executor_class',
default=TransactionExecutorWithBackoff,
#default=TransactionExecutor,
doc='a class that will execute transactions')
def run_query(self, connection):
cursor = connection.cursor()
cursor.execute('SELECT weekly_report_partitions()')
def main(self):
executor = self.config.transaction_executor_class(self.config)
executor(self.run_query)
# alternative use could be like this:
# with self.config.transaction_executor_class(self.config) as connection:
# self.run_query(connection)
if __name__ == '__main__': # pragma: no cover
main(WeeklyReportsPartitions)
from configman import Namespace
class ExampleApp(App):
app_name = 'example'
app_version = '0.1'
app_description = __doc__
# in this section, define any configuration requirements
required_config = Namespace()
required_config.add_option('name', default='Wilma', doc='a name to echo')
required_config.add_option('time',
default=datetime.datetime.now(),
doc='the time of day')
# implementing this constructor is only necessary when there is more
# initialization to be done before main can be called
#def __init__(self, config):
#super(ExampleApp,self).__init__(config)
def main(self):
# this is where we'd implement the app
# the configuraton is already setup as self.config
print 'hello, %s. The time is: %s' % (self.config.name,
self.config.time)
if __name__ == '__main__':
main(ExampleApp)
import os
from socorro.app.generic_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import ApacheModWSGI
import socorro.middleware.middleware_app
if os.path.isfile('/etc/socorro/middleware.ini'):
config_path = '/etc/socorro'
else:
config_path = ApacheModWSGI.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
MiddlewareApp, # the socorro app class
config_path=config_path
)
application = socorro.middleware.middleware_app.application
