def test_exit_codes(self, logging): vsl = (ConfigFileFutureProxy, {'exit_code': 123}) exit_code = main(ExitingApp, values_source_list=vsl) self.assertEqual(exit_code, 123) vsl = (ConfigFileFutureProxy, {'exit_code': 0}) exit_code = main(ExitingApp, values_source_list=vsl) self.assertEqual(exit_code, 0) vsl = (ConfigFileFutureProxy, {'exit_code': None}) exit_code = main(ExitingApp, values_source_list=vsl) self.assertEqual(exit_code, 0)
def test_run_weeklyReportsPartitions(self): """Create a mock function named exactly like the stored procedure in the cron script we want to functionally test. That way we can assert that it was run. """ # provide values for the config to pick up # be explicit about the values_source_list to # avoid configman picking up nosetests arguments main(WeeklyReportsPartitions, values_source_list=[DSN]) # check that something was written to the mock_bucket cursor = self.conn.cursor() cursor.execute('select count(*) from mock_bucket;') self.assertTrue(cursor.fetchone())
def test_overriding_config_path(self, logging): vsl = (ConfigFileFutureProxy, ) exit_code = main(MyApp, values_source_list=vsl) self.assertEqual(exit_code, 0) os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = '/foo/bar' self.assertRaises(IOError, main, (MyApp, ), values_source_list=vsl) os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = self.tempdir exit_code = main(MyApp, values_source_list=vsl) self.assertEqual(exit_code, 0) logging.getLogger().error.assert_called_with('colour') _ini_file = os.path.join(self.tempdir, 'myapp.ini') with open(_ini_file, 'w') as f: f.write('color_or_colour=color\n') exit_code = main(MyApp, values_source_list=vsl) self.assertEqual(exit_code, 0) logging.getLogger().error.assert_called_with('color')
def test_overriding_config_path(self, logging): vsl = (ConfigFileFutureProxy,) exit_code = main(MyApp, values_source_list=vsl) self.assertEqual(exit_code, 0) os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = '/foo/bar' self.assertRaises(IOError, main, (MyApp,), values_source_list=vsl) os.environ['DEFAULT_SOCORRO_CONFIG_PATH'] = self.tempdir exit_code = main(MyApp, values_source_list=vsl) self.assertEqual(exit_code, 0) logging.getLogger().error.assert_called_with('colour') _ini_file = os.path.join(self.tempdir, 'myapp.ini') with open(_ini_file, 'w') as f: f.write('color_or_colour=color\n') exit_code = main(MyApp, values_source_list=vsl) self.assertEqual(exit_code, 0) logging.getLogger().error.assert_called_with('color')
app_name = 'example' app_version = '0.1' app_description = __doc__ #-------------------------------------------------------------------------- # in this section, define any configuration requirements required_config = Namespace() required_config.add_option('name', default='Wilma', doc='a name to echo') required_config.add_option('time', default=datetime.datetime.now(), doc='the time of day') #-------------------------------------------------------------------------- # implementing this constructor is only necessary when there is more # initialization to be done before main can be called #def __init__(self, config): #super(ExampleApp,self).__init__(config) #-------------------------------------------------------------------------- def main(self): # this is where we'd implement the app # the configuraton is already setup as self.config print 'hello, %s. The time is: %s' % (self.config.name, self.config.time) if __name__ == '__main__': main(ExampleApp)
def processor2012(config): from configman.dotdict import DotDict # the following section is a translation of old-style configuration into # the new style configuration. It lets a new style app run without leaving # the old configuration system. trans_config = DotDict() #-------------------------------------------------------------------------- # destination - trans_config.destination = DotDict() # name: destination.crashstorage_class # doc: the destination storage class # converter: configman.converters.class_converter import socorro.external.crashstorage_base trans_config.destination.crashstorage_class = \ socorro.external.crashstorage_base.PolyCrashStorage # name: destination.storage_classes # doc: a comma delimited list of storage classes # converter: configman.converters.class_list_converter import socorro.external.postgresql.crashstorage import socorro.external.hbase.crashstorage import socorro.external.elasticsearch.crashstorage trans_config.destination.storage_classes = ''' socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage, socorro.external.hbase.crashstorage.HBaseCrashStorage, socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage ''' #-------------------------------------------------------------------------- # storage0 - trans_config.destination.storage0 = DotDict() # name: destination.storage0.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.destination.storage0.backoff_delays = [ 10, 30, 60, 120, 300, 300, 300, 300, 300, 300 ] # name: destination.storage0.crashstorage_class # doc: None # converter: configman.converters.class_converter trans_config.destination.storage0.crashstorage_class = \ socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage # name: destination.storage0.database_class # doc: the class responsible for connecting toPostgres # converter: configman.converters.class_converter import socorro.external.postgresql.connection_context trans_config.destination.storage0.database_class = \ socorro.external.postgresql.connection_context.ConnectionContext # name: destination.storage0.database_host # doc: the hostname of the database # converter: str trans_config.destination.storage0.database_host = config.databaseHost # name: destination.storage0.database_name # doc: the name of the database # converter: str trans_config.destination.storage0.database_name = config.databaseName # name: destination.storage0.database_password # doc: the user's database password # converter: str trans_config.destination.storage0.database_password = \ config.databasePassword # name: destination.storage0.database_port # doc: the port for the database # converter: int trans_config.destination.storage0.database_port = config.databasePort # name: destination.storage0.database_user # doc: the name of the user within the database # converter: str trans_config.destination.storage0.database_user = config.databaseUserName # name: destination.storage0.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter import socorro.database.transaction_executor trans_config.destination.storage0.transaction_executor_class = ( socorro.database.transaction_executor. TransactionExecutorWithLimitedBackoff) # name: destination.storage0.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.destination.storage0.wait_log_interval = 5 #-------------------------------------------------------------------------- # storage1 - trans_config.destination.storage1 = DotDict() # name: destination.storage1.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.destination.storage1.backoff_delays = [ 10, 30, 60, 120, 300, 300, 300, 300, 300, 300 ] # name: destination.storage1.crashstorage_class # doc: None # converter: configman.converters.class_converter trans_config.destination.storage1.crashstorage_class = \ socorro.external.hbase.crashstorage.HBaseCrashStorage # name: destination.storage1.forbidden_keys # doc: a comma delimited list of keys banned from the processed crash in # HBase # converter: socorro.external.hbase.crashstorage.<lambda> trans_config.destination.storage1.forbidden_keys = \ ['email', 'url', 'user_id', 'exploitability'] # name: destination.storage1.hbase_host # doc: Host to HBase server # converter: str trans_config.destination.storage1.hbase_host = config.hbaseHost # name: destination.storage1.hbase_port # doc: Port to HBase server # converter: int trans_config.destination.storage1.hbase_port = config.hbasePort # name: destination.storage1.hbase_timeout # doc: timeout in milliseconds for an HBase connection # converter: int trans_config.destination.storage1.hbase_timeout = config.hbaseTimeout # name: destination.storage1.number_of_retries # doc: Max. number of retries when fetching from hbaseClient # converter: int trans_config.destination.storage1.number_of_retries = 2 # name: destination.storage1.transaction_executor_class # doc: a class that will execute transactions # converter: configman.converters.class_converter trans_config.destination.storage1.transaction_executor_class = ( socorro.database.transaction_executor. TransactionExecutorWithLimitedBackoff) # name: destination.storage1.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.destination.storage1.wait_log_interval = 5 # name: destination.storage1.temporary_file_system_storage_path # doc: a local filesystem path dumps can be temporarily written # for processing # converter: str trans_config.destination.storage1.temporary_file_system_storage_path = \ config.temporaryFileSystemStoragePath # name: destination.storage1.dump_file_suffix # doc: the suffix used to identify a dump file (for use in temp files) # converter: str trans_config.destination.storage1.dump_file_suffix = \ config.dumpFileSuffix #-------------------------------------------------------------------------- # storage2 - trans_config.destination.storage2 = DotDict() # name: destination.storage2.crashstorage_class # doc: None # converter: configman.converters.class_converter trans_config.destination.storage2.crashstorage_class = \ socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage # name: destination.storage2.elasticsearch_urls # doc: the urls for the elasticsearch instances (leave blank to disable) # converter: str trans_config.destination.storage2.elasticsearch_urls = \ config.elasticsearch_urls # name: destination.storage2.elasticsearch_index # doc: an index to insert crashes in elasticsearch (use datetime's strftime # format to have daily, weekly or monthly indexes) # converter: str trans_config.destination.storage2.elasticsearch_index = \ config.elasticsearch_index # name: destination.storage2.elasticsearch_doctype # doc: a type to insert crashes in elasticsearch # converter: str trans_config.destination.storage2.elasticsearch_doctype = \ config.elasticsearch_doctype # name: destination.storage2.elasticsearch_index_settings # doc: the mapping of crash reports to insert # converter: str trans_config.destination.storage2.elasticsearch_index_settings = \ config.elasticsearch_index_settings # name: destination.storage2.timeout # doc: how long to wait in seconds for confirmation of a submission # converter: int trans_config.destination.storage2.timeout = 2 # name: destination.storage2.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter import socorro.database.transaction_executor trans_config.destination.storage2.transaction_executor_class = \ socorro.database.transaction_executor.TransactionExecutor #-------------------------------------------------------------------------- # logging - trans_config.logging = DotDict() # name: logging.stderr_error_logging_level # doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - # WARNING, 40 - ERROR, 50 - CRITICAL) # converter: int trans_config.logging.stderr_error_logging_level = \ config.stderrErrorLoggingLevel # name: logging.stderr_line_format_string # doc: python logging system format for logging to stderr # converter: str trans_config.logging.stderr_line_format_string = \ config.stderrLineFormatString # name: logging.syslog_error_logging_level # doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, # 40 - ERROR, 50 - CRITICAL) # converter: int trans_config.logging.syslog_error_logging_level = \ config.syslogErrorLoggingLevel # name: logging.syslog_facility_string # doc: syslog facility string ("user", "local0", etc) # converter: str trans_config.logging.syslog_facility_string = config.syslogFacilityString # name: logging.syslog_host # doc: syslog hostname # converter: str trans_config.logging.syslog_host = config.syslogHost # name: logging.syslog_line_format_string # doc: python logging system format for syslog entries # converter: str # NOTE: the old and new systems use very differet formats for # the logging template. In the entire history of the project # the template has not changed. Rather than spending the effort # to translate the format, this hard coding will do. Since this # chimera processor is transintional, this will go away in the # not too distant future. trans_config.logging.syslog_line_format_string = \ '{app_name} (pid {process}): ' \ '{asctime} {levelname} - {threadName} - ' \ '{message}' # name: logging.syslog_port # doc: syslog port # converter: int trans_config.logging.syslog_port = config.syslogPort #-------------------------------------------------------------------------- # new_crash_source - trans_config.new_crash_source = DotDict() # name: new_crash_source.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.new_crash_source.backoff_delays = [ 10, 30, 60, 120, 300, 300, 300, 300, 300, 300 ] # name: new_crash_source.batchJobLimit # doc: the number of jobs to pull in a time # converter: int trans_config.new_crash_source.batchJobLimit = config.batchJobLimit # name: new_crash_source.database_class # doc: the class of the database # converter: configman.converters.class_converter trans_config.new_crash_source.database_class = \ socorro.external.postgresql.connection_context.ConnectionContext # name: new_crash_source.database_host # doc: the hostname of the database # converter: str trans_config.new_crash_source.database_host = config.databaseHost # name: new_crash_source.database_name # doc: the name of the database # converter: str trans_config.new_crash_source.database_name = config.databaseName # name: new_crash_source.database_password # doc: the user's database password # converter: str trans_config.new_crash_source.database_password = config.databasePassword # name: new_crash_source.database_port # doc: the port for the database # converter: int trans_config.new_crash_source.database_port = config.databasePort # name: new_crash_source.database_user # doc: the name of the user within the database # converter: str trans_config.new_crash_source.database_user = config.databaseUserName # name: new_crash_source.new_crash_source_class # doc: an iterable that will stream crash_ids needing processing # converter: configman.converters.class_converter import socorro.processor.legacy_new_crash_source trans_config.new_crash_source.new_crash_source_class = \ socorro.processor.legacy_new_crash_source.LegacyNewCrashSource # name: new_crash_source.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter trans_config.new_crash_source.transaction_executor_class = ( socorro.database.transaction_executor. TransactionExecutorWithLimitedBackoff) # name: new_crash_source.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.new_crash_source.wait_log_interval = 5 #-------------------------------------------------------------------------- # processor - trans_config.processor = DotDict() # name: processor.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.processor.backoff_delays = [ 10, 30, 60, 120, 300, 300, 300, 300, 300, 300 ] # name: processor.collect_addon # doc: boolean indictating if information about add-ons should be collected # converter: configman.converters.boolean_converter trans_config.processor.collect_addon = config.collectAddon # name: processor.collect_crash_process # doc: boolean indictating if information about process type should be # collected # converter: configman.converters.boolean_converter trans_config.processor.collect_crash_process = config.collectCrashProcess # name: processor.crashing_thread_frame_threshold # doc: the number of frames to keep in the raw dump for the crashing thread # converter: int trans_config.processor.crashing_thread_frame_threshold = \ config.crashingThreadFrameThreshold # name: processor.crashing_thread_tail_frame_threshold # doc: the number of frames to keep in the raw dump at the tail of the # frame # list # converter: int trans_config.processor.crashing_thread_tail_frame_threshold = \ config.crashingThreadTailFrameThreshold # name: processor.database_class # doc: the class of the database # converter: configman.converters.class_converter trans_config.processor.database_class = \ socorro.external.postgresql.connection_context.ConnectionContext # name: processor.database_host # doc: the hostname of the database # converter: str trans_config.processor.database_host = config.databaseHost # name: processor.database_name # doc: the name of the database # converter: str trans_config.processor.database_name = config.databaseName # name: processor.database_password # doc: the user's database password # converter: str trans_config.processor.database_password = config.databasePassword # name: processor.database_port # doc: the port for the database # converter: int trans_config.processor.database_port = config.databasePort # name: processor.database_user # doc: the name of the user within the database # converter: str trans_config.processor.database_user = config.databaseUserName # name: processor.known_flash_identifiers # doc: A subset of the known "debug identifiers" for flash versions, # associated to the version # converter: json.loads trans_config.processor.known_flash_identifiers = \ config.knownFlashIdentifiers # name: processor.minidump_stackwalk_pathname # doc: the full pathname of the extern program minidump_stackwalk (quote # path with embedded spaces) # converter: str trans_config.processor.minidump_stackwalk_pathname = \ config.minidump_stackwalkPathname # name: processor.processor_class # doc: the class that transforms raw crashes into processed crashes # converter: configman.converters.class_converter import socorro.processor.legacy_processor trans_config.processor.processor_class = \ socorro.processor.legacy_processor.LegacyCrashProcessor # name: processor.processor_symbols_pathname_list # doc: comma or space separated list of symbol files for minidump_stackwalk # (quote paths with embedded spaces) # converter: socorro.processor.legacy_processor.create_symbol_path_str trans_config.processor.processor_symbols_pathname_list = \ config.processorSymbolsPathnameList # name: processor.stackwalk_command_line # doc: the template for the command to invoke minidump_stackwalk # converter: str trans_config.processor.stackwalk_command_line = \ config.stackwalkCommandLine.replace( 'minidump_stackwalkPathname', 'minidump_stackwalk_pathname' ).replace( 'processorSymbolsPathnameList', 'processor_symbols_pathname_list' ) # name: exploitability_tool_command_line # doc: the template for the command to invoke the exploitability tool # converter: str trans_config.processor.exploitability_tool_command_line = \ config.exploitability_tool_command_line # name: exploitability_tool_pathname # doc: the full pathname of the extern program exploitability tool # (quote path with embedded spaces) # converter: str trans_config.processor.exploitability_tool_pathname = \ config.exploitability_tool_pathname # name: processor.symbol_cache_path # doc: the path where the symbol cache is found (quote path with embedded # spaces) # converter: str trans_config.processor.symbol_cache_path = config.symbolCachePath # name: processor.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter trans_config.processor.transaction_executor_class = ( socorro.database.transaction_executor. TransactionExecutorWithLimitedBackoff) # name: processor.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.processor.wait_log_interval = 5 #-------------------------------------------------------------------------- # c_signature - trans_config.processor.c_signature = DotDict() # name: processor.c_signature.c_signature_tool_class # doc: the class that can generate a C signature # converter: configman.converters.class_converter import socorro.processor.signature_utilities trans_config.processor.c_signature.c_signature_tool_class = \ socorro.processor.signature_utilities.CSignatureTool # name: processor.c_signature.irrelevant_signature_re # doc: a regular expression matching frame signatures that should be # ignored when generating an overall signature # converter: str trans_config.processor.c_signature.irrelevant_signature_re = \ config.irrelevantSignatureRegEx # name: processor.c_signature.prefix_signature_re # doc: a regular expression matching frame signatures that should always be # coupled with the following frame signature when generating an # overall signature # converter: str trans_config.processor.c_signature.prefix_signature_re = \ config.prefixSignatureRegEx # name: processor.c_signature.signature_sentinels # doc: a list of frame signatures that should always be considered top of # the stack if present in the stack # converter: eval trans_config.processor.c_signature.signature_sentinels = \ config.signatureSentinels # name: processor.c_signature.signatures_with_line_numbers_re # doc: any signatures that match this list should be combined with their # associated source code line numbers # converter: str trans_config.processor.c_signature.signatures_with_line_numbers_re = \ config.signaturesWithLineNumbersRegEx #-------------------------------------------------------------------------- # java_signature - trans_config.processor.java_signature = DotDict() # name: processor.java_signature.java_signature_tool_class # doc: the class that can generate a Java signature # converter: configman.converters.class_converter trans_config.processor.java_signature.java_signature_tool_class = \ socorro.processor.signature_utilities.JavaSignatureTool #-------------------------------------------------------------------------- # producer_consumer - trans_config.producer_consumer = DotDict() # name: producer_consumer.idle_delay # doc: the delay in seconds if no job is found # converter: int trans_config.producer_consumer.idle_delay = \ config.processorLoopTime.seconds # name: producer_consumer.maximum_queue_size # doc: the maximum size of the internal queue # converter: int trans_config.producer_consumer.maximum_queue_size = \ config.numberOfThreads * 2 # name: producer_consumer.number_of_threads # doc: the number of threads # converter: int trans_config.producer_consumer.number_of_threads = config.numberOfThreads # name: producer_consumer.producer_consumer_class # doc: the class implements a threaded producer consumer queue # converter: configman.converters.class_converter import socorro.lib.threaded_task_manager trans_config.producer_consumer.producer_consumer_class = \ socorro.lib.threaded_task_manager.ThreadedTaskManager #-------------------------------------------------------------------------- # registrar - trans_config.registrar = DotDict() # name: registrar.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.registrar.backoff_delays = [ 10, 30, 60, 120, 300, 300, 300, 300, 300, 300 ] # name: registrar.check_in_frequency # doc: how often the processor is required to reregister (hh:mm:ss) # converter: configman.converters.timedelta_converter trans_config.registrar.check_in_frequency = \ config.processorCheckInFrequency # name: registrar.database # doc: the class of the registrar's database # converter: configman.converters.class_converter trans_config.registrar.database = \ socorro.external.postgresql.connection_context.ConnectionContext # name: registrar.database_host # doc: the hostname of the database # converter: str trans_config.registrar.database_host = config.databaseHost # name: registrar.database_name # doc: the name of the database # converter: str trans_config.registrar.database_name = config.databaseName # name: registrar.database_password # doc: the user's database password # converter: str trans_config.registrar.database_password = config.databasePassword # name: registrar.database_port # doc: the port for the database # converter: int trans_config.registrar.database_port = config.databasePort # name: registrar.database_user # doc: the name of the user within the database # converter: str trans_config.registrar.database_user = config.databaseUserName # name: registrar.processor_id # doc: the id number for the processor (must already exist) (0 for create # new Id, "auto" for autodetection, "host" for same host) # converter: str trans_config.registrar.processor_id = config.processorId # name: registrar.registrar_class # doc: the class that registers and tracks processors # converter: configman.converters.class_converter import socorro.processor.registration_client trans_config.registrar.registrar_class = \ socorro.processor.registration_client.ProcessorAppRegistrationClient # name: registrar.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter trans_config.registrar.transaction_executor_class = ( socorro.database.transaction_executor. TransactionExecutorWithLimitedBackoff) # name: registrar.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.registrar.wait_log_interval = 5 #-------------------------------------------------------------------------- # source - trans_config.source = DotDict() # name: source.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.source.backoff_delays = [ 10, 30, 60, 120, 300, 300, 300, 300, 300, 300 ] # name: source.crashstorage_class # doc: the source storage class # converter: configman.converters.class_converter trans_config.source.crashstorage_class = \ socorro.external.hbase.crashstorage.HBaseCrashStorage # name: source.forbidden_keys # doc: a comma delimited list of keys banned from the processed crash in # HBase # converter: socorro.external.hbase.crashstorage.<lambda> trans_config.source.forbidden_keys = \ ['email', 'url', 'user_id', 'exploitability'] # name: source.hbase_host # doc: Host to HBase server # converter: str trans_config.source.hbase_host = config.hbaseHost # name: source.hbase_port # doc: Port to HBase server # converter: int trans_config.source.hbase_port = config.hbasePort # name: source.hbase_timeout # doc: timeout in milliseconds for an HBase connection # converter: int trans_config.source.hbase_timeout = config.hbaseTimeout # name: source.number_of_retries # doc: Max. number of retries when fetching from hbaseClient # converter: int trans_config.source.number_of_retries = 2 # name: source.transaction_executor_class # doc: a class that will execute transactions # converter: configman.converters.class_converter trans_config.source.transaction_executor_class = ( socorro.database.transaction_executor. TransactionExecutorWithLimitedBackoff) # name: source.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.source.wait_log_interval = 5 # name: destination.storage1.temporary_file_system_storage_path # doc: a local filesystem path dumps can be temporarily written # for processing # converter: str trans_config.source.temporary_file_system_storage_path = \ config.temporaryFileSystemStoragePath # name: destination.storage1.dump_file_suffix # doc: the suffix used to identify a dump file (for use in temp files) # converter: str trans_config.source.dump_file_suffix = config.dumpFileSuffix #-------------------------------------------------------------------------- from socorro.app.generic_app import main from socorro.processor.processor_app import ProcessorApp main(ProcessorApp, [trans_config])
#-------------------------------------------------------------------------- def main(self): """this function is run by the main thread. It just starts the subordinate threads and then waits for them to complete.""" standard_job_thread = threading.Thread( name="standard_job_thread", target=self._standard_job_thread ) standard_job_thread.start() priority_job_thread = threading.Thread( name="priority_job_thread", target=self._priority_job_thread ) priority_job_thread.start() job_cleanup_thread = threading.Thread( name="job_cleanup_thread", target=self._job_cleanup_thread ) job_cleanup_thread.start() self.config.logger.debug("waiting to join.") self._responsive_join(job_cleanup_thread) self._responsive_join(priority_job_thread) self._responsive_join(standard_job_thread) if __name__ == '__main__': main(MonitorApp)
id_field='name', ) # Verify data was correctly inserted. es_connection.refresh() total_indexed = es_connection.count( '*', index='socorro', doc_type='supersearch_fields', )['count'] total_expected = len(all_fields) if total_expected != total_indexed: indexed_fields = es_connection.search( '*', index='socorro', doc_type='supersearch_fields', size=total_indexed, ) indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']] self.config.logger.error( 'The SuperSearch fields data was not correctly indexed, ' '%s fields are missing from the database. Missing fields: %s', total_expected - total_indexed, list(set(all_fields.keys()) - set(indexed_fields))) if __name__ == '__main__': generic_app.main(SetupSuperSearchApp)
def processor2012(config): from configman.dotdict import DotDict # the following section is a translation of old-style configuration into # the new style configuration. It lets a new style app run without leaving # the old configuration system. trans_config = DotDict() #-------------------------------------------------------------------------- # destination - trans_config.destination = DotDict() # name: destination.crashstorage_class # doc: the destination storage class # converter: configman.converters.class_converter import socorro.external.crashstorage_base trans_config.destination.crashstorage_class = \ socorro.external.crashstorage_base.PolyCrashStorage # name: destination.storage_classes # doc: a comma delimited list of storage classes # converter: configman.converters.class_list_converter import socorro.external.postgresql.crashstorage import socorro.external.hbase.crashstorage import socorro.external.elasticsearch.crashstorage trans_config.destination.storage_classes = ''' socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage, socorro.external.hbase.crashstorage.HBaseCrashStorage, socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage ''' #-------------------------------------------------------------------------- # storage0 - trans_config.destination.storage0 = DotDict() # name: destination.storage0.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.destination.storage0.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300, 300, 300] # name: destination.storage0.crashstorage_class # doc: None # converter: configman.converters.class_converter trans_config.destination.storage0.crashstorage_class = \ socorro.external.postgresql.crashstorage.PostgreSQLCrashStorage # name: destination.storage0.database_class # doc: the class responsible for connecting toPostgres # converter: configman.converters.class_converter import socorro.external.postgresql.connection_context trans_config.destination.storage0.database_class = \ socorro.external.postgresql.connection_context.ConnectionContext # name: destination.storage0.database_host # doc: the hostname of the database # converter: str trans_config.destination.storage0.database_host = config.databaseHost # name: destination.storage0.database_name # doc: the name of the database # converter: str trans_config.destination.storage0.database_name = config.databaseName # name: destination.storage0.database_password # doc: the user's database password # converter: str trans_config.destination.storage0.database_password = \ config.databasePassword # name: destination.storage0.database_port # doc: the port for the database # converter: int trans_config.destination.storage0.database_port = config.databasePort # name: destination.storage0.database_user # doc: the name of the user within the database # converter: str trans_config.destination.storage0.database_user = config.databaseUserName # name: destination.storage0.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter import socorro.database.transaction_executor trans_config.destination.storage0.transaction_executor_class = ( socorro.database.transaction_executor .TransactionExecutorWithLimitedBackoff ) # name: destination.storage0.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.destination.storage0.wait_log_interval = 5 #-------------------------------------------------------------------------- # storage1 - trans_config.destination.storage1 = DotDict() # name: destination.storage1.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.destination.storage1.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300, 300, 300] # name: destination.storage1.crashstorage_class # doc: None # converter: configman.converters.class_converter trans_config.destination.storage1.crashstorage_class = \ socorro.external.hbase.crashstorage.HBaseCrashStorage # name: destination.storage1.forbidden_keys # doc: a comma delimited list of keys banned from the processed crash in # HBase # converter: socorro.external.hbase.crashstorage.<lambda> trans_config.destination.storage1.forbidden_keys = \ ['email', 'url', 'user_id', 'exploitability'] # name: destination.storage1.hbase_host # doc: Host to HBase server # converter: str trans_config.destination.storage1.hbase_host = config.hbaseHost # name: destination.storage1.hbase_port # doc: Port to HBase server # converter: int trans_config.destination.storage1.hbase_port = config.hbasePort # name: destination.storage1.hbase_timeout # doc: timeout in milliseconds for an HBase connection # converter: int trans_config.destination.storage1.hbase_timeout = config.hbaseTimeout # name: destination.storage1.number_of_retries # doc: Max. number of retries when fetching from hbaseClient # converter: int trans_config.destination.storage1.number_of_retries = 2 # name: destination.storage1.transaction_executor_class # doc: a class that will execute transactions # converter: configman.converters.class_converter trans_config.destination.storage1.transaction_executor_class = ( socorro.database.transaction_executor .TransactionExecutorWithLimitedBackoff ) # name: destination.storage1.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.destination.storage1.wait_log_interval = 5 # name: destination.storage1.temporary_file_system_storage_path # doc: a local filesystem path dumps can be temporarily written # for processing # converter: str trans_config.destination.storage1.temporary_file_system_storage_path = \ config.temporaryFileSystemStoragePath # name: destination.storage1.dump_file_suffix # doc: the suffix used to identify a dump file (for use in temp files) # converter: str trans_config.destination.storage1.dump_file_suffix = \ config.dumpFileSuffix #-------------------------------------------------------------------------- # storage2 - trans_config.destination.storage2 = DotDict() # name: destination.storage2.crashstorage_class # doc: None # converter: configman.converters.class_converter trans_config.destination.storage2.crashstorage_class = \ socorro.external.elasticsearch.crashstorage.ElasticSearchCrashStorage # name: destination.storage2.elasticsearch_urls # doc: the urls for the elasticsearch instances (leave blank to disable) # converter: str trans_config.destination.storage2.elasticsearch_urls = \ config.elasticsearch_urls # name: destination.storage2.elasticsearch_index # doc: an index to insert crashes in elasticsearch (use datetime's strftime # format to have daily, weekly or monthly indexes) # converter: str trans_config.destination.storage2.elasticsearch_index = \ config.elasticsearch_index # name: destination.storage2.elasticsearch_doctype # doc: a type to insert crashes in elasticsearch # converter: str trans_config.destination.storage2.elasticsearch_doctype = \ config.elasticsearch_doctype # name: destination.storage2.elasticsearch_index_settings # doc: the mapping of crash reports to insert # converter: str trans_config.destination.storage2.elasticsearch_index_settings = \ config.elasticsearch_index_settings # name: destination.storage2.timeout # doc: how long to wait in seconds for confirmation of a submission # converter: int trans_config.destination.storage2.timeout = 2 # name: destination.storage2.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter import socorro.database.transaction_executor trans_config.destination.storage2.transaction_executor_class = \ socorro.database.transaction_executor.TransactionExecutor #-------------------------------------------------------------------------- # logging - trans_config.logging = DotDict() # name: logging.stderr_error_logging_level # doc: logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - # WARNING, 40 - ERROR, 50 - CRITICAL) # converter: int trans_config.logging.stderr_error_logging_level = \ config.stderrErrorLoggingLevel # name: logging.stderr_line_format_string # doc: python logging system format for logging to stderr # converter: str trans_config.logging.stderr_line_format_string = \ config.stderrLineFormatString # name: logging.syslog_error_logging_level # doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, # 40 - ERROR, 50 - CRITICAL) # converter: int trans_config.logging.syslog_error_logging_level = \ config.syslogErrorLoggingLevel # name: logging.syslog_facility_string # doc: syslog facility string ("user", "local0", etc) # converter: str trans_config.logging.syslog_facility_string = config.syslogFacilityString # name: logging.syslog_host # doc: syslog hostname # converter: str trans_config.logging.syslog_host = config.syslogHost # name: logging.syslog_line_format_string # doc: python logging system format for syslog entries # converter: str # NOTE: the old and new systems use very differet formats for # the logging template. In the entire history of the project # the template has not changed. Rather than spending the effort # to translate the format, this hard coding will do. Since this # chimera processor is transintional, this will go away in the # not too distant future. trans_config.logging.syslog_line_format_string = \ '{app_name} (pid {process}): ' \ '{asctime} {levelname} - {threadName} - ' \ '{message}' # name: logging.syslog_port # doc: syslog port # converter: int trans_config.logging.syslog_port = config.syslogPort #-------------------------------------------------------------------------- # new_crash_source - trans_config.new_crash_source = DotDict() # name: new_crash_source.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.new_crash_source.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300, 300, 300] # name: new_crash_source.batchJobLimit # doc: the number of jobs to pull in a time # converter: int trans_config.new_crash_source.batchJobLimit = config.batchJobLimit # name: new_crash_source.database_class # doc: the class of the database # converter: configman.converters.class_converter trans_config.new_crash_source.database_class = \ socorro.external.postgresql.connection_context.ConnectionContext # name: new_crash_source.database_host # doc: the hostname of the database # converter: str trans_config.new_crash_source.database_host = config.databaseHost # name: new_crash_source.database_name # doc: the name of the database # converter: str trans_config.new_crash_source.database_name = config.databaseName # name: new_crash_source.database_password # doc: the user's database password # converter: str trans_config.new_crash_source.database_password = config.databasePassword # name: new_crash_source.database_port # doc: the port for the database # converter: int trans_config.new_crash_source.database_port = config.databasePort # name: new_crash_source.database_user # doc: the name of the user within the database # converter: str trans_config.new_crash_source.database_user = config.databaseUserName # name: new_crash_source.new_crash_source_class # doc: an iterable that will stream crash_ids needing processing # converter: configman.converters.class_converter import socorro.processor.legacy_new_crash_source trans_config.new_crash_source.new_crash_source_class = \ socorro.processor.legacy_new_crash_source.LegacyNewCrashSource # name: new_crash_source.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter trans_config.new_crash_source.transaction_executor_class = ( socorro.database.transaction_executor .TransactionExecutorWithLimitedBackoff ) # name: new_crash_source.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.new_crash_source.wait_log_interval = 5 #-------------------------------------------------------------------------- # processor - trans_config.processor = DotDict() # name: processor.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.processor.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300, 300, 300] # name: processor.collect_addon # doc: boolean indictating if information about add-ons should be collected # converter: configman.converters.boolean_converter trans_config.processor.collect_addon = config.collectAddon # name: processor.collect_crash_process # doc: boolean indictating if information about process type should be # collected # converter: configman.converters.boolean_converter trans_config.processor.collect_crash_process = config.collectCrashProcess # name: processor.crashing_thread_frame_threshold # doc: the number of frames to keep in the raw dump for the crashing thread # converter: int trans_config.processor.crashing_thread_frame_threshold = \ config.crashingThreadFrameThreshold # name: processor.crashing_thread_tail_frame_threshold # doc: the number of frames to keep in the raw dump at the tail of the # frame # list # converter: int trans_config.processor.crashing_thread_tail_frame_threshold = \ config.crashingThreadTailFrameThreshold # name: processor.database_class # doc: the class of the database # converter: configman.converters.class_converter trans_config.processor.database_class = \ socorro.external.postgresql.connection_context.ConnectionContext # name: processor.database_host # doc: the hostname of the database # converter: str trans_config.processor.database_host = config.databaseHost # name: processor.database_name # doc: the name of the database # converter: str trans_config.processor.database_name = config.databaseName # name: processor.database_password # doc: the user's database password # converter: str trans_config.processor.database_password = config.databasePassword # name: processor.database_port # doc: the port for the database # converter: int trans_config.processor.database_port = config.databasePort # name: processor.database_user # doc: the name of the user within the database # converter: str trans_config.processor.database_user = config.databaseUserName # name: processor.known_flash_identifiers # doc: A subset of the known "debug identifiers" for flash versions, # associated to the version # converter: json.loads trans_config.processor.known_flash_identifiers = \ config.knownFlashIdentifiers # name: processor.minidump_stackwalk_pathname # doc: the full pathname of the extern program minidump_stackwalk (quote # path with embedded spaces) # converter: str trans_config.processor.minidump_stackwalk_pathname = \ config.minidump_stackwalkPathname # name: processor.processor_class # doc: the class that transforms raw crashes into processed crashes # converter: configman.converters.class_converter import socorro.processor.legacy_processor trans_config.processor.processor_class = \ socorro.processor.legacy_processor.LegacyCrashProcessor # name: processor.processor_symbols_pathname_list # doc: comma or space separated list of symbol files for minidump_stackwalk # (quote paths with embedded spaces) # converter: socorro.processor.legacy_processor.create_symbol_path_str trans_config.processor.processor_symbols_pathname_list = \ config.processorSymbolsPathnameList # name: processor.stackwalk_command_line # doc: the template for the command to invoke minidump_stackwalk # converter: str trans_config.processor.stackwalk_command_line = \ config.stackwalkCommandLine.replace( 'minidump_stackwalkPathname', 'minidump_stackwalk_pathname' ).replace( 'processorSymbolsPathnameList', 'processor_symbols_pathname_list' ) # name: exploitability_tool_command_line # doc: the template for the command to invoke the exploitability tool # converter: str trans_config.processor.exploitability_tool_command_line = \ config.exploitability_tool_command_line # name: exploitability_tool_pathname # doc: the full pathname of the extern program exploitability tool # (quote path with embedded spaces) # converter: str trans_config.processor.exploitability_tool_pathname = \ config.exploitability_tool_pathname # name: processor.symbol_cache_path # doc: the path where the symbol cache is found (quote path with embedded # spaces) # converter: str trans_config.processor.symbol_cache_path = config.symbolCachePath # name: processor.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter trans_config.processor.transaction_executor_class = ( socorro.database.transaction_executor .TransactionExecutorWithLimitedBackoff ) # name: processor.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.processor.wait_log_interval = 5 #-------------------------------------------------------------------------- # c_signature - trans_config.processor.c_signature = DotDict() # name: processor.c_signature.c_signature_tool_class # doc: the class that can generate a C signature # converter: configman.converters.class_converter import socorro.processor.signature_utilities trans_config.processor.c_signature.c_signature_tool_class = \ socorro.processor.signature_utilities.CSignatureTool # name: processor.c_signature.irrelevant_signature_re # doc: a regular expression matching frame signatures that should be # ignored when generating an overall signature # converter: str trans_config.processor.c_signature.irrelevant_signature_re = \ config.irrelevantSignatureRegEx # name: processor.c_signature.prefix_signature_re # doc: a regular expression matching frame signatures that should always be # coupled with the following frame signature when generating an # overall signature # converter: str trans_config.processor.c_signature.prefix_signature_re = \ config.prefixSignatureRegEx # name: processor.c_signature.signature_sentinels # doc: a list of frame signatures that should always be considered top of # the stack if present in the stack # converter: eval trans_config.processor.c_signature.signature_sentinels = \ config.signatureSentinels # name: processor.c_signature.signatures_with_line_numbers_re # doc: any signatures that match this list should be combined with their # associated source code line numbers # converter: str trans_config.processor.c_signature.signatures_with_line_numbers_re = \ config.signaturesWithLineNumbersRegEx #-------------------------------------------------------------------------- # java_signature - trans_config.processor.java_signature = DotDict() # name: processor.java_signature.java_signature_tool_class # doc: the class that can generate a Java signature # converter: configman.converters.class_converter trans_config.processor.java_signature.java_signature_tool_class = \ socorro.processor.signature_utilities.JavaSignatureTool #-------------------------------------------------------------------------- # producer_consumer - trans_config.producer_consumer = DotDict() # name: producer_consumer.idle_delay # doc: the delay in seconds if no job is found # converter: int trans_config.producer_consumer.idle_delay = \ config.processorLoopTime.seconds # name: producer_consumer.maximum_queue_size # doc: the maximum size of the internal queue # converter: int trans_config.producer_consumer.maximum_queue_size = \ config.numberOfThreads * 2 # name: producer_consumer.number_of_threads # doc: the number of threads # converter: int trans_config.producer_consumer.number_of_threads = config.numberOfThreads # name: producer_consumer.producer_consumer_class # doc: the class implements a threaded producer consumer queue # converter: configman.converters.class_converter import socorro.lib.threaded_task_manager trans_config.producer_consumer.producer_consumer_class = \ socorro.lib.threaded_task_manager.ThreadedTaskManager #-------------------------------------------------------------------------- # registrar - trans_config.registrar = DotDict() # name: registrar.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.registrar.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300, 300, 300] # name: registrar.check_in_frequency # doc: how often the processor is required to reregister (hh:mm:ss) # converter: configman.converters.timedelta_converter trans_config.registrar.check_in_frequency = \ config.processorCheckInFrequency # name: registrar.database # doc: the class of the registrar's database # converter: configman.converters.class_converter trans_config.registrar.database = \ socorro.external.postgresql.connection_context.ConnectionContext # name: registrar.database_host # doc: the hostname of the database # converter: str trans_config.registrar.database_host = config.databaseHost # name: registrar.database_name # doc: the name of the database # converter: str trans_config.registrar.database_name = config.databaseName # name: registrar.database_password # doc: the user's database password # converter: str trans_config.registrar.database_password = config.databasePassword # name: registrar.database_port # doc: the port for the database # converter: int trans_config.registrar.database_port = config.databasePort # name: registrar.database_user # doc: the name of the user within the database # converter: str trans_config.registrar.database_user = config.databaseUserName # name: registrar.processor_id # doc: the id number for the processor (must already exist) (0 for create # new Id, "auto" for autodetection, "host" for same host) # converter: str trans_config.registrar.processor_id = config.processorId # name: registrar.registrar_class # doc: the class that registers and tracks processors # converter: configman.converters.class_converter import socorro.processor.registration_client trans_config.registrar.registrar_class = \ socorro.processor.registration_client.ProcessorAppRegistrationClient # name: registrar.transaction_executor_class # doc: a class that will manage transactions # converter: configman.converters.class_converter trans_config.registrar.transaction_executor_class = ( socorro.database.transaction_executor .TransactionExecutorWithLimitedBackoff ) # name: registrar.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.registrar.wait_log_interval = 5 #-------------------------------------------------------------------------- # source - trans_config.source = DotDict() # name: source.backoff_delays # doc: delays in seconds between retries # converter: eval trans_config.source.backoff_delays = [10, 30, 60, 120, 300, 300, 300, 300, 300, 300] # name: source.crashstorage_class # doc: the source storage class # converter: configman.converters.class_converter trans_config.source.crashstorage_class = \ socorro.external.hbase.crashstorage.HBaseCrashStorage # name: source.forbidden_keys # doc: a comma delimited list of keys banned from the processed crash in # HBase # converter: socorro.external.hbase.crashstorage.<lambda> trans_config.source.forbidden_keys = \ ['email', 'url', 'user_id', 'exploitability'] # name: source.hbase_host # doc: Host to HBase server # converter: str trans_config.source.hbase_host = config.hbaseHost # name: source.hbase_port # doc: Port to HBase server # converter: int trans_config.source.hbase_port = config.hbasePort # name: source.hbase_timeout # doc: timeout in milliseconds for an HBase connection # converter: int trans_config.source.hbase_timeout = config.hbaseTimeout # name: source.number_of_retries # doc: Max. number of retries when fetching from hbaseClient # converter: int trans_config.source.number_of_retries = 2 # name: source.transaction_executor_class # doc: a class that will execute transactions # converter: configman.converters.class_converter trans_config.source.transaction_executor_class = ( socorro.database.transaction_executor .TransactionExecutorWithLimitedBackoff ) # name: source.wait_log_interval # doc: seconds between log during retries # converter: int trans_config.source.wait_log_interval = 5 # name: destination.storage1.temporary_file_system_storage_path # doc: a local filesystem path dumps can be temporarily written # for processing # converter: str trans_config.source.temporary_file_system_storage_path = \ config.temporaryFileSystemStoragePath # name: destination.storage1.dump_file_suffix # doc: the suffix used to identify a dump file (for use in temp files) # converter: str trans_config.source.dump_file_suffix = config.dumpFileSuffix #-------------------------------------------------------------------------- from socorro.app.generic_app import main from socorro.processor.processor_app import ProcessorApp main(ProcessorApp, [trans_config])
default=ElasticSearchCrashStorage, doc='The class to use to store crash reports in elasticsearch.') required_config.add_option('processed_crash_file', default='./testcrash/processed_crash.json', doc='The file containing the processed crash.') def main(self): storage = self.config.elasticsearch_storage_class(self.config) crash_file = open(self.config.processed_crash_file) processed_crash = json.load(crash_file) es_index = storage.get_index_for_crash(processed_crash) es_doctype = self.config.elasticsearch_doctype crash_id = processed_crash['uuid'] storage.save_processed(processed_crash) # Verify the crash has been inserted es = pyelasticsearch.ElasticSearch(self.config.elasticsearch_urls) crash = es.get(es_index, es_doctype, crash_id) assert crash['exists'] print 'Success - %s/%s/%s' % (es_index, es_doctype, crash_id) if __name__ == '__main__': generic_app.main(IntegrationTestElasticsearchStorageApp) print 'done'
class IntegrationTestAutomaticEmailsApp(generic_app.App): app_name = 'test_automatic_emails' app_version = '0.1' app_description = __doc__ required_config = Namespace() required_config.add_option( 'automatic_emails_class', default=AutomaticEmailsCronApp, doc='The class to use to send automatic emails.' ) required_config.add_option( 'tester_email_address', default='', doc='Send the automatic email to this address.' ) def main(self): emailer = self.config.automatic_emails_class(self.config, '') report = { 'email': self.config.tester_email_address, } print emailer.send_email(report) if __name__ == '__main__': generic_app.main(IntegrationTestAutomaticEmailsApp)
def connect(self): print "connecting" try: connection = pika.BlockingConnection( pika.ConnectionParameters( host=self.config.test_rabbitmq.rabbitmq_host, port=self.config.test_rabbitmq.rabbitmq_port, virtual_host=self.config.test_rabbitmq.rabbitmq_vhost, credentials=pika.credentials.PlainCredentials( self.config.test_rabbitmq.rabbitmq_user, self.config.test_rabbitmq.rabbitmq_password))) except: print "Failed to connect" raise self.connection = connection def purge_queue(self, queue): self.connect() channel = self.connection.channel() try: channel.queue_delete(queue=queue) except pika.exceptions.ChannelClosed, e: print "Could not find queue %s" % queue print "Deleted queue %s" % queue if __name__ == '__main__': main(TestRabbitMQApp)
import os from socorro.app.generic_app import main from collector.collector_app import CollectorApp from socorro.webapi.servers import ApacheModWSGI import collector.collector_app from configman import ConfigFileFutureProxy if os.path.isfile('/etc/socorro/collector.ini'): config_path = '/etc/socorro' else: config_path = ApacheModWSGI.get_socorro_config_path(__file__) # invoke the generic main function to create the configman app class and which # will then create the wsgi app object. main( CollectorApp, # the socorro app class config_path=config_path) application = collector.collector_app.application
self.config.logger.critical("something's gone horribly wrong", exc_info=True) self.quit = True finally: self.config.logger.info("priorityLoop done.") #-------------------------------------------------------------------------- def main(self): """this function is run by the main thread. It just starts the subordinate threads and then waits for them to complete.""" standard_job_thread = threading.Thread( name="standard_job_thread", target=self._standard_job_thread) standard_job_thread.start() priority_job_thread = threading.Thread( name="priority_job_thread", target=self._priority_job_thread) priority_job_thread.start() job_cleanup_thread = threading.Thread(name="job_cleanup_thread", target=self._job_cleanup_thread) job_cleanup_thread.start() self.config.logger.debug("waiting to join.") self._responsive_join(job_cleanup_thread) self._responsive_join(priority_job_thread) self._responsive_join(standard_job_thread) if __name__ == '__main__': main(MonitorApp)
import os from socorro.app.generic_app import main from socorro.middleware.middleware_app import MiddlewareApp from socorro.webapi.servers import ApacheModWSGI import socorro.middleware.middleware_app from configman import ( ConfigFileFutureProxy, environment ) if os.path.isfile('/etc/socorro/middleware.ini'): config_path = '/etc/socorro' else: config_path = ApacheModWSGI.get_socorro_config_path(__file__) # invoke the generic main function to create the configman app class and which # will then create the wsgi app object. main( MiddlewareApp, # the socorro app class config_path=config_path, values_source_list=[ ConfigFileFutureProxy, environment ] ) application = socorro.middleware.middleware_app.application
import os from socorro.app.generic_app import main from socorro.collector.collector_app import CollectorApp from socorro.webapi.servers import ApacheModWSGI import socorro.collector.collector_app from configman import ConfigFileFutureProxy if os.path.isfile('/etc/socorro/collector.ini'): config_path = '/etc/socorro' else: config_path = ApacheModWSGI.get_socorro_config_path(__file__) # invoke the generic main function to create the configman app class and which # will then create the wsgi app object. main( CollectorApp, # the socorro app class config_path=config_path ) application = socorro.collector.collector_app.application
from socorro.app import generic_app from socorro.cron.jobs.automatic_emails import AutomaticEmailsCronApp class IntegrationTestAutomaticEmailsApp(generic_app.App): app_name = 'test_automatic_emails' app_version = '0.1' app_description = __doc__ required_config = Namespace() required_config.add_option( 'automatic_emails_class', default=AutomaticEmailsCronApp, doc='The class to use to send automatic emails.') required_config.add_option('tester_email_address', default='', doc='Send the automatic email to this address.') def main(self): emailer = self.config.automatic_emails_class(self.config, '') report = { 'email': self.config.tester_email_address, } print emailer.send_email(report) if __name__ == '__main__': generic_app.main(IntegrationTestAutomaticEmailsApp)
'fields': es_fields, 'size': es_size, 'from': es_from } return self.es_storage.es.search( es_query, index=index, )['hits'] def get_index_for_date(self, date, index_format): """return the elasticsearch index for a date""" if not index_format: return None if '%' in index_format: return date.strftime(index_format) return index_format def index_crashes(self, es_index, crashes_to_index): self.es_storage.es.bulk_index( es_index, self.config.elasticsearch.elasticsearch_doctype, crashes_to_index, id_field='crash_id' ) if __name__ == '__main__': generic_app.main(ElasticsearchBackfillApp)
services_list = [] # populate the 'services_list' with the tuples that will define the # urls and services offered by dataservice. for impl_class_namespace in ( self.config.services.service_list.subordinate_namespace_names ): impl_class = self.config.services[impl_class_namespace].cls services_list.append(impl_class) if not services_list: raise RuntimeError( "No services configured." "See the [services].service_list setting in the config file" ) self.web_server = self.config.web_server.wsgi_server_class( self.config, services_list ) # for modwsgi the 'run' method returns the wsgi function that # will use. For other webservers, the 'run' method actually starts # the standalone web server. application = self.web_server.run() #============================================================================== if __name__ == '__main__': main(DataserviceApp)
db_version = db.version() with open('sql/roles.sql') as f: db.execute(f.read()) try: db.execute('CREATE EXTENSION citext') except ProgrammingError, e: if re.match('type "citext" already exists', e.pgerror.strip().split('ERROR: ')[1]): # already done, no need to rerun # pass ok pass if not self.no_schema: with open('sql/schema.sql') as f: db.execute( f.read(), ['type "citext" already exists'], ) db.execute('SELECT weekly_report_partitions()') else: db.execute('ALTER DATABASE %s OWNER TO breakpad_rw' % self.database_name) return 0 if __name__ == '__main__': # pragma: no cover sys.exit(main(SocorroDB))
needs a database connection because instead of doing an insert it just prints. However, it primes the connection by getting a cursor out first (otherwise it'd have to do it every time in a loo[). """ def cursor(self): pass class FTPScraperCronAppRunner(FTPScraperCronApp): # pragma: no cover required_config = Namespace() required_config.add_option( 'date', default=datetime.datetime.utcnow(), doc='Date to run for', from_string_converter='socorro.lib.datetimeutil.string_to_datetime' ) def __init__(self, config): self.config = config self.config.dry_run = True def main(self): assert self.config.dry_run self.run(_MockConnection(), self.config.date) if __name__ == '__main__': # pragma: no cover sys.exit(main(FTPScraperCronAppRunner))
import os from socorro.app.generic_app import main from socorro.middleware.middleware_app import MiddlewareApp from socorro.webapi.servers import WSGIServer import socorro.middleware.middleware_app from configman import (ConfigFileFutureProxy, environment) if os.path.isfile('/etc/socorro/middleware.ini'): config_path = '/etc/socorro' else: config_path = WSGIServer.get_socorro_config_path(__file__) # invoke the generic main function to create the configman app class and which # will then create the wsgi app object. main( MiddlewareApp, # the socorro app class config_path=config_path, values_source_list=[ConfigFileFutureProxy, environment]) application = socorro.middleware.middleware_app.application
#-------------------------------------------------------------------------- required_config.namespace('web_server') required_config.web_server.add_option( 'wsgi_server_class', doc='a class implementing a wsgi web server', default='socorro.webapi.servers.WSGIServer', from_string_converter=class_converter) #-------------------------------------------------------------------------- def main(self): # modwsgi requireds a module level name 'application' global application services_list = (self.config.collector.collector_class, ) self.config.crash_storage = self.config.storage.crashstorage_class( self.config.storage) self.config.throttler = self.config.throttler.throttler_class( self.config.throttler) self.web_server = self.config.web_server.wsgi_server_class( self.config, # needs the whole config not the local namespace services_list) # for modwsgi the 'run' method returns the wsgi function that the web # server will use. For other webservers, the 'run' method actually # starts the standalone web server. application = self.web_server.run() if __name__ == '__main__': main(CollectorApp)
params[args[i]] = args[i + 1] except IndexError: pass for key, value in params.iteritems(): if value.count(terms_sep) and not DONT_TERM_SPLIT.match(value): params[key] = value.split(terms_sep) return params @staticmethod def decode_special_characters(value): """Return a decoded string or list of strings. Because characters '/' and '+' are special in our URL scheme, we need to double-encode them in the client. This function is to decode them so our service can use them as expected. """ def convert(s): return s.replace("%2B", "+").replace("%2F", "/") if isinstance(value, (list, tuple)): return [convert(x) for x in value] assert isinstance(value, basestring) return convert(value) if __name__ == '__main__': main(MiddlewareApp)
# Get data from PostgreSQL. with pg_context() as pg_connection: cursor = pg_connection.cursor() sql = """ SELECT email, last_sending FROM emails WHERE last_sending >= %s """ cursor.execute(sql, (start_date, )) results = cursor.fetchall() if not results: self.config.logger.info('No data in emails table in postgres') return for row in results: # self.config.logger.info('putting %s into ES' % row[0]) if not row[0] or not row[0].strip(): continue es_connection.index( index=self.config.elasticsearch.elasticsearch_emails_index, doc_type='emails', doc={'last_sending': row[1]}, id=row[0]) if __name__ == '__main__': generic_app.main(MoveEmailsApp)
required_config.add_option( 'hbase_crash_storage_class', default=HappyBaseCrashStorage, doc='the class responsible for proving an hbase connection', from_string_converter=class_converter) required_config.add_option( 'command', default=help, doc='command to use', is_argument=True, from_string_converter=lambda s: class_converter(__name__ + '.' + s)) required_config.add_option( 'json', default=False, short_form='j', doc='json output instead of a pretty printed mapping', ) def main(self): self.storage = self.config.hbase_crash_storage_class(self.config) self.config.command(self).run() if __name__ == '__main__': try: generic_app.main(HBaseClientApp, config_manager_cls=HBaseClientConfigurationManager) except NotEnoughArguments as e: print >> sys.stderr, "ERROR: was expecting another argument: " + e.arg print >> sys.stderr, "Use the 'help' command to get help on commands."
self.config.services.services_controller.service_list ): services_list.append( # a tuple associating a URI with a service class ( uri, # a binding of the service class with the configuration # namespace for that service class class_with_partial_init( self.config.services[namespace] .service_implementation_class, self.config.services[namespace], self.config ) ) ) # initialize the wsgi server with the list of URI/service impl tuples self.web_server = self.config.web_server.wsgi_server_class( self.config, # needs the whole config not the local namespace services_list ) # for modwsgi the 'run' method returns the wsgi function that the web # server will use. For other webservers, the 'run' method actually # starts the standalone web server. application = self.web_server.run() if __name__ == '__main__': main(CollectorApp)
if not index: return None if '%' in index: index = day.strftime(index) return index def format_dates_in_crash(self, processed_crash): # HBase returns dates in a format that elasticsearch does not # understand. To keep our elasticsearch mapping simple, we # transform all dates to a recognized format. for attr in processed_crash: try: processed_crash[attr] = datetimeutil.date_to_string( datetimeutil.string_to_datetime(processed_crash[attr])) except (ValueError, TypeError, ISO8601Error): # the attribute is not a date pass return processed_crash def index_crashes(self, es_index, crashes_to_index): self.es_storage.es.bulk_index(es_index, self.config.elasticsearch_doctype, crashes_to_index, id_field='uuid') if __name__ == '__main__': generic_app.main(ElasticsearchBackfillApp)
cursor = pg_connection.cursor() sql = """ SELECT email, last_sending FROM emails WHERE last_sending >= %s """ cursor.execute(sql, (start_date,)) results = cursor.fetchall() if not results: self.config.logger.info('No data in emails table in postgres') return for row in results: # self.config.logger.info('putting %s into ES' % row[0]) if not row[0] or not row[0].strip(): continue es_connection.index( index=self.config.elasticsearch.elasticsearch_emails_index, doc_type='emails', doc={ 'last_sending': row[1] }, id=row[0] ) if __name__ == '__main__': generic_app.main(MoveEmailsApp)
credentials=pika.credentials.PlainCredentials( config.rabbitmq_user, config.rabbitmq_password)) ) except: logging.error("Failed to connect") raise self.connection = connection def main(self): self.connect() channel = self.connection.channel() channel.queue_declare(queue='socorro.reprocessing', durable=True) with open(self.config.reprocesscrashlist.crashes, 'r') as file: for uuid in file.read().splitlines(): channel.basic_publish( exchange='', routing_key="socorro.reprocessing", body=uuid, properties=pika.BasicProperties(delivery_mode=2) ) logging.debug('submitted %s' % uuid) self.connection.close() if __name__ == '__main__': main(ReprocessCrashlistApp)
#-------------------------------------------------------------------------- def main(self): # modwsgi requireds a module level name 'application' global application services_list = [] # populate the 'services_list' with the tuples that will define the # urls and services offered by dataservice. for impl_class_namespace in ( self.config.services.service_list.subordinate_namespace_names): impl_class = self.config.services[impl_class_namespace].cls services_list.append(impl_class) if not services_list: raise RuntimeError( "No services configured." "See the [services].service_list setting in the config file") self.web_server = self.config.web_server.wsgi_server_class( self.config, services_list) # for modwsgi the 'run' method returns the wsgi function that # will use. For other webservers, the 'run' method actually starts # the standalone web server. application = self.web_server.run() #============================================================================== if __name__ == '__main__': main(DataserviceApp)
channel.close() def connect(self): print "connecting" try: connection = pika.BlockingConnection(pika.ConnectionParameters( host=self.config.test_rabbitmq.rabbitmq_host, port=self.config.test_rabbitmq.rabbitmq_port, virtual_host=self.config.test_rabbitmq.rabbitmq_vhost, credentials=pika.credentials.PlainCredentials( self.config.test_rabbitmq.rabbitmq_user, self.config.test_rabbitmq.rabbitmq_password))) except: print "Failed to connect" raise self.connection = connection def purge_queue(self, queue): self.connect() channel = self.connection.channel() try: channel.queue_delete(queue=queue) except pika.exceptions.ChannelClosed, e: print "Could not find queue %s" % queue print "Deleted queue %s" % queue if __name__ == '__main__': main(TestRabbitMQApp)
host=config.host, port=config.port, virtual_host=config.virtual_host, credentials=pika.credentials.PlainCredentials( config.rabbitmq_user, config.rabbitmq_password))) except: logging.error("Failed to connect") raise self.connection = connection def main(self): self.connect() channel = self.connection.channel() channel.queue_declare(queue='socorro.reprocessing', durable=True) with open(self.config.reprocesscrashlist.crashes, 'r') as file: for uuid in file.read().splitlines(): channel.basic_publish( exchange='', routing_key="socorro.reprocessing", body=uuid, properties=pika.BasicProperties(delivery_mode=2)) logging.debug('submitted %s' % uuid) self.connection.close() if __name__ == '__main__': main(ReprocessCrashlistApp)
doc='The file containing the processed crash.' ) def main(self): storage = self.config.elasticsearch_storage_class(self.config) crash_file = open(self.config.processed_crash_file) processed_crash = json.load(crash_file) es_index = storage.get_index_for_crash(processed_crash) es_doctype = self.config.elasticsearch_doctype crash_id = processed_crash['uuid'] storage.save_processed(processed_crash) # Verify the crash has been inserted es = pyelasticsearch.ElasticSearch(self.config.elasticsearch_urls) crash = es.get( es_index, es_doctype, crash_id ) assert crash['exists'] print 'Success - %s/%s/%s' % (es_index, es_doctype, crash_id) if __name__ == '__main__': generic_app.main(IntegrationTestElasticsearchStorageApp) print 'done'
raise dsn = dsn_template % self.database_name with PostgreSQLManager(dsn, self.config.logger) as db: db_version = db.version() with open('sql/roles.sql') as f: db.execute(f.read()) if not self.no_schema: with open('sql/schema.sql') as f: db.execute(f.read(), ['schema "pgx_diag" already exists']) db.execute('SELECT weekly_report_partitions()') else: db.execute( 'ALTER DATABASE %s OWNER TO breakpad_rw' % self.database_name) if re.match(r'9\.0[.*]', db_version): with open(self.citext) as f: db.execute(f.read()) elif re.match(r'9\.1[.*]', db_version): db.execute('CREATE EXTENSION citext from unpackaged') return 0 if __name__ == '__main__': # pragma: no cover sys.exit(main(SocorroDB))
app_version = "0.1" app_description = __doc__ required_config = Namespace() required_config.add_option( 'hbase_crash_storage_class', default=HBaseCrashStorage, doc='the class responsible for proving an hbase connection', from_string_converter=class_converter ) required_config.add_option( 'command', default=help, doc='command to use', from_string_converter=lambda s: class_converter(__name__ + '.' + s) ) def main(self): self.storage = self.config.hbase_crash_storage_class(self.config) self.config.command(self).run() if __name__ == '__main__': try: generic_app.main(HBaseClientApp, config_manager_cls=HBaseClientConfigurationManager) except NotEnoughArguments as e: print >> sys.stderr, "ERROR: was expecting another argument: " + e.arg print >> sys.stderr, "Use the 'help' command to get help on commands."
for class_name, __ in class_list: class_config = self.config.crontabber['class-%s' % class_name] if not self._configtest_one(class_config): failed += 1 return not failed def _configtest_one(self, config): try: seconds = convert_frequency(config.frequency) time_ = config.time if time_: check_time(time_) # if less than 1 day, it doesn't make sense to specify hour if seconds < 60 * 60 * 24: raise FrequencyDefinitionError(config.time) return True except (JobNotFoundError, JobDescriptionError, FrequencyDefinitionError, TimeDefinitionError): exc_type, exc_value, exc_tb = sys.exc_info() print >>sys.stderr, "Error type:", exc_type print >>sys.stderr, "Error value:", exc_value print >>sys.stderr, ''.join(traceback.format_tb(exc_tb)) return False if __name__ == '__main__': # pragma: no cover sys.exit(main(CronTabber))
) # Verify data was correctly inserted. es_connection.refresh() total_indexed = es_connection.count( '*', index='socorro', doc_type='supersearch_fields', )['count'] total_expected = len(all_fields) if total_expected != total_indexed: indexed_fields = es_connection.search( '*', index='socorro', doc_type='supersearch_fields', size=total_indexed, ) indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']] self.config.logger.error( 'The SuperSearch fields data was not correctly indexed, ' '%s fields are missing from the database. Missing fields: %s', total_expected - total_indexed, list(set(all_fields.keys()) - set(indexed_fields)) ) if __name__ == '__main__': generic_app.main(SetupSuperSearchApp)
ekly-report-partitions See https://bugzilla.mozilla.org/show_bug.cgi?id=701253 """ class WeeklyReportsPartitions(App): app_name = 'weekly_reports_partitions' app_version = '0.1' app_description = __doc__ required_config = Namespace() required_config.add_option('transaction_executor_class', default=TransactionExecutorWithBackoff, #default=TransactionExecutor, doc='a class that will execute transactions') def run_query(self, connection): cursor = connection.cursor() cursor.execute('SELECT weekly_report_partitions()') def main(self): executor = self.config.transaction_executor_class(self.config) executor(self.run_query) # alternative use could be like this: # with self.config.transaction_executor_class(self.config) as connection: # self.run_query(connection) if __name__ == '__main__': # pragma: no cover main(WeeklyReportsPartitions)
from configman import Namespace class ExampleApp(App): app_name = 'example' app_version = '0.1' app_description = __doc__ # in this section, define any configuration requirements required_config = Namespace() required_config.add_option('name', default='Wilma', doc='a name to echo') required_config.add_option('time', default=datetime.datetime.now(), doc='the time of day') # implementing this constructor is only necessary when there is more # initialization to be done before main can be called #def __init__(self, config): #super(ExampleApp,self).__init__(config) def main(self): # this is where we'd implement the app # the configuraton is already setup as self.config print 'hello, %s. The time is: %s' % (self.config.name, self.config.time) if __name__ == '__main__': main(ExampleApp)
import os from socorro.app.generic_app import main from socorro.middleware.middleware_app import MiddlewareApp from socorro.webapi.servers import ApacheModWSGI import socorro.middleware.middleware_app if os.path.isfile('/etc/socorro/middleware.ini'): config_path = '/etc/socorro' else: config_path = ApacheModWSGI.get_socorro_config_path(__file__) # invoke the generic main function to create the configman app class and which # will then create the wsgi app object. main( MiddlewareApp, # the socorro app class config_path=config_path ) application = socorro.middleware.middleware_app.application