def _action_after_iteration_completes(self):
self.config.logger.info(
'the queuing iterator is exhausted - waiting to quit')
self.task_manager.wait_for_empty_queue(
5, "waiting for the queue to drain before quitting")
time.sleep(self.config.producer_consumer.number_of_threads * 2)
def _filter_disallowed_values(self, current_value):
"""In this base class there are no disallowed values coming from the
iterators. Other users of these iterator may have some standards and
can detect and reject them here
"""
return current_value is None
def _transform(self, crash_id):
"""Transfers raw data from the source to the destination without changing the data
"""
if self.config.submitter.dry_run:
print crash_id
else:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
self.destination.save_raw_crash_with_file_dumps(
raw_crash, dumps, crash_id)
if __name__ == '__main__':
main(SubmitterApp)
self.config.processor_name = self.app_instance_name
# this function will be called by the MainThread periodically
# while the threaded_task_manager processes crashes.
self.waiting_func = None
self.processor = self.config.processor.processor_class(
self.config.processor,
self.quit_check
)
def close(self):
"""when the processor shutsdown, this function cleans up"""
try:
self.companion_process.close()
except AttributeError:
# there is either no companion or it doesn't have a close method
# we can skip on
pass
try:
self.processor.close()
except AttributeError:
# the processor implementation does not have a close method
# we can blithely skip on
pass
if __name__ == '__main__':
main(ProcessorApp)
else:
self.companion_process = None
self.config.processor_name = self.app_instance_name
# this function will be called by the MainThread periodically
# while the threaded_task_manager processes crashes.
self.waiting_func = None
self.processor = self.config.processor.processor_class(
self.config.processor, self.quit_check)
def close(self):
"""when the processor shutsdown, this function cleans up"""
try:
self.companion_process.close()
except AttributeError:
# there is either no companion or it doesn't have a close method
# we can skip on
pass
try:
self.processor.close()
except AttributeError:
# the processor implementation does not have a close method
# we can blithely skip on
pass
if __name__ == '__main__':
main(ProcessorApp)
#! /usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""this app will move crashes from one storage location to another"""
from configman import Namespace
from socorro.app.fetch_transform_save_app import FetchTransformSaveApp, main
#==============================================================================
class CrashMoverApp(FetchTransformSaveApp):
app_name = 'crashmover'
app_version = '2.0'
app_description = __doc__
required_config = Namespace()
if __name__ == '__main__':
main(CrashMoverApp)
for crash_id in self.new_crash_source.new_crashes():
yield crash_id
# --------------------------------------------------------------------------
def _limited_iterator(self):
i = 0
while True:
for crash_id in self.new_crash_source.new_crashes():
if i == int(self.config.submitter.number_of_submissions):
break
i += 1
yield crash_id
if i == int(self.config.submitter.number_of_submissions):
break
# --------------------------------------------------------------------------
def _setup_source_and_destination(self):
"""instantiate the classes that implement the source and destination
crash storage systems."""
super(SubmitterApp, self)._setup_source_and_destination()
if self.config.new_crash_source.new_crash_source_class:
self.new_crash_source = self.config.new_crash_source.new_crash_source_class(
self.config.new_crash_source, "submitter"
)
else:
self.new_crash_source = self.source
if __name__ == "__main__":
main(SubmitterApp)
processed_crash = self.source.get_unredacted_processed(
crash_id
)
except CrashIDNotFound:
self.config.logger.warning(
'cannot find crash %s',
crash_id,
)
return
except Exception:
self.config.logger.warning(
'error loading crash %s',
crash_id,
exc_info=True
)
return
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
self.destination.save_raw_and_processed(
raw_crash,
None,
processed_crash,
crash_id
)
self.config.logger.info('saved - %s', crash_id)
if __name__ == '__main__':
main(CrashMigrationApp)
def get_all_crash_ids(self):
connection = self.source.database.connection()
sql = 'select uuid from raw_crashes;'
return execute_query_iter(connection, sql)
def _transform(self, crash_id):
try:
raw_crash = self.source.get_raw_crash(crash_id)
processed_crash = self.source.get_unredacted_processed(crash_id)
except CrashIDNotFound:
self.config.logger.warning(
'cannot find crash %s',
crash_id,
)
return
except Exception:
self.config.logger.warning('error loading crash %s',
crash_id,
exc_info=True)
return
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
self.destination.save_raw_and_processed(raw_crash, None,
processed_crash, crash_id)
self.config.logger.info('saved - %s', crash_id)
if __name__ == '__main__':
main(CrashMigrationApp)
