#--------------------------------------------------------------------------
def _action_after_iteration_completes(self):
self.config.logger.info(
'the queuing iterator is exhausted - waiting to quit')
self.task_manager.wait_for_empty_queue(
5, "waiting for the queue to drain before quitting")
time.sleep(self.config.producer_consumer.number_of_threads * 2)
#--------------------------------------------------------------------------
def _filter_disallowed_values(self, current_value):
"""in this base class there are no disallowed values coming from the
iterators. Other users of these iterator may have some standards and
can detect and reject them here"""
return current_value is None
#--------------------------------------------------------------------------
def _transform(self, crash_id):
"""this transform function only transfers raw data from the
source to the destination without changing the data."""
if self.config.submitter.dry_run:
print crash_id
else:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
self.destination.save_raw_crash_with_file_dumps(
raw_crash, dumps, crash_id)
if __name__ == '__main__':
main(SubmitterApp)
"""this implementation is the framework on how a raw crash is
converted into a processed crash. The 'crash_id' passed in is used as
a key to fetch the raw crash from the 'source', the conversion funtion
implemented by the 'processor_class' is applied, the
processed crash is saved to the 'destination', and then 'finished_func'
is called."""
try:
raw_crash = self.source.get_raw_crash(crash_id)
processed_crash = self.source.get_processed(crash_id)
except CrashIDNotFound:
self.processor.reject_raw_crash(
crash_id, 'this crash cannot be found in raw crash storage')
return
except Exception, x:
self.config.logger.warning('error loading crash %s',
crash_id,
exc_info=True)
self.processor.reject_raw_crash(crash_id,
'error in loading: %s' % x)
return
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
self.destination.save_raw_and_processed(raw_crash, None,
processed_crash, crash_id)
self.config.logger.info('saved - %s', crash_id)
if __name__ == '__main__':
main(CrashMoverApp)
except CrashIDNotFound:
self.processor.reject_raw_crash(
crash_id,
'this crash cannot be found in raw crash storage'
)
return
except Exception, x:
self.config.logger.warning(
'error loading crash %s',
crash_id,
exc_info=True
)
self.processor.reject_raw_crash(
crash_id,
'error in loading: %s' % x
)
return
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
self.destination.save_raw_and_processed(
raw_crash,
None,
processed_crash,
crash_id
)
self.config.logger.info('saved - %s', crash_id)
if __name__ == '__main__':
main(CrashMoverApp)
processed_crash = self.source.get_unredacted_processed(
crash_id
)
except CrashIDNotFound:
self.config.logger.warning(
'cannot find crash %s',
crash_id,
)
return
except Exception:
self.config.logger.warning(
'error loading crash %s',
crash_id,
exc_info=True
)
return
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
self.destination.save_raw_and_processed(
raw_crash,
dumps,
processed_crash,
crash_id
)
self.config.logger.info('saved - %s', crash_id)
if __name__ == '__main__':
main(CrashMigrationApp)
5,
"waiting for the queue to drain before quitting"
)
time.sleep(self.config.producer_consumer.number_of_threads * 2)
#--------------------------------------------------------------------------
def _filter_disallowed_values(self, current_value):
"""in this base class there are no disallowed values coming from the
iterators. Other users of these iterator may have some standards and
can detect and reject them here"""
return current_value is None
#--------------------------------------------------------------------------
def _transform(self, crash_id):
"""this transform function only transfers raw data from the
source to the destination without changing the data."""
if self.config.submitter.dry_run:
print crash_id
else:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
self.destination.save_raw_crash_with_file_dumps(
raw_crash,
dumps,
crash_id
)
if __name__ == '__main__':
main(SubmitterApp)
# this function will be called by the MainThread periodically
# while the threaded_task_manager processes crashes.
self.waiting_func = None
self.processor = self.config.processor.processor_class(
self.config.processor,
self.quit_check
)
#--------------------------------------------------------------------------
def close(self):
"""when the processor shutsdown, this function cleans up"""
try:
self.companion_process.close()
except AttributeError:
# there is either no companion or it doesn't have a close method
# we can skip on
pass
try:
self.processor.close()
except AttributeError:
# the processor implementation does not have a close method
# we can blithely skip on
pass
if __name__ == '__main__':
main(ProcessorApp)
def _transform(self, crash_id):
try:
raw_crash = self.source.get_raw_crash(crash_id)
if self.config.no_dumps:
dumps = {}
else:
dumps = self.source.get_raw_dumps(crash_id)
processed_crash = self.source.get_unredacted_processed(crash_id)
except CrashIDNotFound:
self.config.logger.warning(
'cannot find crash %s',
crash_id,
)
return
except Exception:
self.config.logger.warning('error loading crash %s',
crash_id,
exc_info=True)
return
if 'uuid' not in raw_crash:
raw_crash.uuid = crash_id
self.destination.save_raw_and_processed(raw_crash, dumps,
processed_crash, crash_id)
self.config.logger.info('saved - %s', crash_id)
if __name__ == '__main__':
main(CrashMigrationApp)
self.companion_process = None
self.config.processor_name = self.app_instance_name
# this function will be called by the MainThread periodically
# while the threaded_task_manager processes crashes.
self.waiting_func = None
self.processor = self.config.processor.processor_class(
self.config.processor, self.quit_check)
#--------------------------------------------------------------------------
def close(self):
"""when the processor shutsdown, this function cleans up"""
try:
self.companion_process.close()
except AttributeError:
# there is either no companion or it doesn't have a close method
# we can skip on
pass
try:
self.processor.close()
except AttributeError:
# the processor implementation does not have a close method
# we can blithely skip on
pass
if __name__ == '__main__':
main(ProcessorApp)