#-------------------------------------------------------------------------- def _action_after_iteration_completes(self): self.config.logger.info( 'the queuing iterator is exhausted - waiting to quit') self.task_manager.wait_for_empty_queue( 5, "waiting for the queue to drain before quitting") time.sleep(self.config.producer_consumer.number_of_threads * 2) #-------------------------------------------------------------------------- def _filter_disallowed_values(self, current_value): """in this base class there are no disallowed values coming from the iterators. Other users of these iterator may have some standards and can detect and reject them here""" return current_value is None #-------------------------------------------------------------------------- def _transform(self, crash_id): """this transform function only transfers raw data from the source to the destination without changing the data.""" if self.config.submitter.dry_run: print crash_id else: raw_crash = self.source.get_raw_crash(crash_id) dumps = self.source.get_raw_dumps_as_files(crash_id) self.destination.save_raw_crash_with_file_dumps( raw_crash, dumps, crash_id) if __name__ == '__main__': main(SubmitterApp)
"""this implementation is the framework on how a raw crash is converted into a processed crash. The 'crash_id' passed in is used as a key to fetch the raw crash from the 'source', the conversion funtion implemented by the 'processor_class' is applied, the processed crash is saved to the 'destination', and then 'finished_func' is called.""" try: raw_crash = self.source.get_raw_crash(crash_id) processed_crash = self.source.get_processed(crash_id) except CrashIDNotFound: self.processor.reject_raw_crash( crash_id, 'this crash cannot be found in raw crash storage') return except Exception, x: self.config.logger.warning('error loading crash %s', crash_id, exc_info=True) self.processor.reject_raw_crash(crash_id, 'error in loading: %s' % x) return if 'uuid' not in raw_crash: raw_crash.uuid = crash_id self.destination.save_raw_and_processed(raw_crash, None, processed_crash, crash_id) self.config.logger.info('saved - %s', crash_id) if __name__ == '__main__': main(CrashMoverApp)
except CrashIDNotFound: self.processor.reject_raw_crash( crash_id, 'this crash cannot be found in raw crash storage' ) return except Exception, x: self.config.logger.warning( 'error loading crash %s', crash_id, exc_info=True ) self.processor.reject_raw_crash( crash_id, 'error in loading: %s' % x ) return if 'uuid' not in raw_crash: raw_crash.uuid = crash_id self.destination.save_raw_and_processed( raw_crash, None, processed_crash, crash_id ) self.config.logger.info('saved - %s', crash_id) if __name__ == '__main__': main(CrashMoverApp)
processed_crash = self.source.get_unredacted_processed( crash_id ) except CrashIDNotFound: self.config.logger.warning( 'cannot find crash %s', crash_id, ) return except Exception: self.config.logger.warning( 'error loading crash %s', crash_id, exc_info=True ) return if 'uuid' not in raw_crash: raw_crash.uuid = crash_id self.destination.save_raw_and_processed( raw_crash, dumps, processed_crash, crash_id ) self.config.logger.info('saved - %s', crash_id) if __name__ == '__main__': main(CrashMigrationApp)
5, "waiting for the queue to drain before quitting" ) time.sleep(self.config.producer_consumer.number_of_threads * 2) #-------------------------------------------------------------------------- def _filter_disallowed_values(self, current_value): """in this base class there are no disallowed values coming from the iterators. Other users of these iterator may have some standards and can detect and reject them here""" return current_value is None #-------------------------------------------------------------------------- def _transform(self, crash_id): """this transform function only transfers raw data from the source to the destination without changing the data.""" if self.config.submitter.dry_run: print crash_id else: raw_crash = self.source.get_raw_crash(crash_id) dumps = self.source.get_raw_dumps_as_files(crash_id) self.destination.save_raw_crash_with_file_dumps( raw_crash, dumps, crash_id ) if __name__ == '__main__': main(SubmitterApp)
# this function will be called by the MainThread periodically # while the threaded_task_manager processes crashes. self.waiting_func = None self.processor = self.config.processor.processor_class( self.config.processor, self.quit_check ) #-------------------------------------------------------------------------- def close(self): """when the processor shutsdown, this function cleans up""" try: self.companion_process.close() except AttributeError: # there is either no companion or it doesn't have a close method # we can skip on pass try: self.processor.close() except AttributeError: # the processor implementation does not have a close method # we can blithely skip on pass if __name__ == '__main__': main(ProcessorApp)
def _transform(self, crash_id): try: raw_crash = self.source.get_raw_crash(crash_id) if self.config.no_dumps: dumps = {} else: dumps = self.source.get_raw_dumps(crash_id) processed_crash = self.source.get_unredacted_processed(crash_id) except CrashIDNotFound: self.config.logger.warning( 'cannot find crash %s', crash_id, ) return except Exception: self.config.logger.warning('error loading crash %s', crash_id, exc_info=True) return if 'uuid' not in raw_crash: raw_crash.uuid = crash_id self.destination.save_raw_and_processed(raw_crash, dumps, processed_crash, crash_id) self.config.logger.info('saved - %s', crash_id) if __name__ == '__main__': main(CrashMigrationApp)
self.companion_process = None self.config.processor_name = self.app_instance_name # this function will be called by the MainThread periodically # while the threaded_task_manager processes crashes. self.waiting_func = None self.processor = self.config.processor.processor_class( self.config.processor, self.quit_check) #-------------------------------------------------------------------------- def close(self): """when the processor shutsdown, this function cleans up""" try: self.companion_process.close() except AttributeError: # there is either no companion or it doesn't have a close method # we can skip on pass try: self.processor.close() except AttributeError: # the processor implementation does not have a close method # we can blithely skip on pass if __name__ == '__main__': main(ProcessorApp)