Exemplo n.º 1
0
    #--------------------------------------------------------------------------
    def _action_after_iteration_completes(self):
        self.config.logger.info(
            'the queuing iterator is exhausted - waiting to quit')
        self.task_manager.wait_for_empty_queue(
            5, "waiting for the queue to drain before quitting")
        time.sleep(self.config.producer_consumer.number_of_threads * 2)

    #--------------------------------------------------------------------------
    def _filter_disallowed_values(self, current_value):
        """in this base class there are no disallowed values coming from the
        iterators.  Other users of these iterator may have some standards and
        can detect and reject them here"""
        return current_value is None

    #--------------------------------------------------------------------------
    def _transform(self, crash_id):
        """this transform function only transfers raw data from the
        source to the destination without changing the data."""
        if self.config.submitter.dry_run:
            print crash_id
        else:
            raw_crash = self.source.get_raw_crash(crash_id)
            dumps = self.source.get_raw_dumps_as_files(crash_id)
            self.destination.save_raw_crash_with_file_dumps(
                raw_crash, dumps, crash_id)


if __name__ == '__main__':
    main(SubmitterApp)
Exemplo n.º 2
0
        """this implementation is the framework on how a raw crash is
        converted into a processed crash.  The 'crash_id' passed in is used as
        a key to fetch the raw crash from the 'source', the conversion funtion
        implemented by the 'processor_class' is applied, the
        processed crash is saved to the 'destination', and then 'finished_func'
        is called."""
        try:
            raw_crash = self.source.get_raw_crash(crash_id)
            processed_crash = self.source.get_processed(crash_id)
        except CrashIDNotFound:
            self.processor.reject_raw_crash(
                crash_id, 'this crash cannot be found in raw crash storage')
            return
        except Exception, x:
            self.config.logger.warning('error loading crash %s',
                                       crash_id,
                                       exc_info=True)
            self.processor.reject_raw_crash(crash_id,
                                            'error in loading: %s' % x)
            return

        if 'uuid' not in raw_crash:
            raw_crash.uuid = crash_id
        self.destination.save_raw_and_processed(raw_crash, None,
                                                processed_crash, crash_id)
        self.config.logger.info('saved - %s', crash_id)


if __name__ == '__main__':
    main(CrashMoverApp)
Exemplo n.º 3
0
        except CrashIDNotFound:
            self.processor.reject_raw_crash(
                crash_id,
                'this crash cannot be found in raw crash storage'
            )
            return
        except Exception, x:
            self.config.logger.warning(
                'error loading crash %s',
                crash_id,
                exc_info=True
            )
            self.processor.reject_raw_crash(
                crash_id,
                'error in loading: %s' % x
            )
            return

        if 'uuid' not in raw_crash:
            raw_crash.uuid = crash_id
        self.destination.save_raw_and_processed(
            raw_crash,
            None,
            processed_crash,
            crash_id
        )
        self.config.logger.info('saved - %s', crash_id)

if __name__ == '__main__':
    main(CrashMoverApp)
Exemplo n.º 4
0
            processed_crash = self.source.get_unredacted_processed(
                crash_id
            )
        except CrashIDNotFound:
            self.config.logger.warning(
                'cannot find crash %s',
                crash_id,
            )
            return
        except Exception:
            self.config.logger.warning(
                'error loading crash %s',
                crash_id,
                exc_info=True
            )
            return

        if 'uuid' not in raw_crash:
            raw_crash.uuid = crash_id
        self.destination.save_raw_and_processed(
            raw_crash,
            dumps,
            processed_crash,
            crash_id
        )
        self.config.logger.info('saved - %s', crash_id)


if __name__ == '__main__':
    main(CrashMigrationApp)
Exemplo n.º 5
0
            5,
            "waiting for the queue to drain before quitting"
        )
        time.sleep(self.config.producer_consumer.number_of_threads * 2)

    #--------------------------------------------------------------------------
    def _filter_disallowed_values(self, current_value):
        """in this base class there are no disallowed values coming from the
        iterators.  Other users of these iterator may have some standards and
        can detect and reject them here"""
        return current_value is None

    #--------------------------------------------------------------------------
    def _transform(self, crash_id):
        """this transform function only transfers raw data from the
        source to the destination without changing the data."""
        if self.config.submitter.dry_run:
            print crash_id
        else:
            raw_crash = self.source.get_raw_crash(crash_id)
            dumps = self.source.get_raw_dumps_as_files(crash_id)
            self.destination.save_raw_crash_with_file_dumps(
                raw_crash,
                dumps,
                crash_id
            )


if __name__ == '__main__':
    main(SubmitterApp)
Exemplo n.º 6
0
        # this function will be called by the MainThread periodically
        # while the threaded_task_manager processes crashes.
        self.waiting_func = None

        self.processor = self.config.processor.processor_class(
            self.config.processor,
            self.quit_check
        )

    #--------------------------------------------------------------------------
    def close(self):
        """when  the processor shutsdown, this function cleans up"""
        try:
            self.companion_process.close()
        except AttributeError:
            # there is either no companion or it doesn't have a close method
            # we can skip on
            pass
        try:
            self.processor.close()
        except AttributeError:
            # the processor implementation does not have a close method
            # we can blithely skip on
            pass


if __name__ == '__main__':
    main(ProcessorApp)

Exemplo n.º 7
0
    def _transform(self, crash_id):
        try:
            raw_crash = self.source.get_raw_crash(crash_id)
            if self.config.no_dumps:
                dumps = {}
            else:
                dumps = self.source.get_raw_dumps(crash_id)
            processed_crash = self.source.get_unredacted_processed(crash_id)
        except CrashIDNotFound:
            self.config.logger.warning(
                'cannot find crash %s',
                crash_id,
            )
            return
        except Exception:
            self.config.logger.warning('error loading crash %s',
                                       crash_id,
                                       exc_info=True)
            return

        if 'uuid' not in raw_crash:
            raw_crash.uuid = crash_id
        self.destination.save_raw_and_processed(raw_crash, dumps,
                                                processed_crash, crash_id)
        self.config.logger.info('saved - %s', crash_id)


if __name__ == '__main__':
    main(CrashMigrationApp)
Exemplo n.º 8
0
            self.companion_process = None

        self.config.processor_name = self.app_instance_name

        # this function will be called by the MainThread periodically
        # while the threaded_task_manager processes crashes.
        self.waiting_func = None

        self.processor = self.config.processor.processor_class(
            self.config.processor, self.quit_check)

    #--------------------------------------------------------------------------
    def close(self):
        """when  the processor shutsdown, this function cleans up"""
        try:
            self.companion_process.close()
        except AttributeError:
            # there is either no companion or it doesn't have a close method
            # we can skip on
            pass
        try:
            self.processor.close()
        except AttributeError:
            # the processor implementation does not have a close method
            # we can blithely skip on
            pass


if __name__ == '__main__':
    main(ProcessorApp)