def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, **kw): doc_provider = self.get_document_provider() iterable = doc_provider.get_document_iterator(chunk_size) progress = ProgressManager( iterable, total=doc_provider.get_total_document_count(), reset=reset, chunk_size=chunk_size, logger=CouchProcessorProgressLogger(self.doc_types), ) if self.has_worker_pool: assert "iterable" not in kw, kw kw.update(iterable=iterable, max_retry=max_retry) with self.get_doc_migrator(filename, **kw) as migrator, progress: for doc in iterable: success = migrator.migrate(doc) if success: progress.add() else: try: iterable.retry(doc, max_retry) except TooManyRetries: progress.skip(doc) if not progress.skipped: self.write_migration_completed_state() return progress.total, progress.skipped
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, **kw): provider_kwargs = {} for kwarg in ('date_range', 'domain'): if kwarg in kw: provider_kwargs[kwarg] = kw.pop(kwarg) doc_provider = self.get_document_provider(**provider_kwargs) iterable = doc_provider.get_document_iterator(chunk_size) progress = ProgressManager( iterable, total=doc_provider.get_total_document_count(), reset=reset, chunk_size=chunk_size, logger=CouchProcessorProgressLogger(self.doc_types), ) if self.has_worker_pool: assert "iterable" not in kw, kw kw.update(iterable=iterable, max_retry=max_retry) with self.get_doc_migrator(filename, **kw) as migrator, progress: for doc in iterable: success = migrator.migrate(doc) if success: progress.add() else: progress.skip(doc) if not progress.skipped: self.write_migration_completed_state() return progress.total, progress.skipped
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100): doc_migrator = self.doc_migrator_class(self.slug, self.couchdb, filename) progress_logger = CouchProcessorProgressLogger(self.doc_types) document_provider = CouchDocumentProvider(self.iteration_key, self.doc_types) processor = DocumentProcessorController( document_provider, doc_migrator, reset, max_retry, chunk_size, progress_logger=progress_logger) return processor.run()
def _get_progress_logger(self): return CouchProcessorProgressLogger(self.doc_types)