def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, **kw):
doc_provider = self.get_document_provider()
iterable = doc_provider.get_document_iterator(chunk_size)
progress = ProgressManager(
iterable,
total=doc_provider.get_total_document_count(),
reset=reset,
chunk_size=chunk_size,
logger=CouchProcessorProgressLogger(self.doc_types),
)
if self.has_worker_pool:
assert "iterable" not in kw, kw
kw.update(iterable=iterable, max_retry=max_retry)
with self.get_doc_migrator(filename, **kw) as migrator, progress:
for doc in iterable:
success = migrator.migrate(doc)
if success:
progress.add()
else:
try:
iterable.retry(doc, max_retry)
except TooManyRetries:
progress.skip(doc)
if not progress.skipped:
self.write_migration_completed_state()
return progress.total, progress.skipped
def migrate(self,
filename=None,
reset=False,
max_retry=2,
chunk_size=100,
**kw):
doc_provider = self.get_document_provider()
iterable = doc_provider.get_document_iterator(chunk_size)
progress = ProgressManager(
iterable,
total=doc_provider.get_total_document_count(),
reset=reset,
chunk_size=chunk_size,
logger=CouchProcessorProgressLogger(self.doc_types),
)
if self.has_worker_pool:
assert "iterable" not in kw, kw
kw.update(iterable=iterable, max_retry=max_retry)
with self.get_doc_migrator(filename, **kw) as migrator, progress:
for doc in iterable:
success = migrator.migrate(doc)
if success:
progress.add()
else:
try:
iterable.retry(doc, max_retry)
except TooManyRetries:
progress.skip(doc)
if not progress.skipped:
self.write_migration_completed_state()
return progress.total, progress.skipped
def migrate(self,
filename=None,
reset=False,
max_retry=2,
chunk_size=100,
**kw):
provider_kwargs = {}
for kwarg in ('date_range', 'domain'):
if kwarg in kw:
provider_kwargs[kwarg] = kw.pop(kwarg)
doc_provider = self.get_document_provider(**provider_kwargs)
iterable = doc_provider.get_document_iterator(chunk_size)
progress = ProgressManager(
iterable,
total=doc_provider.get_total_document_count(),
reset=reset,
chunk_size=chunk_size,
logger=CouchProcessorProgressLogger(self.doc_types),
)
if self.has_worker_pool:
assert "iterable" not in kw, kw
kw.update(iterable=iterable, max_retry=max_retry)
with self.get_doc_migrator(filename, **kw) as migrator, progress:
for doc in iterable:
success = migrator.migrate(doc)
if success:
progress.add()
else:
progress.skip(doc)
if not progress.skipped:
self.write_migration_completed_state()
return progress.total, progress.skipped