def _get_resumable_chunked_iterator(dbaccessor, iteration_key, log_prefix):
total_docs = 0
for db in dbaccessor.sql_db_aliases:
total_docs += dbaccessor.get_approximate_doc_count(db)
iterable = resumable_sql_model_iterator(iteration_key,
dbaccessor,
chunk_size=CHUNK_SIZE,
transform=lambda x: x)
progress = ProgressManager(iterable,
total=total_docs,
reset=False,
chunk_size=CHUNK_SIZE,
logger=ProcessorProgressLogger(
log_prefix, sys.stderr))
with progress:
for chunk in chunked(iterable, CHUNK_SIZE):
yield chunk
progress.add(len(chunk))
def get_iterator(self, deleted_doc_ids=None, chunk_size=2):
reindex_accessor = SimulateDeleteReindexAccessor(self.reindex_accessor, deleted_doc_ids)
return resumable_sql_model_iterator(self.iteration_key, reindex_accessor, chunk_size)
