def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, **kw): doc_provider = self.get_document_provider() iterable = doc_provider.get_document_iterator(chunk_size) progress = ProgressManager( iterable, total=doc_provider.get_total_document_count(), reset=reset, chunk_size=chunk_size, logger=CouchProcessorProgressLogger(self.doc_types), ) if self.has_worker_pool: assert "iterable" not in kw, kw kw.update(iterable=iterable, max_retry=max_retry) with self.get_doc_migrator(filename, **kw) as migrator, progress: for doc in iterable: success = migrator.migrate(doc) if success: progress.add() else: try: iterable.retry(doc, max_retry) except TooManyRetries: progress.skip(doc) if not progress.skipped: self.write_migration_completed_state() return progress.total, progress.skipped
def _get_couch_form_chunks(run_config): db = XFormInstance.get_db() view_name = 'by_domain_doc_type_date/view' keys = [{ 'startkey': [ couch_domain, doc_type, json_format_datetime(run_config.start_date) ], 'endkey': [ couch_domain, doc_type, json_format_datetime(run_config.end_date) ], } for couch_domain in _get_matching_couch_domains(run_config) for doc_type in ['XFormArchived', 'XFormInstance']] if not keys: return def _get_length(): length = 0 for key in keys: result = db.view(view_name, reduce=True, **key).one() if result: length += result['value'] return length iteration_key = f'couch_forms-{run_config.iteration_key}' iterable = resumable_view_iterator(XFormInstance.get_db(), iteration_key, view_name, keys, chunk_size=CHUNK_SIZE, full_row=True) progress = ProgressManager(iterable, total=_get_length(), reset=False, chunk_size=CHUNK_SIZE, logger=ProcessorProgressLogger( '[Couch Forms] ', sys.stderr)) with progress: for chunk in chunked(iterable, CHUNK_SIZE): records = [] for row in chunk: form_id = row['id'] domain, doc_type, received_on = row['key'] received_on = iso_string_to_datetime(received_on) assert run_config.domain in (domain, ALL_DOMAINS) records.append( (form_id, doc_type, 'COUCH_XMLNS_NOT_SUPPORTED', received_on, domain)) yield records progress.add(len(chunk))
def _get_resumable_chunked_iterator(dbaccessor, iteration_key, log_prefix): total_docs = 0 for db in dbaccessor.sql_db_aliases: total_docs += dbaccessor.get_approximate_doc_count(db) iterable = resumable_sql_model_iterator(iteration_key, dbaccessor, chunk_size=CHUNK_SIZE, transform=lambda x: x) progress = ProgressManager(iterable, total=total_docs, reset=False, chunk_size=CHUNK_SIZE, logger=ProcessorProgressLogger( log_prefix, sys.stderr)) with progress: for chunk in chunked(iterable, CHUNK_SIZE): yield chunk progress.add(len(chunk))
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, **kw): doc_provider = self.get_document_provider() iterable = doc_provider.get_document_iterator(chunk_size) progress = ProgressManager( iterable, total=doc_provider.get_total_document_count(), reset=reset, chunk_size=chunk_size, logger=CouchProcessorProgressLogger(self.doc_types), ) if self.has_worker_pool: assert "iterable" not in kw, kw kw.update(iterable=iterable, max_retry=max_retry) with self.get_doc_migrator(filename, **kw) as migrator, progress: for doc in iterable: success = migrator.migrate(doc) if success: progress.add() else: try: iterable.retry(doc, max_retry) except TooManyRetries: progress.skip(doc) if not progress.skipped: self.write_migration_completed_state() return progress.total, progress.skipped
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, **kw): provider_kwargs = {} for kwarg in ('date_range', 'domain'): if kwarg in kw: provider_kwargs[kwarg] = kw.pop(kwarg) doc_provider = self.get_document_provider(**provider_kwargs) iterable = doc_provider.get_document_iterator(chunk_size) progress = ProgressManager( iterable, total=doc_provider.get_total_document_count(), reset=reset, chunk_size=chunk_size, logger=CouchProcessorProgressLogger(self.doc_types), ) if self.has_worker_pool: assert "iterable" not in kw, kw kw.update(iterable=iterable, max_retry=max_retry) with self.get_doc_migrator(filename, **kw) as migrator, progress: for doc in iterable: success = migrator.migrate(doc) if success: progress.add() else: progress.skip(doc) if not progress.skipped: self.write_migration_completed_state() return progress.total, progress.skipped