def verify_model(self, couch_ids, couch_model, sql_model, fields, log_file): log_file.write("Verifying %ss\n" % sql_model.__name__) count = 0 total_count = len(couch_ids) for doc in iter_docs_with_retry(couch_model.get_db(), couch_ids): try: self.clean_doc(doc) couch_obj = couch_model.wrap(doc) except Exception: log_file.write('ERROR: Could wrap %s %s\n' % (couch_model.__name__, doc.get('_id'))) continue try: sql_obj = sql_model.objects.get(couch_id=couch_obj._id) except sql_model.DoesNotExist: log_file.write('ERROR: Postgres record for %s with couch_id %s was not found\n' % (sql_model.__name__, couch_obj._id)) continue for field in fields: if getattr(couch_obj, field) != getattr(sql_obj, field): log_file.write('ERROR: Mismatch found for %s with couch_id %s\n' % (sql_model.__name__, couch_obj._id)) break count += 1 if (count % 10000) == 0: print 'Processed %s / %s %s documents' % (count, total_count, couch_model.__name__)
def verify_model(self, couch_ids, couch_model, sql_model, fields, log_file): log_file.write("Verifying %ss\n" % sql_model.__name__) count = 0 total_count = len(couch_ids) for doc in iter_docs_with_retry(couch_model.get_db(), couch_ids): try: self.clean_doc(doc) couch_obj = couch_model.wrap(doc) except Exception: log_file.write('ERROR: Could wrap %s %s\n' % (couch_model.__name__, doc.get('_id'))) continue try: sql_obj = sql_model.objects.get(couch_id=couch_obj._id) except sql_model.DoesNotExist: log_file.write( 'ERROR: Postgres record for %s with couch_id %s was not found\n' % (sql_model.__name__, couch_obj._id)) continue for field in fields: if getattr(couch_obj, field) != getattr(sql_obj, field): log_file.write( 'ERROR: Mismatch found for %s with couch_id %s\n' % (sql_model.__name__, couch_obj._id)) break count += 1 if (count % 10000) == 0: print 'Processed %s / %s %s documents' % (count, total_count, couch_model.__name__)
def migrate(self, log_file): count = 0 ids = self.get_couch_ids() total_count = len(ids) for doc in iter_docs_with_retry(SurveyKeyword.get_db(), ids): try: couch_obj = SurveyKeyword.wrap(doc) couch_obj._migration_do_sync() except Exception as e: log_file.write('Could not sync SurveyKeyword %s: %s\n' % (doc['_id'], e)) count += 1 if (count % 1000) == 0: print 'Processed %s / %s documents' % (count, total_count)
def migrate_model(self, get_couch_ids_method, couch_model, errors): count = 0 ids = get_couch_ids_method() total_count = len(ids) for doc in iter_docs_with_retry(couch_model.get_db(), ids): try: self.clean_doc(doc) couch_obj = couch_model.wrap(doc) couch_obj._migration_do_sync() except Exception as e: errors.append('Could not sync %s %s: %s' % (couch_model.__name__, doc['_id'], e)) count += 1 if (count % 10000) == 0: print 'Processed %s / %s %s documents' % (count, total_count, couch_model.__name__)
def dump_to_file(self): try: doc_count = XFormInstance.get_db().view( 'couchforms/by_xmlns', key=DEVICE_LOG_XMLNS, reduce=True, ).one()['value'] except TypeError: doc_count = 0 device_log_ids = [row['id'] for row in XFormInstance.get_db().view( 'couchforms/by_xmlns', key=DEVICE_LOG_XMLNS, reduce=False, )] with open(self.filename, 'w') as f: device_log_docs = iter_docs_with_retry(XFormInstance.get_db(), device_log_ids) for doc in with_progress_bar(device_log_docs, length=doc_count): f.write(json.dumps(doc) + '\n')