def verify_model(self, couch_ids, couch_model, sql_model, fields, log_file):
log_file.write("Verifying %ss\n" % sql_model.__name__)
count = 0
total_count = len(couch_ids)
for doc in iter_docs_with_retry(couch_model.get_db(), couch_ids):
try:
self.clean_doc(doc)
couch_obj = couch_model.wrap(doc)
except Exception:
log_file.write('ERROR: Could wrap %s %s\n' % (couch_model.__name__, doc.get('_id')))
continue
try:
sql_obj = sql_model.objects.get(couch_id=couch_obj._id)
except sql_model.DoesNotExist:
log_file.write('ERROR: Postgres record for %s with couch_id %s was not found\n' %
(sql_model.__name__, couch_obj._id))
continue
for field in fields:
if getattr(couch_obj, field) != getattr(sql_obj, field):
log_file.write('ERROR: Mismatch found for %s with couch_id %s\n' %
(sql_model.__name__, couch_obj._id))
break
count += 1
if (count % 10000) == 0:
print 'Processed %s / %s %s documents' % (count, total_count, couch_model.__name__)
def verify_model(self, couch_ids, couch_model, sql_model, fields,
log_file):
log_file.write("Verifying %ss\n" % sql_model.__name__)
count = 0
total_count = len(couch_ids)
for doc in iter_docs_with_retry(couch_model.get_db(), couch_ids):
try:
self.clean_doc(doc)
couch_obj = couch_model.wrap(doc)
except Exception:
log_file.write('ERROR: Could wrap %s %s\n' %
(couch_model.__name__, doc.get('_id')))
continue
try:
sql_obj = sql_model.objects.get(couch_id=couch_obj._id)
except sql_model.DoesNotExist:
log_file.write(
'ERROR: Postgres record for %s with couch_id %s was not found\n'
% (sql_model.__name__, couch_obj._id))
continue
for field in fields:
if getattr(couch_obj, field) != getattr(sql_obj, field):
log_file.write(
'ERROR: Mismatch found for %s with couch_id %s\n' %
(sql_model.__name__, couch_obj._id))
break
count += 1
if (count % 10000) == 0:
print 'Processed %s / %s %s documents' % (count, total_count,
couch_model.__name__)
def migrate(self, log_file):
count = 0
ids = self.get_couch_ids()
total_count = len(ids)
for doc in iter_docs_with_retry(SurveyKeyword.get_db(), ids):
try:
couch_obj = SurveyKeyword.wrap(doc)
couch_obj._migration_do_sync()
except Exception as e:
log_file.write('Could not sync SurveyKeyword %s: %s\n' % (doc['_id'], e))
count += 1
if (count % 1000) == 0:
print 'Processed %s / %s documents' % (count, total_count)
def migrate_model(self, get_couch_ids_method, couch_model, errors):
count = 0
ids = get_couch_ids_method()
total_count = len(ids)
for doc in iter_docs_with_retry(couch_model.get_db(), ids):
try:
self.clean_doc(doc)
couch_obj = couch_model.wrap(doc)
couch_obj._migration_do_sync()
except Exception as e:
errors.append('Could not sync %s %s: %s' % (couch_model.__name__, doc['_id'], e))
count += 1
if (count % 10000) == 0:
print 'Processed %s / %s %s documents' % (count, total_count, couch_model.__name__)
def dump_to_file(self):
try:
doc_count = XFormInstance.get_db().view(
'couchforms/by_xmlns',
key=DEVICE_LOG_XMLNS,
reduce=True,
).one()['value']
except TypeError:
doc_count = 0
device_log_ids = [row['id'] for row in XFormInstance.get_db().view(
'couchforms/by_xmlns',
key=DEVICE_LOG_XMLNS,
reduce=False,
)]
with open(self.filename, 'w') as f:
device_log_docs = iter_docs_with_retry(XFormInstance.get_db(), device_log_ids)
for doc in with_progress_bar(device_log_docs, length=doc_count):
f.write(json.dumps(doc) + '\n')
