def _get_form_ids(self, domain):
if should_use_sql_backend(domain):
problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, 'XFormError')
else:
problem_ids = get_form_ids_by_type(domain, 'XFormError')
return problem_ids
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'location_id' in doc[
'form'] and not doc['form']['location_id']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(
re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
def generate_all_form_ids_by_domain(start, end):
form_ids_by_domain = {
domain: get_form_ids_by_type(domain, 'XFormInstance', start, end)
for domain in iter_domains()
}
return {
domain: form_ids
for domain, form_ids in form_ids_by_domain.items() if form_ids
}
def generate_all_form_ids_by_domain(start, end):
form_ids_by_domain = {
domain: get_form_ids_by_type(domain, 'XFormInstance', start, end)
for domain in iter_domains()
}
return {
domain: form_ids
for domain, form_ids in form_ids_by_domain.items()
if form_ids
}
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {location_id: name for (location_id, name) in locations}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def handle(self, domain, indicator_config_id, start_date, end_date,
**options):
start_date = datetime.strptime(start_date, '%Y-%m-%d')
end_date = datetime.strptime(end_date, '%Y-%m-%d') + timedelta(days=1)
form_ids = get_form_ids_by_type(
domain=domain,
type_='XFormInstance',
start=start_date,
end=end_date,
)
rebuild_indicators_in_place(indicator_config_id, None, form_ids)
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(
domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {
location_id: name
for (location_id, name) in locations
}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod[
'PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(
re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(
re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(
re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
def get_diff_stats(self, domain):
db = get_diff_db(domain)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count,
num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain,
doc_type)))
form_ids_in_sql = len(
set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(
set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db())))
form_ids_in_sql = len(
set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch,
form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(
set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if self.strict:
# only care about these in strict mode
case_ids_in_couch = len(
set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db())))
case_ids_in_sql = len(
set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch,
case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def handle(self, *args, **options):
xform_db = XFormInstance.get_db()
for domain in get_indicator_domains():
namespaces = get_namespaces(domain)
indicators = []
for namespace in namespaces:
indicators.extend(FormIndicatorDefinition.get_all(namespace, domain))
form_ids = get_form_ids_by_type(domain, 'XFormInstance',
start=datetime.date(2013, 8, 1),
end=datetime.date(2013, 10, 15))
for doc in iter_docs(xform_db, form_ids):
xfrom_doc = XFormInstance.wrap(doc)
xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
def prepare_planning_db(domain):
db_filepath = get_planning_db_filepath(domain)
planning_db = PlanningDB.init(db_filepath)
xform_ids = get_form_ids_by_type(domain, 'XFormInstance')
xform_db = XFormInstance.get_db()
for i, xform in enumerate(iter_docs(xform_db, xform_ids)):
xform_id = xform['_id']
case_actions_by_case_id = collections.defaultdict(list)
try:
xml = _get_submission_xml(xform, xform_db)
except ResourceNotFound:
continue
new_form_json = _get_new_form_json(xml, xform_id)
case_updates = get_case_updates(new_form_json)
xform_copy = deepcopy(xform)
xform_copy['form'] = new_form_json
xformdoc = XFormInstance.wrap(xform_copy)
xformdoc_json = xformdoc.to_json()
planning_db.add_form(xform_id, xformdoc_json)
planning_db.add_diffs('form', xform_id,
json_diff(xform, xformdoc_json))
case_actions = [
(case_update.id, action.xform_id, action.to_json())
for case_update in case_updates
for action in case_update.get_case_actions(xformdoc)
]
stock_report_helpers, stock_case_actions = get_stock_actions(xformdoc)
case_actions.extend(stock_case_actions)
for case_id, xform_id, case_action in case_actions:
case_actions_by_case_id[case_id].append((xform_id, case_action))
for case_id, case_actions in case_actions_by_case_id.items():
planning_db.ensure_case(case_id)
planning_db.add_case_actions(case_id, case_actions)
planning_db.add_stock_report_helpers([
stock_report_helper.to_json()
for stock_report_helper in stock_report_helpers
])
return prepare_case_json(planning_db)
def prepare_planning_db(domain):
db_filepath = get_planning_db_filepath(domain)
planning_db = PlanningDB.init(db_filepath)
xform_ids = get_form_ids_by_type(domain, 'XFormInstance')
xform_db = XFormInstance.get_db()
for i, xform in enumerate(iter_docs(xform_db, xform_ids)):
xform_id = xform['_id']
case_actions_by_case_id = collections.defaultdict(list)
try:
xml = _get_submission_xml(xform, xform_db)
except ResourceNotFound:
continue
new_form_json = _get_new_form_json(xml, xform_id)
case_updates = get_case_updates(new_form_json)
xform_copy = deepcopy(xform)
xform_copy['form'] = new_form_json
xformdoc = XFormInstance.wrap(xform_copy)
xformdoc_json = xformdoc.to_json()
planning_db.add_form(xform_id, xformdoc_json)
planning_db.add_diffs('form', xform_id,
json_diff(xform, xformdoc_json))
case_actions = [
(case_update.id, action.xform_id, action.to_json())
for case_update in case_updates
for action in case_update.get_case_actions(xformdoc)
]
stock_report_helpers, stock_case_actions = get_stock_actions(xformdoc)
case_actions.extend(stock_case_actions)
for case_id, xform_id, case_action in case_actions:
case_actions_by_case_id[case_id].append((xform_id, case_action))
for case_id, case_actions in case_actions_by_case_id.items():
planning_db.ensure_case(case_id)
planning_db.add_case_actions(case_id, case_actions)
planning_db.add_stock_report_helpers([
stock_report_helper.to_json()
for stock_report_helper in stock_report_helpers
])
return prepare_case_json(planning_db)
def valiate_forms_and_cases(self, domain):
form_ids_in_couch = set(get_form_ids_by_type(domain, 'XFormInstance'))
form_ids_in_sqlite = set(self.planning_db.get_all_form_ids())
print 'Forms in Couch: {}'.format(len(form_ids_in_couch))
print 'Forms in Sqlite: {}'.format(len(form_ids_in_sqlite))
if form_ids_in_couch ^ form_ids_in_sqlite:
print 'In Couch only: {}'.format(
list(form_ids_in_couch - form_ids_in_sqlite))
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sqlite = set(self.planning_db.get_all_case_ids())
print 'Cases in Couch: {}'.format(len(case_ids_in_couch))
print 'Cases in Sqlite: {}'.format(len(case_ids_in_sqlite))
if case_ids_in_couch ^ case_ids_in_sqlite:
print 'In Couch only: {}'.format(
list(case_ids_in_couch - case_ids_in_sqlite))
print 'In Sqlite only: {}'.format(
list(case_ids_in_sqlite - case_ids_in_couch))
def valiate_forms_and_cases(self, domain):
form_ids_in_couch = set(get_form_ids_by_type(domain, 'XFormInstance'))
form_ids_in_sqlite = set(self.planning_db.get_all_form_ids())
print 'Forms in Couch: {}'.format(len(form_ids_in_couch))
print 'Forms in Sqlite: {}'.format(len(form_ids_in_sqlite))
if form_ids_in_couch ^ form_ids_in_sqlite:
print 'In Couch only: {}'.format(
list(form_ids_in_couch - form_ids_in_sqlite))
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sqlite = set(self.planning_db.get_all_case_ids())
print 'Cases in Couch: {}'.format(len(case_ids_in_couch))
print 'Cases in Sqlite: {}'.format(len(case_ids_in_sqlite))
if case_ids_in_couch ^ case_ids_in_sqlite:
print 'In Couch only: {}'.format(
list(case_ids_in_couch - case_ids_in_sqlite))
print 'In Sqlite only: {}'.format(
list(case_ids_in_sqlite - case_ids_in_couch))
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'location_id' in doc['form'] and not doc['form']['location_id']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def get_diff_stats(self, domain):
db = get_diff_db(domain)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type)))
form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
))
form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if self.strict:
# only care about these in strict mode
case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
))
case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def test_get_form_ids_by_type_xforminstance(self):
form_ids = get_form_ids_by_type(self.domain, 'XFormInstance')
self.assertEqual(len(form_ids), len(self.xforms))
self.assertEqual(set(form_ids), {form._id for form in self.xforms})
def test_get_form_ids_by_type_xforminstance(self):
form_ids = get_form_ids_by_type(self.domain, 'XFormInstance')
self.assertEqual(len(form_ids), len(self.xforms))
self.assertEqual(set(form_ids), {form._id for form in self.xforms})
def test_get_form_ids_by_type_xformerror(self):
form_ids = get_form_ids_by_type(self.domain, 'XFormError')
self.assertEqual(len(form_ids), len(self.xform_errors))
self.assertEqual(set(form_ids),
{form._id
for form in self.xform_errors})
def test_get_form_ids_by_type(self):
form_ids = get_form_ids_by_type(self.domain, 'XFormError')
self.assertEqual(form_ids, [form._id for form in self.xform_errors])
def assert_forms_in_range(start, end, forms):
form_ids = get_form_ids_by_type(self.domain,
'XFormInstance',
start=start,
end=end)
self.assertEqual(set(form_ids), {form._id for form in forms})
def get_form_ids_in_domain_by_type(domain, type_):
return get_form_ids_by_type(domain, type_)
def test_get_form_ids_by_type(self):
form_ids = get_form_ids_by_type(self.domain, 'XFormError')
self.assertEqual(form_ids, [form._id for form in self.xform_errors])
def assert_forms_in_range(start, end, forms):
form_ids = get_form_ids_by_type(self.domain, 'XFormInstance',
start=start, end=end)
self.assertEqual(set(form_ids), {form._id for form in forms})
def test_get_form_ids_by_type_xformerror(self):
form_ids = get_form_ids_by_type(self.domain, 'XFormError')
self.assertEqual(len(form_ids), len(self.xform_errors))
self.assertEqual(set(form_ids), {form._id for form in self.xform_errors})
def _get_form_ids(self, domain):
if should_use_sql_backend(domain):
problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, 'XFormError')
else:
problem_ids = get_form_ids_by_type(domain, 'XFormError')
return problem_ids
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def get_form_ids_in_domain_by_type(domain, type_):
return get_form_ids_by_type(domain, type_)
def iter_problem_forms(domain, since=None):
problem_ids = get_form_ids_by_type(domain, 'XFormError', start=since)
for doc in iter_docs(XFormError.get_db(), problem_ids):
yield XFormError.wrap(doc)
