def _get_form_ids(self, domain): if should_use_sql_backend(domain): problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type( domain, 'XFormError') else: problem_ids = get_form_ids_by_type(domain, 'XFormError') return problem_ids
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'location_id' in doc[ 'form'] and not doc['form']['location_id']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['location_id'] = case.location_id # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) location_id_node = xml_object.find( re.sub('}.*', '}location_id', xml_object.tag)) location_id_node.text = case.location_id updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder( instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml'] ['content_type']) instance._attachments = attachment_builder.to_json() print 'Updating XFormInstance:', doc['_id'] to_save.append(instance)
def generate_all_form_ids_by_domain(start, end): form_ids_by_domain = { domain: get_form_ids_by_type(domain, 'XFormInstance', start, end) for domain in iter_domains() } return { domain: form_ids for domain, form_ids in form_ids_by_domain.items() if form_ids }
def generate_all_form_ids_by_domain(start, end): form_ids_by_domain = { domain: get_form_ids_by_type(domain, 'XFormInstance', start, end) for domain in iter_domains() } return { domain: form_ids for domain, form_ids in form_ids_by_domain.items() if form_ids }
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format(domain, status)) db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0)) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status( key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format(domain, status)) db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0)) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status( key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name') locations_map = {location_id: name for (location_id, name) in locations} for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': print 'Updating XFormInstance:', doc['_id'] pps_name = locations_map[case.location_id] instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['PPS_name'] = pps_name for instance_prod in instance.form['products']: instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag)) pps_name_node.text = pps_name products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag)) for product_node in products_nodes: product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag)) product_pps_name_node.text = pps_name updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder(instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml']['content_type'] ) instance._attachments = attachment_builder.to_json() to_save.append(instance) except Exception: print 'Failed to save XFormInstance:', doc['_id'] if len(to_save) > 500: XFormInstance.get_db().bulk_save(to_save) to_save = [] if to_save: XFormInstance.get_db().bulk_save(to_save)
def handle(self, domain, indicator_config_id, start_date, end_date, **options): start_date = datetime.strptime(start_date, '%Y-%m-%d') end_date = datetime.strptime(end_date, '%Y-%m-%d') + timedelta(days=1) form_ids = get_form_ids_by_type( domain=domain, type_='XFormInstance', start=start_date, end=end_date, ) rebuild_indicators_in_place(indicator_config_id, None, form_ids)
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] locations = SQLLocation.objects.filter( domain='ipm-senegal').values_list('location_id', 'name') locations_map = { location_id: name for (location_id, name) in locations } for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': print 'Updating XFormInstance:', doc['_id'] pps_name = locations_map[case.location_id] instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['PPS_name'] = pps_name for instance_prod in instance.form['products']: instance_prod['PPS_name'] = instance_prod[ 'PPS_name'] or pps_name # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) pps_name_node = xml_object.find( re.sub('}.*', '}PPS_name', xml_object.tag)) pps_name_node.text = pps_name products_nodes = xml_object.findall( re.sub('}.*', '}products', xml_object.tag)) for product_node in products_nodes: product_pps_name_node = product_node.find( re.sub('}.*', '}PPS_name', xml_object.tag)) product_pps_name_node.text = pps_name updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder( instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml'] ['content_type']) instance._attachments = attachment_builder.to_json() to_save.append(instance)
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len( set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db()))) form_ids_in_sql = len( set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len( set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len( set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db()))) case_ids_in_sql = len( set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def handle(self, *args, **options): xform_db = XFormInstance.get_db() for domain in get_indicator_domains(): namespaces = get_namespaces(domain) indicators = [] for namespace in namespaces: indicators.extend(FormIndicatorDefinition.get_all(namespace, domain)) form_ids = get_form_ids_by_type(domain, 'XFormInstance', start=datetime.date(2013, 8, 1), end=datetime.date(2013, 10, 15)) for doc in iter_docs(xform_db, form_ids): xfrom_doc = XFormInstance.wrap(doc) xfrom_doc.update_indicators_in_bulk(indicators, logger=logging)
def prepare_planning_db(domain): db_filepath = get_planning_db_filepath(domain) planning_db = PlanningDB.init(db_filepath) xform_ids = get_form_ids_by_type(domain, 'XFormInstance') xform_db = XFormInstance.get_db() for i, xform in enumerate(iter_docs(xform_db, xform_ids)): xform_id = xform['_id'] case_actions_by_case_id = collections.defaultdict(list) try: xml = _get_submission_xml(xform, xform_db) except ResourceNotFound: continue new_form_json = _get_new_form_json(xml, xform_id) case_updates = get_case_updates(new_form_json) xform_copy = deepcopy(xform) xform_copy['form'] = new_form_json xformdoc = XFormInstance.wrap(xform_copy) xformdoc_json = xformdoc.to_json() planning_db.add_form(xform_id, xformdoc_json) planning_db.add_diffs('form', xform_id, json_diff(xform, xformdoc_json)) case_actions = [ (case_update.id, action.xform_id, action.to_json()) for case_update in case_updates for action in case_update.get_case_actions(xformdoc) ] stock_report_helpers, stock_case_actions = get_stock_actions(xformdoc) case_actions.extend(stock_case_actions) for case_id, xform_id, case_action in case_actions: case_actions_by_case_id[case_id].append((xform_id, case_action)) for case_id, case_actions in case_actions_by_case_id.items(): planning_db.ensure_case(case_id) planning_db.add_case_actions(case_id, case_actions) planning_db.add_stock_report_helpers([ stock_report_helper.to_json() for stock_report_helper in stock_report_helpers ]) return prepare_case_json(planning_db)
def prepare_planning_db(domain): db_filepath = get_planning_db_filepath(domain) planning_db = PlanningDB.init(db_filepath) xform_ids = get_form_ids_by_type(domain, 'XFormInstance') xform_db = XFormInstance.get_db() for i, xform in enumerate(iter_docs(xform_db, xform_ids)): xform_id = xform['_id'] case_actions_by_case_id = collections.defaultdict(list) try: xml = _get_submission_xml(xform, xform_db) except ResourceNotFound: continue new_form_json = _get_new_form_json(xml, xform_id) case_updates = get_case_updates(new_form_json) xform_copy = deepcopy(xform) xform_copy['form'] = new_form_json xformdoc = XFormInstance.wrap(xform_copy) xformdoc_json = xformdoc.to_json() planning_db.add_form(xform_id, xformdoc_json) planning_db.add_diffs('form', xform_id, json_diff(xform, xformdoc_json)) case_actions = [ (case_update.id, action.xform_id, action.to_json()) for case_update in case_updates for action in case_update.get_case_actions(xformdoc) ] stock_report_helpers, stock_case_actions = get_stock_actions(xformdoc) case_actions.extend(stock_case_actions) for case_id, xform_id, case_action in case_actions: case_actions_by_case_id[case_id].append((xform_id, case_action)) for case_id, case_actions in case_actions_by_case_id.items(): planning_db.ensure_case(case_id) planning_db.add_case_actions(case_id, case_actions) planning_db.add_stock_report_helpers([ stock_report_helper.to_json() for stock_report_helper in stock_report_helpers ]) return prepare_case_json(planning_db)
def valiate_forms_and_cases(self, domain): form_ids_in_couch = set(get_form_ids_by_type(domain, 'XFormInstance')) form_ids_in_sqlite = set(self.planning_db.get_all_form_ids()) print 'Forms in Couch: {}'.format(len(form_ids_in_couch)) print 'Forms in Sqlite: {}'.format(len(form_ids_in_sqlite)) if form_ids_in_couch ^ form_ids_in_sqlite: print 'In Couch only: {}'.format( list(form_ids_in_couch - form_ids_in_sqlite)) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sqlite = set(self.planning_db.get_all_case_ids()) print 'Cases in Couch: {}'.format(len(case_ids_in_couch)) print 'Cases in Sqlite: {}'.format(len(case_ids_in_sqlite)) if case_ids_in_couch ^ case_ids_in_sqlite: print 'In Couch only: {}'.format( list(case_ids_in_couch - case_ids_in_sqlite)) print 'In Sqlite only: {}'.format( list(case_ids_in_sqlite - case_ids_in_couch))
def valiate_forms_and_cases(self, domain): form_ids_in_couch = set(get_form_ids_by_type(domain, 'XFormInstance')) form_ids_in_sqlite = set(self.planning_db.get_all_form_ids()) print 'Forms in Couch: {}'.format(len(form_ids_in_couch)) print 'Forms in Sqlite: {}'.format(len(form_ids_in_sqlite)) if form_ids_in_couch ^ form_ids_in_sqlite: print 'In Couch only: {}'.format( list(form_ids_in_couch - form_ids_in_sqlite)) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sqlite = set(self.planning_db.get_all_case_ids()) print 'Cases in Couch: {}'.format(len(case_ids_in_couch)) print 'Cases in Sqlite: {}'.format(len(case_ids_in_sqlite)) if case_ids_in_couch ^ case_ids_in_sqlite: print 'In Couch only: {}'.format( list(case_ids_in_couch - case_ids_in_sqlite)) print 'In Sqlite only: {}'.format( list(case_ids_in_sqlite - case_ids_in_couch))
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'location_id' in doc['form'] and not doc['form']['location_id']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['location_id'] = case.location_id # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) location_id_node = xml_object.find(re.sub('}.*', '}location_id', xml_object.tag)) location_id_node.text = case.location_id updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder(instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml']['content_type'] ) instance._attachments = attachment_builder.to_json() print 'Updating XFormInstance:', doc['_id'] to_save.append(instance) except Exception: print 'Failed to save XFormInstance:', doc['_id'] if len(to_save) > 500: XFormInstance.get_db().bulk_save(to_save) to_save = [] if to_save: XFormInstance.get_db().bulk_save(to_save)
def get_diff_stats(self, domain): db = get_diff_db(domain) diff_stats = db.get_diff_stats() stats = {} def _update_stats(doc_type, couch_count, sql_count): diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) if diff_count or couch_count != sql_count: stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs) for doc_type in doc_types(): form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type))) form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))) _update_stats(doc_type, form_ids_in_couch, form_ids_in_sql) form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) )) form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))) _update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql) case_ids_in_couch = len(set(get_case_ids_in_domain(domain))) case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain))) _update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql) if self.strict: # only care about these in strict mode case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) )) case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))) _update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql) if diff_stats: for key in diff_stats.keys(): _update_stats(key, 0, 0) return stats
def test_get_form_ids_by_type_xforminstance(self): form_ids = get_form_ids_by_type(self.domain, 'XFormInstance') self.assertEqual(len(form_ids), len(self.xforms)) self.assertEqual(set(form_ids), {form._id for form in self.xforms})
def test_get_form_ids_by_type_xforminstance(self): form_ids = get_form_ids_by_type(self.domain, 'XFormInstance') self.assertEqual(len(form_ids), len(self.xforms)) self.assertEqual(set(form_ids), {form._id for form in self.xforms})
def test_get_form_ids_by_type_xformerror(self): form_ids = get_form_ids_by_type(self.domain, 'XFormError') self.assertEqual(len(form_ids), len(self.xform_errors)) self.assertEqual(set(form_ids), {form._id for form in self.xform_errors})
def test_get_form_ids_by_type(self): form_ids = get_form_ids_by_type(self.domain, 'XFormError') self.assertEqual(form_ids, [form._id for form in self.xform_errors])
def assert_forms_in_range(start, end, forms): form_ids = get_form_ids_by_type(self.domain, 'XFormInstance', start=start, end=end) self.assertEqual(set(form_ids), {form._id for form in forms})
def get_form_ids_in_domain_by_type(domain, type_): return get_form_ids_by_type(domain, type_)
def test_get_form_ids_by_type(self): form_ids = get_form_ids_by_type(self.domain, 'XFormError') self.assertEqual(form_ids, [form._id for form in self.xform_errors])
def assert_forms_in_range(start, end, forms): form_ids = get_form_ids_by_type(self.domain, 'XFormInstance', start=start, end=end) self.assertEqual(set(form_ids), {form._id for form in forms})
def test_get_form_ids_by_type_xformerror(self): form_ids = get_form_ids_by_type(self.domain, 'XFormError') self.assertEqual(len(form_ids), len(self.xform_errors)) self.assertEqual(set(form_ids), {form._id for form in self.xform_errors})
def _get_form_ids(self, domain): if should_use_sql_backend(domain): problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, 'XFormError') else: problem_ids = get_form_ids_by_type(domain, 'XFormError') return problem_ids
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format( domain, status)) db = open_state_db(domain, self.state_dir) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) if doc_type == "XFormInstance": form_ids_in_couch.update( get_doc_ids_in_domain_by_type(domain, "HQSubmission", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) ZERO = Counts(0, 0) if db.has_doc_counts(): doc_counts = db.get_doc_counts() couch_missing_cases = doc_counts.get("CommCareCase-couch", ZERO).missing else: doc_counts = None couch_missing_cases = 0 for doc_type in CASE_DOC_TYPES: if doc_counts is not None: counts = doc_counts.get(doc_type, ZERO) case_ids_in_couch = db.get_missing_doc_ids( doc_type) if counts.missing else set() case_ids_in_sql = counts elif doc_type == "CommCareCase": case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set( CaseAccessorSQL.get_case_ids_in_domain(domain)) elif doc_type == "CommCareCase-Deleted": case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) else: raise NotImplementedError(doc_type) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only, ) if doc_type == "CommCareCase" and couch_missing_cases: has_diffs = True print( shell_red("%s cases could not be loaded from Couch" % couch_missing_cases)) if not short: for case_id in db.get_missing_doc_ids( "CommCareCase-couch"): print(case_id) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status(key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only) if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs
def get_form_ids_in_domain_by_type(domain, type_): return get_form_ids_by_type(domain, type_)
def iter_problem_forms(domain, since=None): problem_ids = get_form_ids_by_type(domain, 'XFormError', start=since) for doc in iter_docs(XFormError.get_db(), problem_ids): yield XFormError.wrap(doc)