def handle(self, *args, **options): count = 1 filename = 'cases_with_mismatched_forms.log' dump_logs_to_file(filename) seen_cases = set() cases_rebuilt = set() with open(filename, 'r') as file: while True: line = file.readline() if not line: return print 'Processing record {}'.format(count) record = json.loads(line) exception = ExceptionRecord.wrap(record['exception']) case_id = record.get('case_id', None) if not case_id or case_id in seen_cases: count += 1 archive_exception(exception) continue try: seen_cases.add(case_id) if should_rebuild(case_id): cases_rebuilt.add(case_id) rebuild_case_from_forms(case_id) print 'rebuilt case {}'.format(case_id) archive_exception(exception) except Exception, e: logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e))) finally: count += 1
def test_rebuild_duplicate_index_identifiers(self): parent_case_id = uuid.uuid4().hex post_case_blocks([ CaseBlock(parent_case_id, create=True).as_xml() ], domain=REBUILD_TEST_DOMAIN) child_case_id = uuid.uuid4().hex post_case_blocks([ CaseBlock(child_case_id, create=True).as_xml() ], domain=REBUILD_TEST_DOMAIN) post_case_blocks([ CaseBlock(child_case_id, index={'mom': ('mother', parent_case_id)}).as_xml() ], domain=REBUILD_TEST_DOMAIN) child_case = CaseAccessors(REBUILD_TEST_DOMAIN).get_case(child_case_id) self.assertEqual(1, len(child_case.indices)) index = child_case.indices[0] index.id = None index.relationship = 'extension' index.save() child_case = CaseAccessors(REBUILD_TEST_DOMAIN).get_case(child_case_id) self.assertEqual(2, len(child_case.indices)) with self.assertRaises(AssertionError): rebuild_case_from_forms(REBUILD_TEST_DOMAIN, child_case_id, RebuildWithReason(reason='test'))
def handle(self, log_file, **options): to_archive = [] cases_affected = set() episode_case_ids = CaseAccessors(domain).get_case_ids_in_domain( "episode") if options.get('limit'): episode_case_ids = episode_case_ids[:options.get('limit')] for episode_case_id in with_progress_bar(episode_case_ids): nikshay_to_archive = [] dots_99_to_archvie = [] case_forms = FormProcessorInterface(domain).get_case_forms( episode_case_id) for form in case_forms: if form.user_id in ("system", "", None ) and form.metadata.username == "system": updates = get_case_updates(form) update_actions = [ update.get_update_action() for update in updates if update.id == episode_case_id ] for action in update_actions: if isinstance(action, CaseUpdateAction): if set(action.dynamic_properties.keys()) == { "nikshay_registered", "nikshay_error" }: nikshay_to_archive.append(form) cases_affected.add(episode_case_id) elif set(action.dynamic_properties.keys()) == { "dots_99_registered", "dots_99_error" }: cases_affected.add(episode_case_id) dots_99_to_archvie.append(form) # get_case_updates() returns the forms in the correct order, but sorting is probably a good idea in case # get_case_updates ever changes. nikshay_to_archive.sort(key=lambda f: f.received_on) dots_99_to_archvie.sort(key=lambda f: f.received_on) nikshay_to_archive = nikshay_to_archive[:-1] dots_99_to_archvie = dots_99_to_archvie[:-1] to_archive.extend(nikshay_to_archive) to_archive.extend(dots_99_to_archvie) print("Will archive {} forms".format(len(to_archive))) xform_archived.disconnect(rebuild_form_cases) with open(log_file, "w") as f: for form in with_progress_bar(to_archive): f.write(form.form_id + "\n") f.flush() if options['commit']: form.archive(user_id="remove_duplicate_forms_script") xform_archived.connect(rebuild_form_cases) print("Will rebuild {} cases".format(len(cases_affected))) for case_id in with_progress_bar(cases_affected): rebuild_case_from_forms( domain, case_id, UserRequestedRebuild(user_id="remove_duplicate_forms_script"))
def test_couch_reconcile_actions(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, form_extras={'received_on': now}) _post_util(case_id=case_id, p1='p1-1', p2='p2-1', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p2='p2-2', p3='p3-2', form_extras={'received_on': now + timedelta(seconds=2)}) case = CommCareCase.get(case_id) update_strategy = CouchCaseUpdateStrategy(case) original_actions = [deepcopy(a) for a in case.actions] original_form_ids = [id for id in case.xform_ids] self.assertEqual(4, len(original_actions)) self.assertEqual(3, len(original_form_ids)) self._assertListEqual(original_actions, case.actions) # test reordering case.actions = [ case.actions[3], case.actions[2], case.actions[1], case.actions[0] ] self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListEqual(original_actions, case.actions) # test duplication case.actions = case.actions * 3 self.assertEqual(12, len(case.actions)) self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListEqual(original_actions, case.actions) # test duplication, even when dates are off case.actions = original_actions + [deepcopy(case.actions[2])] case.actions[-1].server_date = case.actions[ -1].server_date + timedelta(seconds=1) self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListEqual(original_actions, case.actions) # test duplication with different properties is actually # treated differently case.actions = original_actions + [deepcopy(case.actions[2])] case.actions[-1].updated_unknown_properties['new'] = 'mismatch' self.assertEqual(5, len(case.actions)) self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListNotEqual(original_actions, case.actions) # test clean slate rebuild case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test')) self._assertListEqual(original_actions, primary_actions(case)) self._assertListEqual(original_form_ids, case.xform_ids)
def test_case_actions(self): # make sure that when a case is rebuilt (using rebuild_case) # stock transactions show up as well form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) case_id = self.case.case_id rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test')) case = CaseAccessors(self.domain).get_case(self.case.case_id) self.assertEqual(case.xform_ids[1:], [form_id]) self.assertTrue(form_id in [action.form_id for action in case.actions])
def rebuild_form_cases(sender, xform, *args, **kwargs): from casexml.apps.case.xform import get_case_ids_from_form from casexml.apps.case.cleanup import rebuild_case_from_forms domain = xform.domain case_ids = get_case_ids_from_form(xform) detail = FormArchiveRebuild(xmlns=xform.xmlns, form_id=xform.form_id, archived=xform.is_archived) for case_id in case_ids: rebuild_case_from_forms(domain, case_id, detail)
def test_case_actions(self): # make sure that when a case is rebuilt (using rebuild_case) # stock transactions show up as well form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) case_id = self.case.case_id rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test')) case = CaseAccessors(self.domain).get_case(self.case.case_id) self.assertEqual(case.xform_ids[1:], [form_id]) self.assertTrue(form_id in [action.form_id for action in case.actions])
def rebuild_form_cases(sender, xform, *args, **kwargs): from casexml.apps.case.xform import get_case_ids_from_form from casexml.apps.case.cleanup import rebuild_case_from_forms case_ids = get_case_ids_from_form(xform) transactions = StockTransaction.objects.filter(report__form_id=xform.form_id) stock_case_ids = transactions.values_list('case_id', flat=True).distinct() case_ids.update(stock_case_ids) for case_id in case_ids: rebuild_case_from_forms(case_id)
def rebuild_form_cases(sender, xform, *args, **kwargs): from casexml.apps.case.xform import get_case_ids_from_form from casexml.apps.case.cleanup import rebuild_case_from_forms from corehq.form_processor.parsers.ledgers.form import get_case_ids_from_stock_transactions domain = xform.domain case_ids = get_case_ids_from_form(xform) | get_case_ids_from_stock_transactions(xform) detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived) for case_id in case_ids: rebuild_case_from_forms(domain, case_id, detail)
def rebuild_form_cases(sender, xform, *args, **kwargs): from casexml.apps.case.xform import get_case_ids_from_form from casexml.apps.case.cleanup import rebuild_case_from_forms from corehq.form_processor.parsers.ledgers.form import get_case_ids_from_stock_transactions domain = xform.domain case_ids = get_case_ids_from_form(xform) | get_case_ids_from_stock_transactions(xform) detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived) for case_id in case_ids: rebuild_case_from_forms(domain, case_id, detail)
def test_case_actions(self): """ make sure that when a case is rebuilt (using rebuild_case) stock transactions show up as well """ form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE) case_id = self.case.case_id rebuild_case_from_forms(case_id) case = CommCareCase.get(case_id) self.assertEqual(case.xform_ids, [form_id]) self.assertEqual(case.actions[0].xform_id, form_id)
def _rebuild_cases(self): user = CouchUser.get_by_user_id(self.user_id) reason = "User %s forms archived for domain %s by system" % (user.raw_username, self.domain) form_processor_interface = FormProcessorInterface(self.domain) with open("cases_rebuilt.txt", "w+b") as case_log: for case_id in with_progress_bar(self.case_ids_to_rebuild): case_log.write("%s\n" % case_id) rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason=reason)) ledgers = form_processor_interface.ledger_db.get_ledgers_for_case(case_id) for ledger in ledgers: form_processor_interface.ledger_processor.rebuild_ledger_state( case_id, ledger.section_id, ledger.entry_id)
def rebuild_form_cases(sender, xform, *args, **kwargs): from casexml.apps.case.xform import get_case_ids_from_form from casexml.apps.case.cleanup import rebuild_case_from_forms domain = xform.domain case_ids = get_case_ids_from_form(xform) transactions = StockTransaction.objects.filter(report__form_id=xform.form_id) stock_case_ids = transactions.values_list('case_id', flat=True).distinct() case_ids.update(stock_case_ids) detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived) for case_id in case_ids: rebuild_case_from_forms(domain, case_id, detail)
def handle(self, domain, reason, **options): if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') ids = get_case_ids_in_domain(domain) for count, case_id in enumerate(ids): try: rebuild_case_from_forms(domain, case_id, RebuildWithReason(reason=reason)) if count % 100 == 0: print('rebuilt %s/%s cases' % (count, len(ids))) except Exception as e: logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def handle(self, *args, **options): if len(args) == 1: domain = args[0] else: raise CommandError('Usage: %s\n%s' % (self.args, self.help)) ids = get_case_ids_in_domain(domain) for count, case_id in enumerate(ids): try: rebuild_case_from_forms(case_id) if count % 100 == 0: print 'rebuilt %s/%s cases' % (count, len(ids)) except Exception, e: logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def handle(self, domain, reason, **options): if should_use_sql_backend(domain): raise CommandError( 'This command only works for couch-based domains.') ids = get_case_ids_in_domain(domain) for count, case_id in enumerate(ids): try: rebuild_case_from_forms(domain, case_id, RebuildWithReason(reason=reason)) if count % 100 == 0: print('rebuilt %s/%s cases' % (count, len(ids))) except Exception as e: logging.exception("couldn't rebuild case {id}. {msg}".format( id=case_id, msg=str(e)))
def _rebuild_case_with_retries(self, domain, case_id, detail): """ Rebuild a case with retries - retry in 5 min if failure occurs after (default_retry_delay) - retry a total of 3 times """ from casexml.apps.case.cleanup import rebuild_case_from_forms try: rebuild_case_from_forms(domain, case_id, detail) except (UnexpectedDeletedXForm, ResourceConflict) as exc: try: self.retry(exc=exc) except MaxRetriesExceededError: notify_exception("Maximum Retries Exceeded while rebuilding case {} during deletion.".format(case_id))
def _rebuild_case_with_retries(self, domain, case_id, detail): """ Rebuild a case with retries - retry in 5 min if failure occurs after (default_retry_delay) - retry a total of 3 times """ from casexml.apps.case.cleanup import rebuild_case_from_forms try: rebuild_case_from_forms(domain, case_id, detail) except (UnexpectedDeletedXForm, ResourceConflict) as exc: try: self.retry(exc=exc) except MaxRetriesExceededError: notify_exception( "Maximum Retries Exceeded while rebuilding case {} during deletion." .format(case_id))
def testRebuildEmpty(self): self.assertEqual(None, rebuild_case_from_forms('notarealid')) try: CommCareCase.get_with_rebuild('notarealid') self.fail('get with rebuild should still fail for unknown cases') except ResourceNotFound: pass
def test_case_rebuild(self): # Cases with rebuild actions were failing because rebuild actions have no form # https://manage.dimagi.com/default.asp?276216#1494409 self.factory.create_or_update_case( CaseStructure( self.case.case_id, attrs={ "update": { 'sword': '' }, }), ) rebuild_case_from_forms(self.domain, self.case.case_id, RebuildWithReason()) case = CaseAccessors(self.domain).get_case(self.case.case_id) changes, _ = get_paged_changes_to_case_property(case, 'sword') self.assertEqual(len(changes), 2) self.assertEqual(changes[0].new_value, '') self.assertEqual(changes[1].new_value, 'Narsil')
def test_couch_rebuild_deleted_case(self): # Note: Can't run this on SQL because if a case gets hard deleted then # there is no way to find out which forms created / updated it without # going through ALL the forms in the domain. ie. there is no SQL # equivalent to the "form_case_index/form_case_index" couch view case_id = _post_util(create=True) _post_util(case_id=case_id, p1='p1', p2='p2') # delete initial case delete_all_cases() with self.assertRaises(CaseNotFound): CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id) case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test')) self.assertEqual(case.p1, 'p1') self.assertEqual(case.p2, 'p2') self.assertEqual(3, len(primary_actions(case))) # create + update
def handle(self, log_file, **options): to_archive = [] case_id = options.get('case_id') case_forms = FormProcessorInterface(domain).get_case_forms(case_id) for form in case_forms: if form.user_id in ("system", "", None) and form.metadata.username == "system": updates = get_case_updates(form) if options.get('case_property') is not None: update_actions = [ update.get_update_action() for update in updates if update.id == case_id ] for action in update_actions: if isinstance(action, CaseUpdateAction): if options.get('case_property') in set( action.dynamic_properties.keys()): to_archive.append(form) else: to_archive.append(form) to_archive.sort(key=lambda f: f.received_on) to_archive = to_archive[:-1] print("Will archive {} forms".format(len(to_archive))) xform_archived.disconnect(rebuild_form_cases) with open(log_file, "w") as f: for form in with_progress_bar(to_archive): f.write(form.form_id + "\n") f.flush() if options['commit']: form.archive( user_id="archive_single_case_repeater_forms_script") xform_archived.connect(rebuild_form_cases) rebuild_case_from_forms( domain, case_id, UserRequestedRebuild( user_id="archive_single_case_repeater_forms_script"))
def testReconcileActions(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, form_extras={'received_on': now}) _post_util(case_id=case_id, p1='p1-1', p2='p2-1', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p2='p2-2', p3='p3-2', form_extras={'received_on': now + timedelta(seconds=2)}) case = CommCareCase.get(case_id) update_strategy = ActionsUpdateStrategy(case) original_actions = [deepcopy(a) for a in case.actions] original_form_ids = [id for id in case.xform_ids] self.assertEqual(3, len(original_actions)) self.assertEqual(3, len(original_form_ids)) self._assertListEqual(original_actions, case.actions) # test reordering case.actions = [case.actions[2], case.actions[1], case.actions[0]] self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListEqual(original_actions, case.actions) # test duplication case.actions = case.actions * 3 self.assertEqual(9, len(case.actions)) self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListEqual(original_actions, case.actions) # test duplication, even when dates are off case.actions = original_actions + [deepcopy(case.actions[2])] case.actions[-1].server_date = case.actions[-1].server_date + timedelta(seconds=1) self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListEqual(original_actions, case.actions) # test duplication with different properties is actually # treated differently case.actions = original_actions + [deepcopy(case.actions[2])] case.actions[-1].updated_unknown_properties['new'] = 'mismatch' self.assertEqual(4, len(case.actions)) self._assertListNotEqual(original_actions, case.actions) update_strategy.reconcile_actions() self._assertListNotEqual(original_actions, case.actions) # test clean slate rebuild case = rebuild_case_from_forms(case_id) self._assertListEqual(original_actions, primary_actions(case)) self._assertListEqual(original_form_ids, case.xform_ids)
def testRebuildCreateCase(self): case_id = _post_util(create=True) _post_util(case_id=case_id, p1='p1', p2='p2') # delete initial case case = CommCareCase.get(case_id) case.delete() try: CommCareCase.get(case_id) self.fail('get should fail on deleted cases') except ResourceNotFound: pass case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test')) self.assertEqual(case.p1, 'p1') self.assertEqual(case.p2, 'p2') self.assertEqual(2, len(primary_actions(case))) # create + update
def test_couch_rebuild_deleted_case(self): # Note: Can't run this on SQL because if a case gets hard deleted then # there is no way to find out which forms created / updated it without # going through ALL the forms in the domain. ie. there is no SQL # equivalent to the "form_case_index/form_case_index" couch view case_id = _post_util(create=True) _post_util(case_id=case_id, p1='p1', p2='p2') # delete initial case delete_all_cases() with self.assertRaises(CaseNotFound): CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id) case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test')) self.assertEqual(case.p1, 'p1') self.assertEqual(case.p2, 'p2') self.assertEqual(3, len(primary_actions(case))) # create + update
def rebuild_form_cases(sender, xform, *args, **kwargs): from casexml.apps.case.xform import get_case_ids_from_form from casexml.apps.case.cleanup import rebuild_case_from_forms for case_id in get_case_ids_from_form(xform): rebuild_case_from_forms(case_id)
def test_rebuild_empty(self): self.assertEqual( None, rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test')) )
def test_rebuild_empty(self): self.assertEqual( None, rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test')) )