def handle(self, *args, **options):
        count = 1
        filename = 'cases_with_mismatched_forms.log'
        dump_logs_to_file(filename)

        seen_cases = set()
        cases_rebuilt = set()
        with open(filename, 'r') as file:
            while True:
                line = file.readline()
                if not line:
                    return

                print 'Processing record {}'.format(count)
                record = json.loads(line)
                exception = ExceptionRecord.wrap(record['exception'])
                case_id = record.get('case_id', None)
                if not case_id or case_id in seen_cases:
                    count += 1
                    archive_exception(exception)
                    continue

                try:
                    seen_cases.add(case_id)
                    if should_rebuild(case_id):
                        cases_rebuilt.add(case_id)
                        rebuild_case_from_forms(case_id)
                        print 'rebuilt case {}'.format(case_id)
                    archive_exception(exception)
                except Exception, e:
                    logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
                finally:
                    count += 1
Exemplo n.º 2
0
    def test_rebuild_duplicate_index_identifiers(self):
        parent_case_id = uuid.uuid4().hex
        post_case_blocks([
            CaseBlock(parent_case_id, create=True).as_xml()
        ], domain=REBUILD_TEST_DOMAIN)

        child_case_id = uuid.uuid4().hex
        post_case_blocks([
            CaseBlock(child_case_id, create=True).as_xml()
        ], domain=REBUILD_TEST_DOMAIN)
        post_case_blocks([
            CaseBlock(child_case_id, index={'mom': ('mother', parent_case_id)}).as_xml()
        ], domain=REBUILD_TEST_DOMAIN)

        child_case = CaseAccessors(REBUILD_TEST_DOMAIN).get_case(child_case_id)
        self.assertEqual(1, len(child_case.indices))
        index = child_case.indices[0]
        index.id = None
        index.relationship = 'extension'
        index.save()

        child_case = CaseAccessors(REBUILD_TEST_DOMAIN).get_case(child_case_id)
        self.assertEqual(2, len(child_case.indices))

        with self.assertRaises(AssertionError):
            rebuild_case_from_forms(REBUILD_TEST_DOMAIN, child_case_id, RebuildWithReason(reason='test'))
Exemplo n.º 3
0
    def handle(self, log_file, **options):
        to_archive = []
        cases_affected = set()
        episode_case_ids = CaseAccessors(domain).get_case_ids_in_domain(
            "episode")
        if options.get('limit'):
            episode_case_ids = episode_case_ids[:options.get('limit')]
        for episode_case_id in with_progress_bar(episode_case_ids):
            nikshay_to_archive = []
            dots_99_to_archvie = []
            case_forms = FormProcessorInterface(domain).get_case_forms(
                episode_case_id)
            for form in case_forms:
                if form.user_id in ("system", "", None
                                    ) and form.metadata.username == "system":
                    updates = get_case_updates(form)
                    update_actions = [
                        update.get_update_action() for update in updates
                        if update.id == episode_case_id
                    ]
                    for action in update_actions:
                        if isinstance(action, CaseUpdateAction):
                            if set(action.dynamic_properties.keys()) == {
                                    "nikshay_registered", "nikshay_error"
                            }:
                                nikshay_to_archive.append(form)
                                cases_affected.add(episode_case_id)
                            elif set(action.dynamic_properties.keys()) == {
                                    "dots_99_registered", "dots_99_error"
                            }:
                                cases_affected.add(episode_case_id)
                                dots_99_to_archvie.append(form)

            # get_case_updates() returns the forms in the correct order, but sorting is probably a good idea in case
            # get_case_updates ever changes.
            nikshay_to_archive.sort(key=lambda f: f.received_on)
            dots_99_to_archvie.sort(key=lambda f: f.received_on)
            nikshay_to_archive = nikshay_to_archive[:-1]
            dots_99_to_archvie = dots_99_to_archvie[:-1]

            to_archive.extend(nikshay_to_archive)
            to_archive.extend(dots_99_to_archvie)

        print("Will archive {} forms".format(len(to_archive)))

        xform_archived.disconnect(rebuild_form_cases)
        with open(log_file, "w") as f:
            for form in with_progress_bar(to_archive):
                f.write(form.form_id + "\n")
                f.flush()
                if options['commit']:
                    form.archive(user_id="remove_duplicate_forms_script")
        xform_archived.connect(rebuild_form_cases)

        print("Will rebuild {} cases".format(len(cases_affected)))
        for case_id in with_progress_bar(cases_affected):
            rebuild_case_from_forms(
                domain, case_id,
                UserRequestedRebuild(user_id="remove_duplicate_forms_script"))
Exemplo n.º 4
0
    def test_couch_reconcile_actions(self):
        now = datetime.utcnow()
        # make sure we timestamp everything so they have the right order
        case_id = _post_util(create=True, form_extras={'received_on': now})
        _post_util(case_id=case_id,
                   p1='p1-1',
                   p2='p2-1',
                   form_extras={'received_on': now + timedelta(seconds=1)})
        _post_util(case_id=case_id,
                   p2='p2-2',
                   p3='p3-2',
                   form_extras={'received_on': now + timedelta(seconds=2)})
        case = CommCareCase.get(case_id)
        update_strategy = CouchCaseUpdateStrategy(case)

        original_actions = [deepcopy(a) for a in case.actions]
        original_form_ids = [id for id in case.xform_ids]
        self.assertEqual(4, len(original_actions))
        self.assertEqual(3, len(original_form_ids))
        self._assertListEqual(original_actions, case.actions)

        # test reordering
        case.actions = [
            case.actions[3], case.actions[2], case.actions[1], case.actions[0]
        ]
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListEqual(original_actions, case.actions)

        # test duplication
        case.actions = case.actions * 3
        self.assertEqual(12, len(case.actions))
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListEqual(original_actions, case.actions)

        # test duplication, even when dates are off
        case.actions = original_actions + [deepcopy(case.actions[2])]
        case.actions[-1].server_date = case.actions[
            -1].server_date + timedelta(seconds=1)
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListEqual(original_actions, case.actions)

        # test duplication with different properties is actually
        # treated differently
        case.actions = original_actions + [deepcopy(case.actions[2])]
        case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
        self.assertEqual(5, len(case.actions))
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListNotEqual(original_actions, case.actions)

        # test clean slate rebuild
        case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id,
                                       RebuildWithReason(reason='test'))
        self._assertListEqual(original_actions, primary_actions(case))
        self._assertListEqual(original_form_ids, case.xform_ids)
Exemplo n.º 5
0
 def test_case_actions(self):
     # make sure that when a case is rebuilt (using rebuild_case)
     # stock transactions show up as well
     form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE)
     case_id = self.case.case_id
     rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test'))
     case = CaseAccessors(self.domain).get_case(self.case.case_id)
     self.assertEqual(case.xform_ids[1:], [form_id])
     self.assertTrue(form_id in [action.form_id for action in case.actions])
Exemplo n.º 6
0
def rebuild_form_cases(sender, xform, *args, **kwargs):
    from casexml.apps.case.xform import get_case_ids_from_form
    from casexml.apps.case.cleanup import rebuild_case_from_forms

    domain = xform.domain
    case_ids = get_case_ids_from_form(xform)
    detail = FormArchiveRebuild(xmlns=xform.xmlns, form_id=xform.form_id, archived=xform.is_archived)
    for case_id in case_ids:
        rebuild_case_from_forms(domain, case_id, detail)
Exemplo n.º 7
0
 def test_case_actions(self):
     # make sure that when a case is rebuilt (using rebuild_case)
     # stock transactions show up as well
     form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE)
     case_id = self.case.case_id
     rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test'))
     case = CaseAccessors(self.domain).get_case(self.case.case_id)
     self.assertEqual(case.xform_ids[1:], [form_id])
     self.assertTrue(form_id in [action.form_id for action in case.actions])
Exemplo n.º 8
0
def rebuild_form_cases(sender, xform, *args, **kwargs):
    from casexml.apps.case.xform import get_case_ids_from_form
    from casexml.apps.case.cleanup import rebuild_case_from_forms

    case_ids = get_case_ids_from_form(xform)
    transactions = StockTransaction.objects.filter(report__form_id=xform.form_id)
    stock_case_ids = transactions.values_list('case_id', flat=True).distinct()
    case_ids.update(stock_case_ids)
    for case_id in case_ids:
        rebuild_case_from_forms(case_id)
Exemplo n.º 9
0
def rebuild_form_cases(sender, xform, *args, **kwargs):
    from casexml.apps.case.xform import get_case_ids_from_form
    from casexml.apps.case.cleanup import rebuild_case_from_forms
    from corehq.form_processor.parsers.ledgers.form import get_case_ids_from_stock_transactions

    domain = xform.domain
    case_ids = get_case_ids_from_form(xform) | get_case_ids_from_stock_transactions(xform)
    detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived)
    for case_id in case_ids:
        rebuild_case_from_forms(domain, case_id, detail)
Exemplo n.º 10
0
def rebuild_form_cases(sender, xform, *args, **kwargs):
    from casexml.apps.case.xform import get_case_ids_from_form
    from casexml.apps.case.cleanup import rebuild_case_from_forms
    from corehq.form_processor.parsers.ledgers.form import get_case_ids_from_stock_transactions

    domain = xform.domain
    case_ids = get_case_ids_from_form(xform) | get_case_ids_from_stock_transactions(xform)
    detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived)
    for case_id in case_ids:
        rebuild_case_from_forms(domain, case_id, detail)
Exemplo n.º 11
0
 def test_case_actions(self):
     """
     make sure that when a case is rebuilt (using rebuild_case)
     stock transactions show up as well
     """
     form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE)
     case_id = self.case.case_id
     rebuild_case_from_forms(case_id)
     case = CommCareCase.get(case_id)
     self.assertEqual(case.xform_ids, [form_id])
     self.assertEqual(case.actions[0].xform_id, form_id)
 def _rebuild_cases(self):
     user = CouchUser.get_by_user_id(self.user_id)
     reason = "User %s forms archived for domain %s by system" % (user.raw_username, self.domain)
     form_processor_interface = FormProcessorInterface(self.domain)
     with open("cases_rebuilt.txt", "w+b") as case_log:
         for case_id in with_progress_bar(self.case_ids_to_rebuild):
             case_log.write("%s\n" % case_id)
             rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason=reason))
             ledgers = form_processor_interface.ledger_db.get_ledgers_for_case(case_id)
             for ledger in ledgers:
                 form_processor_interface.ledger_processor.rebuild_ledger_state(
                     case_id, ledger.section_id, ledger.entry_id)
Exemplo n.º 13
0
def rebuild_form_cases(sender, xform, *args, **kwargs):
    from casexml.apps.case.xform import get_case_ids_from_form
    from casexml.apps.case.cleanup import rebuild_case_from_forms

    domain = xform.domain
    case_ids = get_case_ids_from_form(xform)
    transactions = StockTransaction.objects.filter(report__form_id=xform.form_id)
    stock_case_ids = transactions.values_list('case_id', flat=True).distinct()
    case_ids.update(stock_case_ids)
    detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived)
    for case_id in case_ids:
        rebuild_case_from_forms(domain, case_id, detail)
Exemplo n.º 14
0
    def handle(self, domain, reason, **options):
        if should_use_sql_backend(domain):
            raise CommandError('This command only works for couch-based domains.')

        ids = get_case_ids_in_domain(domain)
        for count, case_id in enumerate(ids):
            try:
                rebuild_case_from_forms(domain, case_id, RebuildWithReason(reason=reason))
                if count % 100 == 0:
                    print('rebuilt %s/%s cases' % (count, len(ids)))
            except Exception as e:
                logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
Exemplo n.º 15
0
    def handle(self, *args, **options):
        if len(args) == 1:
            domain = args[0]
        else:
            raise CommandError('Usage: %s\n%s' % (self.args, self.help))

        ids = get_case_ids_in_domain(domain)
        for count, case_id in enumerate(ids):
            try:
                rebuild_case_from_forms(case_id)
                if count % 100 == 0:
                    print 'rebuilt %s/%s cases' % (count, len(ids))
            except Exception, e:
                logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
Exemplo n.º 16
0
    def handle(self, domain, reason, **options):
        if should_use_sql_backend(domain):
            raise CommandError(
                'This command only works for couch-based domains.')

        ids = get_case_ids_in_domain(domain)
        for count, case_id in enumerate(ids):
            try:
                rebuild_case_from_forms(domain, case_id,
                                        RebuildWithReason(reason=reason))
                if count % 100 == 0:
                    print('rebuilt %s/%s cases' % (count, len(ids)))
            except Exception as e:
                logging.exception("couldn't rebuild case {id}. {msg}".format(
                    id=case_id, msg=str(e)))
Exemplo n.º 17
0
def _rebuild_case_with_retries(self, domain, case_id, detail):
    """
    Rebuild a case with retries
    - retry in 5 min if failure occurs after (default_retry_delay)
    - retry a total of 3 times
    """
    from casexml.apps.case.cleanup import rebuild_case_from_forms

    try:
        rebuild_case_from_forms(domain, case_id, detail)
    except (UnexpectedDeletedXForm, ResourceConflict) as exc:
        try:
            self.retry(exc=exc)
        except MaxRetriesExceededError:
            notify_exception("Maximum Retries Exceeded while rebuilding case {} during deletion.".format(case_id))
Exemplo n.º 18
0
def _rebuild_case_with_retries(self, domain, case_id, detail):
    """
    Rebuild a case with retries
    - retry in 5 min if failure occurs after (default_retry_delay)
    - retry a total of 3 times
    """
    from casexml.apps.case.cleanup import rebuild_case_from_forms
    try:
        rebuild_case_from_forms(domain, case_id, detail)
    except (UnexpectedDeletedXForm, ResourceConflict) as exc:
        try:
            self.retry(exc=exc)
        except MaxRetriesExceededError:
            notify_exception(
                "Maximum Retries Exceeded while rebuilding case {} during deletion."
                .format(case_id))
Exemplo n.º 19
0
 def testRebuildEmpty(self):
     self.assertEqual(None, rebuild_case_from_forms('notarealid'))
     try:
         CommCareCase.get_with_rebuild('notarealid')
         self.fail('get with rebuild should still fail for unknown cases')
     except ResourceNotFound:
         pass
 def test_case_rebuild(self):
     # Cases with rebuild actions were failing because rebuild actions have no form
     # https://manage.dimagi.com/default.asp?276216#1494409
     self.factory.create_or_update_case(
         CaseStructure(
             self.case.case_id,
             attrs={
                 "update": {
                     'sword': ''
                 },
             }),
     )
     rebuild_case_from_forms(self.domain, self.case.case_id, RebuildWithReason())
     case = CaseAccessors(self.domain).get_case(self.case.case_id)
     changes, _ = get_paged_changes_to_case_property(case, 'sword')
     self.assertEqual(len(changes), 2)
     self.assertEqual(changes[0].new_value, '')
     self.assertEqual(changes[1].new_value, 'Narsil')
Exemplo n.º 21
0
    def test_couch_rebuild_deleted_case(self):
        # Note: Can't run this on SQL because if a case gets hard deleted then
        # there is no way to find out which forms created / updated it without
        # going through ALL the forms in the domain. ie. there is no SQL
        # equivalent to the "form_case_index/form_case_index" couch view

        case_id = _post_util(create=True)
        _post_util(case_id=case_id, p1='p1', p2='p2')

        # delete initial case
        delete_all_cases()

        with self.assertRaises(CaseNotFound):
            CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id)

        case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
        self.assertEqual(case.p1, 'p1')
        self.assertEqual(case.p2, 'p2')
        self.assertEqual(3, len(primary_actions(case)))  # create + update
Exemplo n.º 22
0
    def handle(self, log_file, **options):
        to_archive = []
        case_id = options.get('case_id')
        case_forms = FormProcessorInterface(domain).get_case_forms(case_id)
        for form in case_forms:
            if form.user_id in ("system", "",
                                None) and form.metadata.username == "system":
                updates = get_case_updates(form)
                if options.get('case_property') is not None:
                    update_actions = [
                        update.get_update_action() for update in updates
                        if update.id == case_id
                    ]
                    for action in update_actions:
                        if isinstance(action, CaseUpdateAction):
                            if options.get('case_property') in set(
                                    action.dynamic_properties.keys()):
                                to_archive.append(form)
                else:
                    to_archive.append(form)

        to_archive.sort(key=lambda f: f.received_on)
        to_archive = to_archive[:-1]

        print("Will archive {} forms".format(len(to_archive)))

        xform_archived.disconnect(rebuild_form_cases)
        with open(log_file, "w") as f:
            for form in with_progress_bar(to_archive):
                f.write(form.form_id + "\n")
                f.flush()
                if options['commit']:
                    form.archive(
                        user_id="archive_single_case_repeater_forms_script")
        xform_archived.connect(rebuild_form_cases)

        rebuild_case_from_forms(
            domain, case_id,
            UserRequestedRebuild(
                user_id="archive_single_case_repeater_forms_script"))
Exemplo n.º 23
0
    def testReconcileActions(self):
        now = datetime.utcnow()
        # make sure we timestamp everything so they have the right order
        case_id = _post_util(create=True, form_extras={'received_on': now})
        _post_util(case_id=case_id, p1='p1-1', p2='p2-1', form_extras={'received_on': now + timedelta(seconds=1)})
        _post_util(case_id=case_id, p2='p2-2', p3='p3-2', form_extras={'received_on': now + timedelta(seconds=2)})
        case = CommCareCase.get(case_id)
        update_strategy = ActionsUpdateStrategy(case)

        original_actions = [deepcopy(a) for a in case.actions]
        original_form_ids = [id for id in case.xform_ids]
        self.assertEqual(3, len(original_actions))
        self.assertEqual(3, len(original_form_ids))
        self._assertListEqual(original_actions, case.actions)

        # test reordering
        case.actions = [case.actions[2], case.actions[1], case.actions[0]]
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListEqual(original_actions, case.actions)

        # test duplication
        case.actions = case.actions * 3
        self.assertEqual(9, len(case.actions))
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListEqual(original_actions, case.actions)

        # test duplication, even when dates are off
        case.actions = original_actions + [deepcopy(case.actions[2])]
        case.actions[-1].server_date = case.actions[-1].server_date + timedelta(seconds=1)
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListEqual(original_actions, case.actions)

        # test duplication with different properties is actually
        # treated differently
        case.actions = original_actions + [deepcopy(case.actions[2])]
        case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
        self.assertEqual(4, len(case.actions))
        self._assertListNotEqual(original_actions, case.actions)
        update_strategy.reconcile_actions()
        self._assertListNotEqual(original_actions, case.actions)

        # test clean slate rebuild
        case = rebuild_case_from_forms(case_id)
        self._assertListEqual(original_actions, primary_actions(case))
        self._assertListEqual(original_form_ids, case.xform_ids)
Exemplo n.º 24
0
    def testRebuildCreateCase(self):
        case_id = _post_util(create=True)
        _post_util(case_id=case_id, p1='p1', p2='p2')

        # delete initial case
        case = CommCareCase.get(case_id)
        case.delete()

        try:
            CommCareCase.get(case_id)
            self.fail('get should fail on deleted cases')
        except ResourceNotFound:
            pass

        case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
        self.assertEqual(case.p1, 'p1')
        self.assertEqual(case.p2, 'p2')
        self.assertEqual(2, len(primary_actions(case)))  # create + update
Exemplo n.º 25
0
    def test_couch_rebuild_deleted_case(self):
        # Note: Can't run this on SQL because if a case gets hard deleted then
        # there is no way to find out which forms created / updated it without
        # going through ALL the forms in the domain. ie. there is no SQL
        # equivalent to the "form_case_index/form_case_index" couch view

        case_id = _post_util(create=True)
        _post_util(case_id=case_id, p1='p1', p2='p2')

        # delete initial case
        delete_all_cases()

        with self.assertRaises(CaseNotFound):
            CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id)

        case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
        self.assertEqual(case.p1, 'p1')
        self.assertEqual(case.p2, 'p2')
        self.assertEqual(3, len(primary_actions(case)))  # create + update
Exemplo n.º 26
0
def rebuild_form_cases(sender, xform, *args, **kwargs):
    from casexml.apps.case.xform import get_case_ids_from_form
    from casexml.apps.case.cleanup import rebuild_case_from_forms
    for case_id in get_case_ids_from_form(xform):
        rebuild_case_from_forms(case_id)
Exemplo n.º 27
0
 def test_rebuild_empty(self):
     self.assertEqual(
         None,
         rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test'))
     )
Exemplo n.º 28
0
 def test_rebuild_empty(self):
     self.assertEqual(
         None,
         rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test'))
     )