def handle(self, *args, **options):
count = 1
filename = 'cases_with_mismatched_forms.log'
dump_logs_to_file(filename)
seen_cases = set()
cases_rebuilt = set()
with open(filename, 'r') as file:
while True:
line = file.readline()
if not line:
return
print 'Processing record {}'.format(count)
record = json.loads(line)
exception = ExceptionRecord.wrap(record['exception'])
case_id = record.get('case_id', None)
if not case_id or case_id in seen_cases:
count += 1
archive_exception(exception)
continue
try:
seen_cases.add(case_id)
if should_rebuild(case_id):
cases_rebuilt.add(case_id)
rebuild_case_from_forms(case_id)
print 'rebuilt case {}'.format(case_id)
archive_exception(exception)
except Exception, e:
logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
finally:
count += 1
def test_rebuild_duplicate_index_identifiers(self):
parent_case_id = uuid.uuid4().hex
post_case_blocks([
CaseBlock(parent_case_id, create=True).as_xml()
], domain=REBUILD_TEST_DOMAIN)
child_case_id = uuid.uuid4().hex
post_case_blocks([
CaseBlock(child_case_id, create=True).as_xml()
], domain=REBUILD_TEST_DOMAIN)
post_case_blocks([
CaseBlock(child_case_id, index={'mom': ('mother', parent_case_id)}).as_xml()
], domain=REBUILD_TEST_DOMAIN)
child_case = CaseAccessors(REBUILD_TEST_DOMAIN).get_case(child_case_id)
self.assertEqual(1, len(child_case.indices))
index = child_case.indices[0]
index.id = None
index.relationship = 'extension'
index.save()
child_case = CaseAccessors(REBUILD_TEST_DOMAIN).get_case(child_case_id)
self.assertEqual(2, len(child_case.indices))
with self.assertRaises(AssertionError):
rebuild_case_from_forms(REBUILD_TEST_DOMAIN, child_case_id, RebuildWithReason(reason='test'))
def handle(self, log_file, **options):
to_archive = []
cases_affected = set()
episode_case_ids = CaseAccessors(domain).get_case_ids_in_domain(
"episode")
if options.get('limit'):
episode_case_ids = episode_case_ids[:options.get('limit')]
for episode_case_id in with_progress_bar(episode_case_ids):
nikshay_to_archive = []
dots_99_to_archvie = []
case_forms = FormProcessorInterface(domain).get_case_forms(
episode_case_id)
for form in case_forms:
if form.user_id in ("system", "", None
) and form.metadata.username == "system":
updates = get_case_updates(form)
update_actions = [
update.get_update_action() for update in updates
if update.id == episode_case_id
]
for action in update_actions:
if isinstance(action, CaseUpdateAction):
if set(action.dynamic_properties.keys()) == {
"nikshay_registered", "nikshay_error"
}:
nikshay_to_archive.append(form)
cases_affected.add(episode_case_id)
elif set(action.dynamic_properties.keys()) == {
"dots_99_registered", "dots_99_error"
}:
cases_affected.add(episode_case_id)
dots_99_to_archvie.append(form)
# get_case_updates() returns the forms in the correct order, but sorting is probably a good idea in case
# get_case_updates ever changes.
nikshay_to_archive.sort(key=lambda f: f.received_on)
dots_99_to_archvie.sort(key=lambda f: f.received_on)
nikshay_to_archive = nikshay_to_archive[:-1]
dots_99_to_archvie = dots_99_to_archvie[:-1]
to_archive.extend(nikshay_to_archive)
to_archive.extend(dots_99_to_archvie)
print("Will archive {} forms".format(len(to_archive)))
xform_archived.disconnect(rebuild_form_cases)
with open(log_file, "w") as f:
for form in with_progress_bar(to_archive):
f.write(form.form_id + "\n")
f.flush()
if options['commit']:
form.archive(user_id="remove_duplicate_forms_script")
xform_archived.connect(rebuild_form_cases)
print("Will rebuild {} cases".format(len(cases_affected)))
for case_id in with_progress_bar(cases_affected):
rebuild_case_from_forms(
domain, case_id,
UserRequestedRebuild(user_id="remove_duplicate_forms_script"))
def test_couch_reconcile_actions(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True, form_extras={'received_on': now})
_post_util(case_id=case_id,
p1='p1-1',
p2='p2-1',
form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id,
p2='p2-2',
p3='p3-2',
form_extras={'received_on': now + timedelta(seconds=2)})
case = CommCareCase.get(case_id)
update_strategy = CouchCaseUpdateStrategy(case)
original_actions = [deepcopy(a) for a in case.actions]
original_form_ids = [id for id in case.xform_ids]
self.assertEqual(4, len(original_actions))
self.assertEqual(3, len(original_form_ids))
self._assertListEqual(original_actions, case.actions)
# test reordering
case.actions = [
case.actions[3], case.actions[2], case.actions[1], case.actions[0]
]
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication
case.actions = case.actions * 3
self.assertEqual(12, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication, even when dates are off
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].server_date = case.actions[
-1].server_date + timedelta(seconds=1)
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication with different properties is actually
# treated differently
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
self.assertEqual(5, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListNotEqual(original_actions, case.actions)
# test clean slate rebuild
case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id,
RebuildWithReason(reason='test'))
self._assertListEqual(original_actions, primary_actions(case))
self._assertListEqual(original_form_ids, case.xform_ids)
def test_case_actions(self):
# make sure that when a case is rebuilt (using rebuild_case)
# stock transactions show up as well
form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE)
case_id = self.case.case_id
rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test'))
case = CaseAccessors(self.domain).get_case(self.case.case_id)
self.assertEqual(case.xform_ids[1:], [form_id])
self.assertTrue(form_id in [action.form_id for action in case.actions])
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case_from_forms
domain = xform.domain
case_ids = get_case_ids_from_form(xform)
detail = FormArchiveRebuild(xmlns=xform.xmlns, form_id=xform.form_id, archived=xform.is_archived)
for case_id in case_ids:
rebuild_case_from_forms(domain, case_id, detail)
def test_case_actions(self):
# make sure that when a case is rebuilt (using rebuild_case)
# stock transactions show up as well
form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE)
case_id = self.case.case_id
rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason='test'))
case = CaseAccessors(self.domain).get_case(self.case.case_id)
self.assertEqual(case.xform_ids[1:], [form_id])
self.assertTrue(form_id in [action.form_id for action in case.actions])
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case_from_forms
case_ids = get_case_ids_from_form(xform)
transactions = StockTransaction.objects.filter(report__form_id=xform.form_id)
stock_case_ids = transactions.values_list('case_id', flat=True).distinct()
case_ids.update(stock_case_ids)
for case_id in case_ids:
rebuild_case_from_forms(case_id)
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case_from_forms
from corehq.form_processor.parsers.ledgers.form import get_case_ids_from_stock_transactions
domain = xform.domain
case_ids = get_case_ids_from_form(xform) | get_case_ids_from_stock_transactions(xform)
detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived)
for case_id in case_ids:
rebuild_case_from_forms(domain, case_id, detail)
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case_from_forms
from corehq.form_processor.parsers.ledgers.form import get_case_ids_from_stock_transactions
domain = xform.domain
case_ids = get_case_ids_from_form(xform) | get_case_ids_from_stock_transactions(xform)
detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived)
for case_id in case_ids:
rebuild_case_from_forms(domain, case_id, detail)
def test_case_actions(self):
"""
make sure that when a case is rebuilt (using rebuild_case)
stock transactions show up as well
"""
form_id = self._submit_ledgers(LEDGER_BLOCKS_SIMPLE)
case_id = self.case.case_id
rebuild_case_from_forms(case_id)
case = CommCareCase.get(case_id)
self.assertEqual(case.xform_ids, [form_id])
self.assertEqual(case.actions[0].xform_id, form_id)
def _rebuild_cases(self):
user = CouchUser.get_by_user_id(self.user_id)
reason = "User %s forms archived for domain %s by system" % (user.raw_username, self.domain)
form_processor_interface = FormProcessorInterface(self.domain)
with open("cases_rebuilt.txt", "w+b") as case_log:
for case_id in with_progress_bar(self.case_ids_to_rebuild):
case_log.write("%s\n" % case_id)
rebuild_case_from_forms(self.domain, case_id, RebuildWithReason(reason=reason))
ledgers = form_processor_interface.ledger_db.get_ledgers_for_case(case_id)
for ledger in ledgers:
form_processor_interface.ledger_processor.rebuild_ledger_state(
case_id, ledger.section_id, ledger.entry_id)
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case_from_forms
domain = xform.domain
case_ids = get_case_ids_from_form(xform)
transactions = StockTransaction.objects.filter(report__form_id=xform.form_id)
stock_case_ids = transactions.values_list('case_id', flat=True).distinct()
case_ids.update(stock_case_ids)
detail = FormArchiveRebuild(form_id=xform.form_id, archived=xform.is_archived)
for case_id in case_ids:
rebuild_case_from_forms(domain, case_id, detail)
def handle(self, domain, reason, **options):
if should_use_sql_backend(domain):
raise CommandError('This command only works for couch-based domains.')
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case_from_forms(domain, case_id, RebuildWithReason(reason=reason))
if count % 100 == 0:
print('rebuilt %s/%s cases' % (count, len(ids)))
except Exception as e:
logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case_from_forms(case_id)
if count % 100 == 0:
print 'rebuilt %s/%s cases' % (count, len(ids))
except Exception, e:
logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def handle(self, domain, reason, **options):
if should_use_sql_backend(domain):
raise CommandError(
'This command only works for couch-based domains.')
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case_from_forms(domain, case_id,
RebuildWithReason(reason=reason))
if count % 100 == 0:
print('rebuilt %s/%s cases' % (count, len(ids)))
except Exception as e:
logging.exception("couldn't rebuild case {id}. {msg}".format(
id=case_id, msg=str(e)))
def _rebuild_case_with_retries(self, domain, case_id, detail):
"""
Rebuild a case with retries
- retry in 5 min if failure occurs after (default_retry_delay)
- retry a total of 3 times
"""
from casexml.apps.case.cleanup import rebuild_case_from_forms
try:
rebuild_case_from_forms(domain, case_id, detail)
except (UnexpectedDeletedXForm, ResourceConflict) as exc:
try:
self.retry(exc=exc)
except MaxRetriesExceededError:
notify_exception("Maximum Retries Exceeded while rebuilding case {} during deletion.".format(case_id))
def _rebuild_case_with_retries(self, domain, case_id, detail):
"""
Rebuild a case with retries
- retry in 5 min if failure occurs after (default_retry_delay)
- retry a total of 3 times
"""
from casexml.apps.case.cleanup import rebuild_case_from_forms
try:
rebuild_case_from_forms(domain, case_id, detail)
except (UnexpectedDeletedXForm, ResourceConflict) as exc:
try:
self.retry(exc=exc)
except MaxRetriesExceededError:
notify_exception(
"Maximum Retries Exceeded while rebuilding case {} during deletion."
.format(case_id))
def testRebuildEmpty(self):
self.assertEqual(None, rebuild_case_from_forms('notarealid'))
try:
CommCareCase.get_with_rebuild('notarealid')
self.fail('get with rebuild should still fail for unknown cases')
except ResourceNotFound:
pass
def test_case_rebuild(self):
# Cases with rebuild actions were failing because rebuild actions have no form
# https://manage.dimagi.com/default.asp?276216#1494409
self.factory.create_or_update_case(
CaseStructure(
self.case.case_id,
attrs={
"update": {
'sword': ''
},
}),
)
rebuild_case_from_forms(self.domain, self.case.case_id, RebuildWithReason())
case = CaseAccessors(self.domain).get_case(self.case.case_id)
changes, _ = get_paged_changes_to_case_property(case, 'sword')
self.assertEqual(len(changes), 2)
self.assertEqual(changes[0].new_value, '')
self.assertEqual(changes[1].new_value, 'Narsil')
def test_couch_rebuild_deleted_case(self):
# Note: Can't run this on SQL because if a case gets hard deleted then
# there is no way to find out which forms created / updated it without
# going through ALL the forms in the domain. ie. there is no SQL
# equivalent to the "form_case_index/form_case_index" couch view
case_id = _post_util(create=True)
_post_util(case_id=case_id, p1='p1', p2='p2')
# delete initial case
delete_all_cases()
with self.assertRaises(CaseNotFound):
CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id)
case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
self.assertEqual(3, len(primary_actions(case))) # create + update
def handle(self, log_file, **options):
to_archive = []
case_id = options.get('case_id')
case_forms = FormProcessorInterface(domain).get_case_forms(case_id)
for form in case_forms:
if form.user_id in ("system", "",
None) and form.metadata.username == "system":
updates = get_case_updates(form)
if options.get('case_property') is not None:
update_actions = [
update.get_update_action() for update in updates
if update.id == case_id
]
for action in update_actions:
if isinstance(action, CaseUpdateAction):
if options.get('case_property') in set(
action.dynamic_properties.keys()):
to_archive.append(form)
else:
to_archive.append(form)
to_archive.sort(key=lambda f: f.received_on)
to_archive = to_archive[:-1]
print("Will archive {} forms".format(len(to_archive)))
xform_archived.disconnect(rebuild_form_cases)
with open(log_file, "w") as f:
for form in with_progress_bar(to_archive):
f.write(form.form_id + "\n")
f.flush()
if options['commit']:
form.archive(
user_id="archive_single_case_repeater_forms_script")
xform_archived.connect(rebuild_form_cases)
rebuild_case_from_forms(
domain, case_id,
UserRequestedRebuild(
user_id="archive_single_case_repeater_forms_script"))
def testReconcileActions(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True, form_extras={'received_on': now})
_post_util(case_id=case_id, p1='p1-1', p2='p2-1', form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id, p2='p2-2', p3='p3-2', form_extras={'received_on': now + timedelta(seconds=2)})
case = CommCareCase.get(case_id)
update_strategy = ActionsUpdateStrategy(case)
original_actions = [deepcopy(a) for a in case.actions]
original_form_ids = [id for id in case.xform_ids]
self.assertEqual(3, len(original_actions))
self.assertEqual(3, len(original_form_ids))
self._assertListEqual(original_actions, case.actions)
# test reordering
case.actions = [case.actions[2], case.actions[1], case.actions[0]]
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication
case.actions = case.actions * 3
self.assertEqual(9, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication, even when dates are off
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].server_date = case.actions[-1].server_date + timedelta(seconds=1)
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication with different properties is actually
# treated differently
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
self.assertEqual(4, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListNotEqual(original_actions, case.actions)
# test clean slate rebuild
case = rebuild_case_from_forms(case_id)
self._assertListEqual(original_actions, primary_actions(case))
self._assertListEqual(original_form_ids, case.xform_ids)
def testRebuildCreateCase(self):
case_id = _post_util(create=True)
_post_util(case_id=case_id, p1='p1', p2='p2')
# delete initial case
case = CommCareCase.get(case_id)
case.delete()
try:
CommCareCase.get(case_id)
self.fail('get should fail on deleted cases')
except ResourceNotFound:
pass
case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
self.assertEqual(2, len(primary_actions(case))) # create + update
def test_couch_rebuild_deleted_case(self):
# Note: Can't run this on SQL because if a case gets hard deleted then
# there is no way to find out which forms created / updated it without
# going through ALL the forms in the domain. ie. there is no SQL
# equivalent to the "form_case_index/form_case_index" couch view
case_id = _post_util(create=True)
_post_util(case_id=case_id, p1='p1', p2='p2')
# delete initial case
delete_all_cases()
with self.assertRaises(CaseNotFound):
CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id)
case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
self.assertEqual(3, len(primary_actions(case))) # create + update
def rebuild_form_cases(sender, xform, *args, **kwargs):
from casexml.apps.case.xform import get_case_ids_from_form
from casexml.apps.case.cleanup import rebuild_case_from_forms
for case_id in get_case_ids_from_form(xform):
rebuild_case_from_forms(case_id)
def test_rebuild_empty(self):
self.assertEqual(
None,
rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test'))
)
def test_rebuild_empty(self):
self.assertEqual(
None,
rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test'))
)