def test_couch_reconcile_actions(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True, form_extras={'received_on': now})
_post_util(case_id=case_id, p1='p1-1', p2='p2-1', form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id, p2='p2-2', p3='p3-2', form_extras={'received_on': now + timedelta(seconds=2)})
case = CommCareCase.get(case_id)
update_strategy = CouchCaseUpdateStrategy(case)
original_actions = [deepcopy(a) for a in case.actions]
original_form_ids = [id for id in case.xform_ids]
self.assertEqual(4, len(original_actions))
self.assertEqual(3, len(original_form_ids))
self._assertListEqual(original_actions, case.actions)
# test reordering
case.actions = [case.actions[3], case.actions[2], case.actions[1], case.actions[0]]
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication
case.actions = case.actions * 3
self.assertEqual(12, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication, even when dates are off
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].server_date = case.actions[-1].server_date + timedelta(seconds=1)
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListEqual(original_actions, case.actions)
# test duplication with different properties is actually
# treated differently
case.actions = original_actions + [deepcopy(case.actions[2])]
case.actions[-1].updated_unknown_properties['new'] = 'mismatch'
self.assertEqual(5, len(case.actions))
self._assertListNotEqual(original_actions, case.actions)
update_strategy.reconcile_actions()
self._assertListNotEqual(original_actions, case.actions)
# test clean slate rebuild
case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
self._assertListEqual(original_actions, primary_actions(case))
self._assertListEqual(original_form_ids, case.xform_ids)
def reconcile_transactions(self):
transactions = self.case.transactions
sorted_transactions = sorted(transactions,
key=_transaction_sort_key_function(
self.case))
if sorted_transactions:
if not sorted_transactions[0].is_case_create:
error = "Case {0} first transaction not create transaction: {1}"
raise ReconciliationError(
error.format(self.case.case_id, sorted_transactions[0]))
self._fetch_case_transaction_forms(sorted_transactions)
rebuild_detail = RebuildWithReason(reason="client_date_reconciliation")
rebuild_transaction = CaseTransaction.rebuild_transaction(
self.case, rebuild_detail)
self.rebuild_from_transactions(sorted_transactions,
rebuild_transaction)
def test_couch_rebuild_deleted_case(self):
# Note: Can't run this on SQL because if a case gets hard deleted then
# there is no way to find out which forms created / updated it without
# going through ALL the forms in the domain. ie. there is no SQL
# equivalent to the "form_case_index/form_case_index" couch view
case_id = _post_util(create=True)
_post_util(case_id=case_id, p1='p1', p2='p2')
# delete initial case
delete_all_cases()
with self.assertRaises(CaseNotFound):
CaseAccessors(REBUILD_TEST_DOMAIN).get_case(case_id)
case = rebuild_case_from_forms(REBUILD_TEST_DOMAIN, case_id, RebuildWithReason(reason='test'))
self.assertEqual(case.p1, 'p1')
self.assertEqual(case.p2, 'p2')
self.assertEqual(3, len(primary_actions(case))) # create + update
def handle(self, *args, **options):
if len(args) == 2:
domain = args[0]
reason = args[1]
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
if should_use_sql_backend(domain):
raise CommandError(
'This command only works for couch-based domains.')
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case_from_forms(domain, case_id,
RebuildWithReason(reason=reason))
if count % 100 == 0:
print 'rebuilt %s/%s cases' % (count, len(ids))
except Exception, e:
logging.exception("couldn't rebuild case {id}. {msg}".format(
id=case_id, msg=str(e)))
def unsort_transactions(sql_case, commit):
rebuilds = [t for t in sql_case.transactions if is_rebuild(t)]
others = [t for t in sql_case.transactions if not is_rebuild(t)]
assert len(rebuilds) == 1, rebuilds
changes = []
for trans in sorted(others, key=lambda t: t.server_date):
if not trans.form_id:
continue
received_on = FormAccessorSQL.get_form(trans.form_id).received_on
if received_on != trans.server_date:
changes.append((trans, received_on))
if commit:
trans.server_date = received_on
trans.save()
case_id = sql_case.case_id
if changes:
chg = "\n".join(f" {t.form_id}: {t.server_date} -> {received_on}"
for t, received_on in changes)
log.info("unsort transactions for case %s:\n%s", case_id, chg)
if commit:
detail = RebuildWithReason(reason=UNSORT_REBUILD_REASON)
rebuild_case(sql_case, detail)
else:
log.info("no changes for case %s", case_id)
def rebuild_cases(cases_to_rebuild_by_domain, logger):
detail = RebuildWithReason(reason='undo UUID clash')
for domain, case_ids in six.iteritems(cases_to_rebuild_by_domain):
for case_id in case_ids:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)
logger.log('Case %s rebuilt' % case_id)
def test_ignores_before_rebuild_transaction(self):
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-11"):
new_old_xform = self._create_form()
with freeze_time("2018-10-08"):
new_old_trans = self._create_case_transaction(case, new_old_xform)
with freeze_time("2018-10-11"):
self._save(new_old_xform, case, new_old_trans)
self.assertFalse(case.check_transaction_order())
with freeze_time("2018-10-13"):
new_rebuild_xform = self._create_form()
rebuild_detail = RebuildWithReason(reason="shadow's golden coin")
rebuild_transaction = CaseTransaction.rebuild_transaction(
case, rebuild_detail)
self._save(new_rebuild_xform, case, rebuild_transaction)
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
def test_first_transaction_not_create(self):
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-08"):
new_old_xform = self._create_form()
new_old_trans = self._create_case_transaction(case, new_old_xform)
def test_rebuild_empty(self):
self.assertEqual(
None,
rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test'))
)
def test_blank_change(self):
self.factory.create_or_update_case(
CaseStructure(self.case.case_id, attrs={
"update": {
'sword': ''
},
}), )
case = CaseAccessors(self.domain).get_case(self.case.case_id)
changes, _ = get_paged_changes_to_case_property(case, 'sword')
self.assertEqual(len(changes), 2)
self.assertEqual(changes[0].new_value, '')
self.assertEqual(changes[1].new_value, 'Narsil')
@run_with_all_backends
def test_case_rebuild(self):
# Cases with rebuild actions were failing because rebuild actions have no form
# https://manage.dimagi.com/default.asp?276216#1494409
self.factory.create_or_update_case(
CaseStructure(self.case.case_id, attrs={
"update": {
'sword': ''
},
}), )
rebuild_case_from_forms(self.domain, self.case.case_id,
RebuildWithReason())
case = CaseAccessors(self.domain).get_case(self.case.case_id)
changes, _ = get_paged_changes_to_case_property(case, 'sword')
self.assertEqual(len(changes), 2)
self.assertEqual(changes[0].new_value, '')
self.assertEqual(changes[1].new_value, 'Narsil')
def rebuild_case(sql_case, detail=None):
if detail is None:
detail = RebuildWithReason(reason=COUCH_SQL_REBUILD_REASON)
return FormProcessorSQL.hard_rebuild_case(sql_case.domain, sql_case.case_id, detail)
def rebuild_cases(cases_to_rebuild_by_domain):
detail = RebuildWithReason(reason='undo UUID clash')
for domain, case_ids in cases_to_rebuild_by_domain.iteritems():
for case_id in case_ids:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)