def test_patch_opened_by_with_xform_ids():
diffs = [
Diff("diff", ["opened_by"], "old", "new"),
Diff("set_mismatch", ["xform_ids", "[*]"], "old", "new"),
]
case = mod.PatchCase(FakeCase(), diffs)
check_diff_block(case, diffs)
def test_can_patch_opened_by_with_user_id():
diffs = [
Diff("diff", ["opened_by"], "old", "new"),
Diff("diff", ["user_id"], "old", "new"),
]
case = mod.PatchCase(FakeCase(), diffs)
eq(case.dynamic_case_properties(), {})
check_diff_block(case, diffs)
def add_cases_missing_from_couch(data, case_ids):
sql_cases = {c.case_id: c for c in get_sql_cases(list(case_ids))}
data.doc_ids.extend(case_ids)
for case_id in case_ids:
if case_id in sql_cases:
new = "present"
if sql_cases[case_id].deleted:
miss = Diff("missing", path=["*"], old_value=MISSING, new_value=new)
data.diffs.append(("CommCareCase-Deleted", case_id, [miss]))
new = None
else:
forms = find_processed_and_unmigrated_form_ids(case_id)
new = f"missing with forms {forms}" if forms else None
diff = Diff("missing", path=["*"], old_value=MISSING, new_value=new)
data.diffs.append(("CommCareCase", case_id, [diff] if new else []))
def _load_cases(self, pending):
"""Load cases and establish total and processed form counts
Cases for which all forms have been processed are enqueued to be
diffed.
:param pending: dict `{<case_id>: <processed_form_count>, ...}`
"""
log.debug("enqueue or load %s", pending)
cases = self.cases
case_ids = list(pending)
loaded_case_ids = set()
case_records = []
stock_forms = get_stock_forms_by_case_id(case_ids)
for case in get_couch_cases(case_ids):
loaded_case_ids.add(case.case_id)
case_stock_forms = stock_forms.get(case.case_id, [])
rec = CaseRecord(case, case_stock_forms, pending[case.case_id])
case_records.append(rec)
if rec.should_memorize_case:
log.debug("memorize %s", rec)
cases[case.case_id] = case
if case_records:
result = self.statedb.update_cases(case_records)
for case_id, total_forms, processed_forms in result:
if total_forms <= processed_forms:
self.enqueue(case_id, processed_forms)
missing = set(case_ids) - loaded_case_ids
if missing:
log.error("Found %s missing Couch cases", len(missing))
self.statedb.replace_case_diffs([(
"CommCareCase",
case_id,
[Diff("missing", path=["*"], old_value=MISSING, new_value="?")],
) for case_id in missing])
def add_missing_docs(data, couch_cases, sql_case_ids, dd_count):
def as_change(item, reason):
kind, doc_id, diffs = item
return kind, doc_id, diffs_to_changes(diffs, reason)
if len(couch_cases) != len(sql_case_ids):
only_in_sql = sql_case_ids - couch_cases.keys()
assert not only_in_sql, only_in_sql
only_in_couch = couch_cases.keys() - sql_case_ids
data.doc_ids.extend(only_in_couch)
dd_count("commcare.couchsqlmigration.case.missing_from_sql", value=len(only_in_couch))
for case_id in only_in_couch:
couch_case = couch_cases[case_id]
diff = change = (couch_case["doc_type"], case_id, [])
item = (
couch_case["doc_type"],
case_id,
[Diff("missing", path=["*"], old_value="*", new_value=MISSING)],
)
if has_only_deleted_forms(couch_case):
change = as_change(item, "deleted forms")
elif is_orphaned_case(couch_case):
change = as_change(item, "orphaned case")
else:
diff = item
data.diffs.append(diff)
data.changes.append(change)
def add_cases_missing_from_couch(data, case_ids):
sql_ids = {c.case_id for c in CaseAccessorSQL.get_cases(list(case_ids))}
data.doc_ids.extend(case_ids)
for case_id in case_ids:
new = "present" if case_id in sql_ids else MISSING
data.diffs.append((
"CommCareCase",
case_id,
[Diff("missing", path=["*"], old_value=MISSING, new_value=new)],
))
def diff_missing_ledger(self, ledger, *, sql_miss=False):
"""Get the state of the form reference by ledger
:param ledger: Object having `last_modified_form_id` attribute
(`LedgerValue` or `StockState`).
"""
form_id = ledger.last_modified_form_id
old, new = diff_form_state(form_id, in_couch=form_id in self.ledger_refs)
if sql_miss:
old["ledger"] = ledger.to_json()
else:
new["ledger"] = ledger.to_json()
return Diff("missing", path=["*"], old_value=old, new_value=new)
def assert_patched(form, diffs, expect_patched=True):
patch_diffs = [
diff for diff in diffs
if not mod.is_patchable(diff) or diff.old_value is MISSING
]
if not any(d.path[0] == "xform_ids" for d in patch_diffs):
patch_diffs.append(
Diff("set_mismatch", ["xform_ids", "[*]"], "old", "new"))
with patch.object(casediff, "get_sql_forms", lambda x, **k: [form]):
actual_patched = casediff.is_case_patched(FakeCase.case_id,
patch_diffs)
sep = "\n"
assert actual_patched == expect_patched, (
f"is_case_patched(case_id, diffs) -> {actual_patched}\n"
f"{sep.join(repr(d) for d in patch_diffs)}\n\n{form.diff_block}")
def add_missing_docs(data, couch_cases, sql_case_ids, dd_count):
if len(couch_cases) != len(sql_case_ids):
only_in_sql = sql_case_ids - couch_cases.keys()
assert not only_in_sql, only_in_sql
only_in_couch = couch_cases.keys() - sql_case_ids
data.doc_ids.extend(only_in_couch)
dd_count("commcare.couchsqlmigration.case.missing_from_sql", value=len(only_in_couch))
for case_id in only_in_couch:
couch_case = couch_cases[case_id]
if is_orphaned_case(couch_case):
log.info("Ignoring orphaned case: %s", couch_case["_id"])
continue
data.diffs.append((
couch_case["doc_type"],
case_id,
[Diff("missing", path=["*"], old_value="*", new_value=MISSING)],
))
def diff_case_forms(couch_json, sql_json):
couch_ids = {a["xform_id"] for a in couch_json["actions"] if a["xform_id"]}
sql_ids = {t["xform_id"] for t in sql_json["actions"] if t["xform_id"]}
only_in_couch = couch_ids - sql_ids
if not only_in_couch:
return []
old_forms = {}
new_forms = {}
for form_id in only_in_couch:
old, new = diff_form_state(form_id)
old_forms[form_id] = old["form_state"]
new_forms[form_id] = new["form_state"]
if any(v != FORM_PRESENT for v in old_forms.values()):
return [Diff(
"diff",
path=["?"],
old_value={"forms": old_forms},
new_value={"forms": new_forms},
)]
return []
def test_patch_missing_case_property():
diffs = [Diff("diff", ["gone"], MISSING, "new")]
case = mod.PatchCase(FakeCase(), diffs)
eq(case.dynamic_case_properties(), {"gone": ""})
check_diff_block(case, diffs, [Diff("diff", ["gone"], MISSING, "")])
def test_patch_modified_by():
diffs = [Diff("diff", ["modified_by"], "old", "new")]
case = mod.PatchCase(FakeCase(), diffs)
check_diff_block(case, diffs)
def ledger_diff(*args):
ref = mod.UniqueLedgerReference.from_id("led/ger/ref")
return mod.LedgerDiff(Diff(*args), ref)