def _load_cases(self, pending):
"""Load cases and establish total and processed form counts
Cases for which all forms have been processed are enqueued to be
diffed.
:param pending: dict `{<case_id>: <processed_form_count>, ...}`
"""
log.debug("enqueue or load %s", pending)
cases = self.cases
case_ids = list(pending)
loaded_case_ids = set()
case_records = []
stock_forms = get_stock_forms_by_case_id(case_ids)
for case in CaseAccessorCouch.get_cases(case_ids):
loaded_case_ids.add(case.case_id)
case_stock_forms = stock_forms.get(case.case_id, [])
rec = CaseRecord(case, case_stock_forms, pending[case.case_id])
case_records.append(rec)
if rec.should_memorize_case:
log.debug("memorize %s", rec)
cases[case.case_id] = case
if case_records:
result = self.statedb.update_cases(case_records)
for case_id, total_forms, processed_forms in result:
if total_forms <= processed_forms:
self.enqueue(case_id)
missing = set(case_ids) - loaded_case_ids
if missing:
log.error("Found %s missing Couch cases", len(missing))
self.statedb.add_missing_docs("CommCareCase-couch", missing)
def load_and_diff_cases(case_ids, log_cases=False):
from ...casediff import _diff_state, diff_cases
should_diff = _diff_state.should_diff
couch_cases = {
c.case_id: c.to_json()
for c in CaseAccessorCouch.get_cases(case_ids) if should_diff(c)
}
if log_cases:
skipped = [id for id in case_ids if id not in couch_cases]
if skipped:
log.info("skipping cases modified since cutoff date: %s", skipped)
return diff_cases(couch_cases, log_cases=log_cases)
def _diff_cases(self):
def diff(case_ids):
couch_cases = {}
to_load = []
pop_case = self.cases.pop
for case_id in case_ids:
case = pop_case(case_id, None)
if case is None:
to_load.append(case_id)
else:
couch_cases[case_id] = case.to_json()
for case in CaseAccessorCouch.get_cases(to_load):
couch_cases[case.case_id] = case.to_json()
diff_cases(couch_cases, statedb=self.statedb)
self.cache_hits[0] += len(case_ids) - len(to_load)
self.cache_hits[1] += len(case_ids)
self.num_diffed_cases += len(case_ids)
def _diff_cases(self):
def diff(case_ids):
couch_cases = []
to_load = []
pop_case = self.cases.pop
for case_id in case_ids:
case = pop_case(case_id, None)
if case is None:
to_load.append(case_id)
else:
couch_cases.append(case)
popped_cases = list(couch_cases)
couch_cases.extend(CaseAccessorCouch.get_cases(to_load))
prune = (iter if self._is_flushing
else partial(prune_premature_diffs, statedb=self.statedb))
json_by_id = {c.case_id: c.to_json() for c in prune(couch_cases)}
restore_cached(popped_cases, json_by_id)
if json_by_id:
diff_cases_and_save_state(json_by_id, statedb=self.statedb)
self.cache_hits[0] += len(json_by_id.keys() - to_load)
self.cache_hits[1] += len(json_by_id)
self.num_diffed_cases += len(json_by_id)
def get_couch_cases(case_ids):
return CaseAccessorCouch.get_cases(case_ids)