def _load_cases(self, pending): """Load cases and establish total and processed form counts Cases for which all forms have been processed are enqueued to be diffed. :param pending: dict `{<case_id>: <processed_form_count>, ...}` """ log.debug("enqueue or load %s", pending) cases = self.cases case_ids = list(pending) loaded_case_ids = set() case_records = [] stock_forms = get_stock_forms_by_case_id(case_ids) for case in CaseAccessorCouch.get_cases(case_ids): loaded_case_ids.add(case.case_id) case_stock_forms = stock_forms.get(case.case_id, []) rec = CaseRecord(case, case_stock_forms, pending[case.case_id]) case_records.append(rec) if rec.should_memorize_case: log.debug("memorize %s", rec) cases[case.case_id] = case if case_records: result = self.statedb.update_cases(case_records) for case_id, total_forms, processed_forms in result: if total_forms <= processed_forms: self.enqueue(case_id) missing = set(case_ids) - loaded_case_ids if missing: log.error("Found %s missing Couch cases", len(missing)) self.statedb.add_missing_docs("CommCareCase-couch", missing)
def load_and_diff_cases(case_ids, log_cases=False): from ...casediff import _diff_state, diff_cases should_diff = _diff_state.should_diff couch_cases = { c.case_id: c.to_json() for c in CaseAccessorCouch.get_cases(case_ids) if should_diff(c) } if log_cases: skipped = [id for id in case_ids if id not in couch_cases] if skipped: log.info("skipping cases modified since cutoff date: %s", skipped) return diff_cases(couch_cases, log_cases=log_cases)
def _diff_cases(self): def diff(case_ids): couch_cases = {} to_load = [] pop_case = self.cases.pop for case_id in case_ids: case = pop_case(case_id, None) if case is None: to_load.append(case_id) else: couch_cases[case_id] = case.to_json() for case in CaseAccessorCouch.get_cases(to_load): couch_cases[case.case_id] = case.to_json() diff_cases(couch_cases, statedb=self.statedb) self.cache_hits[0] += len(case_ids) - len(to_load) self.cache_hits[1] += len(case_ids) self.num_diffed_cases += len(case_ids)
def _diff_cases(self): def diff(case_ids): couch_cases = [] to_load = [] pop_case = self.cases.pop for case_id in case_ids: case = pop_case(case_id, None) if case is None: to_load.append(case_id) else: couch_cases.append(case) popped_cases = list(couch_cases) couch_cases.extend(CaseAccessorCouch.get_cases(to_load)) prune = (iter if self._is_flushing else partial(prune_premature_diffs, statedb=self.statedb)) json_by_id = {c.case_id: c.to_json() for c in prune(couch_cases)} restore_cached(popped_cases, json_by_id) if json_by_id: diff_cases_and_save_state(json_by_id, statedb=self.statedb) self.cache_hits[0] += len(json_by_id.keys() - to_load) self.cache_hits[1] += len(json_by_id) self.num_diffed_cases += len(json_by_id)
def get_couch_cases(case_ids): return CaseAccessorCouch.get_cases(case_ids)