def iter_ledger_diffs(case_ids, dd_count):
couch_state_map = {
state.ledger_reference: state
for state in StockState.objects.filter(case_id__in=case_ids)
}
sql_refs = set()
for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(case_ids):
ref = ledger_value.ledger_reference
sql_refs.add(ref)
couch_state = couch_state_map.get(ref, None)
if couch_state is None:
couch_json = get_stock_state_json(ledger_value)
dd_count("commcare.couchsqlmigration.ledger.rebuild")
else:
couch_json = couch_state.to_json()
dd_count("commcare.couchsqlmigration.ledger.diffed")
diffs = json_diff(couch_json, ledger_value.to_json(), track_list_indices=False)
diffs = filter_ledger_diffs(diffs)
if diffs:
dd_count("commcare.couchsqlmigration.ledger.has_diff")
yield "stock state", ref.as_id(), diffs
for ref, couch_state in couch_state_map.items():
if ref not in sql_refs:
diffs = json_diff(couch_state.to_json(), {}, track_list_indices=False)
dd_count("commcare.couchsqlmigration.ledger.diffed")
dd_count("commcare.couchsqlmigration.ledger.has_diff")
yield "stock state", ref.as_id(), filter_ledger_diffs(diffs)
def ledger_data():
from casexml.apps.stock.mock import Balance
from casexml.apps.case.mock import CaseFactory
from casexml.apps.stock.mock import Entry
factory = CaseFactory('domain')
with case_data() as case_ids:
balance_blocks = [
Balance(
entity_id=case_id,
date=datetime.utcnow(),
section_id='test',
entry=Entry(id='chocolate', quantity=4),
).as_xml()
for case_id in case_ids
]
form, _ = factory.post_case_blocks(balance_blocks)
ledgers = LedgerAccessorSQL.get_ledger_values_for_cases(case_ids)
try:
yield [ledger.ledger_id for ledger in ledgers]
finally:
form.delete()
for ledger in ledgers:
ledger.delete()
def diff_ledgers(case_ids, dd_count):
def diff(couch_state, ledger_value):
couch_json = couch_state.to_json() if couch_state is not None else {}
diffs = json_diff(couch_json,
ledger_value.to_json(),
track_list_indices=False)
return filter_ledger_diffs(diffs)
stock_tx = StockTransactionLoader()
couch_state_map = {
state.ledger_reference: state
for state in StockState.objects.filter(case_id__in=case_ids)
}
sql_refs = set()
all_diffs = []
all_changes = []
for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(
case_ids):
ref = ledger_value.ledger_reference
sql_refs.add(ref)
dd_count("commcare.couchsqlmigration.ledger.diffed")
couch_state = couch_state_map.get(ref, None)
if couch_state is None:
couch_state = stock_tx.get_stock_state(ref)
dd_count("commcare.couchsqlmigration.ledger.rebuild")
if couch_state is None:
diffs = [stock_tx.diff_missing_ledger(ledger_value)]
old_value = diffs[0].old_value
if old_value[
"form_state"] == FORM_PRESENT and "ledger" not in old_value:
changes = diffs_to_changes(diffs,
"missing couch stock transaction")
all_changes.append(("stock state", ref.as_id(), changes))
dd_count("commcare.couchsqlmigration.ledger.did_change")
diffs = []
else:
diffs = diff(couch_state, ledger_value)
if diffs and stock_tx.has_duplicate_transactions(ref):
changes = diffs_to_changes(diffs,
"duplicate stock transaction")
all_changes.append(("stock state", ref.as_id(), changes))
dd_count("commcare.couchsqlmigration.ledger.did_change")
diffs = []
if diffs:
dd_count("commcare.couchsqlmigration.ledger.has_diff")
all_diffs.append(("stock state", ref.as_id(), diffs))
for ref, couch_state in couch_state_map.items():
if ref not in sql_refs:
diffs = [stock_tx.diff_missing_ledger(couch_state, sql_miss=True)]
dd_count("commcare.couchsqlmigration.ledger.diffed")
dd_count("commcare.couchsqlmigration.ledger.has_diff")
all_diffs.append(("stock state", ref.as_id(), diffs))
return all_diffs, all_changes
def diff_ledgers(case_ids, statedb):
log.debug('Calculating ledger diffs for {} cases'.format(len(case_ids)))
couch_state_map = {
state.ledger_reference: state
for state in StockState.objects.filter(case_id__in=case_ids)
}
for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(
case_ids):
couch_state = couch_state_map.get(ledger_value.ledger_reference, None)
diffs = json_diff(couch_state.to_json(),
ledger_value.to_json(),
track_list_indices=False)
statedb.add_diffs('stock state', ledger_value.ledger_reference.as_id(),
filter_ledger_diffs(diffs))
def _diff_ledgers(self, case_ids):
from corehq.apps.tzmigration.timezonemigration import json_diff
from corehq.apps.commtrack.models import StockState
couch_state_map = {
state.ledger_reference: state
for state in StockState.objects.filter(case_id__in=case_ids)
}
self.log_debug('Calculating ledger diffs for {} cases'.format(len(case_ids)))
for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(case_ids):
couch_state = couch_state_map.get(ledger_value.ledger_reference, None)
diffs = json_diff(couch_state.to_json(), ledger_value.to_json(), track_list_indices=False)
self.diff_db.add_diffs(
'stock state', ledger_value.ledger_reference.as_id(),
filter_ledger_diffs(diffs)
)
def _diff_ledgers(self, case_ids):
from corehq.apps.tzmigration.timezonemigration import json_diff
from corehq.apps.commtrack.models import StockState
couch_state_map = {
state.ledger_reference: state
for state in StockState.objects.filter(case_id__in=case_ids)
}
self.log_debug('Calculating ledger diffs for {} cases'.format(len(case_ids)))
for ledger_value in LedgerAccessorSQL.get_ledger_values_for_cases(case_ids):
couch_state = couch_state_map.get(ledger_value.ledger_reference, None)
diffs = json_diff(couch_state.to_json(), ledger_value.to_json(), track_list_indices=False)
self.diff_db.add_diffs(
'stock state', ledger_value.ledger_reference.as_id(),
filter_ledger_diffs(diffs)
)
def get_tasks_case_immunization_ledger_values(tasks_case):
if tasks_case.type != 'tasks':
raise ValueError("Expected 'tasks' case")
return LedgerAccessorSQL.get_ledger_values_for_cases([tasks_case.case_id], section_id='immuns')
def get_sql_ledger_values(case_ids):
return LedgerAccessorSQL.get_ledger_values_for_cases(case_ids)
