def _migrate_case_actions(couch_case, sql_case):
from casexml.apps.case import const
transactions = {}
for action in couch_case.actions:
if action.xform_id and action.xform_id in transactions:
transaction = transactions[action.xform_id]
else:
transaction = CaseTransaction(
case=sql_case,
form_id=action.xform_id,
sync_log_id=action.sync_log_id,
type=CaseTransaction.TYPE_FORM if action.xform_id else 0,
server_date=action.server_date,
)
if action.xform_id:
transactions[action.xform_id] = transaction
else:
sql_case.track_create(transaction)
if action.action_type == const.CASE_ACTION_CREATE:
transaction.type |= CaseTransaction.TYPE_CASE_CREATE
if action.action_type == const.CASE_ACTION_CLOSE:
transaction.type |= CaseTransaction.TYPE_CASE_CLOSE
if action.action_type == const.CASE_ACTION_INDEX:
transaction.type |= CaseTransaction.TYPE_CASE_INDEX
if action.action_type == const.CASE_ACTION_ATTACHMENT:
transaction.type |= CaseTransaction.TYPE_CASE_ATTACHMENT
if action.action_type == const.CASE_ACTION_REBUILD:
transaction.type = CaseTransaction.TYPE_REBUILD_WITH_REASON
transaction.details = RebuildWithReason(reason="Unknown")
for transaction in transactions.values():
sql_case.track_create(transaction)
def add_transaction_for_form(case, case_update, form):
types = [CaseTransaction.type_from_action_type_slug(a.action_type_slug) for a in case_update.actions]
transaction = CaseTransaction.form_transaction(case, form, case_update.guess_modified_on(), types)
for trans in case.get_tracked_models_to_create(CaseTransaction):
if transaction == trans:
trans.type |= transaction.type
break
else:
case.track_create(transaction)
class TestCaseESAccessorsSQL(TestCaseESAccessors):
def _send_case_to_es(self,
domain=None,
owner_id=None,
user_id=None,
case_type=None,
opened_on=None,
closed_on=None):
case = CommCareCaseSQL(case_id=uuid.uuid4().hex,
domain=domain or self.domain,
owner_id=owner_id or self.owner_id,
modified_by=user_id or self.user_id,
type=case_type or self.case_type,
opened_on=opened_on or datetime.now(),
opened_by=user_id or self.user_id,
closed_on=closed_on,
modified_on=datetime.now(),
closed_by=user_id or self.user_id,
server_modified_on=datetime.utcnow(),
closed=bool(closed_on))
case.track_create(
CaseTransaction(
type=CaseTransaction.TYPE_FORM,
form_id=uuid.uuid4().hex,
case=case,
server_date=opened_on,
))
def apply_action_intent(self, case_action_intent):
if not case_action_intent.is_deprecation:
# for now we only allow commtrack actions to be processed this way so just assert that's the case
assert case_action_intent.action_type == CASE_ACTION_COMMTRACK
transaction = CaseTransaction.ledger_transaction(self.case, case_action_intent.form)
if transaction not in self.case.get_tracked_models_to_create(CaseTransaction):
self.case.track_create(transaction)
class SqlCaseUpdateStrategy(UpdateStrategy):
case_implementation_class = CommCareCaseSQL
def apply_action_intents(self, primary_intent, deprecation_intent=None):
# for now we only allow commtrack actions to be processed this way so just assert that's the case
if primary_intent:
assert primary_intent.action_type == CASE_ACTION_COMMTRACK
transaction = CaseTransaction.ledger_transaction(self.case, primary_intent.form)
if deprecation_intent:
assert transaction.is_saved()
elif transaction not in self.case.get_tracked_models_to_create(CaseTransaction):
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
transaction.sync_log_id = None
self.case.track_create(transaction)
elif deprecation_intent:
transaction = self.case.get_transaction_by_form_id(deprecation_intent.form.orig_id)
transaction.sync_log_id = None
transaction.type -= CaseTransaction.TYPE_LEDGER
self.case.track_update(transaction)
def update_from_case_update(self, case_update, xformdoc, other_forms=None):
self._apply_case_update(case_update, xformdoc)
self.add_transaction_for_form(self.case, case_update, xformdoc)
@staticmethod
def add_transaction_for_form(case, case_update, form):
types = [CaseTransaction.type_from_action_type_slug(a.action_type_slug) for a in case_update.actions]
transaction = CaseTransaction.form_transaction(case, form, case_update.guess_modified_on(), types)
for trans in case.get_tracked_models_to_create(CaseTransaction):
if transaction == trans:
trans.type |= transaction.type
break
def update_from_case_update(self, case_update, xformdoc, other_forms=None):
self._apply_case_update(case_update, xformdoc)
transaction = CaseTransaction.form_transaction(self.case, xformdoc)
if transaction not in self.case.get_tracked_models_to_create(CaseTransaction):
# don't add multiple transactions for the same form
self.case.track_create(transaction)
def _rebuild_case_from_transactions(case, detail, updated_xforms=None):
transactions = get_case_transactions(case.case_id, updated_xforms=updated_xforms)
strategy = SqlCaseUpdateStrategy(case)
rebuild_transaction = CaseTransaction.rebuild_transaction(case, detail)
strategy.rebuild_from_transactions(transactions, rebuild_transaction)
return case
def update_from_case_update(self, case_update, xformdoc, other_forms=None):
self._apply_case_update(case_update, xformdoc)
types = [CaseTransaction.type_from_action_type_slug(a.action_type_slug) for a in case_update.actions]
transaction = CaseTransaction.form_transaction(self.case, xformdoc, types)
if transaction not in self.case.get_tracked_models_to_create(CaseTransaction):
# don't add multiple transactions for the same form
self.case.track_create(transaction)
def _fetch_case_transaction_forms(self, transactions, updated_xforms=None):
"""
Fetch the forms for a list of transactions, caching them on each transaction
:param transactions: list of ``CaseTransaction`` objects:
:param updated_xforms: optional list of forms that have been changed.
"""
form_ids = {tx.form_id for tx in transactions if tx.form_id}
updated_xforms_map = {
xform.form_id: xform
for xform in updated_xforms if not xform.is_deprecated
} if updated_xforms else {}
updated_xform_ids = set(updated_xforms_map)
form_ids_to_fetch = list(form_ids - updated_xform_ids)
form_load_counter("rebuild_case",
self.case.domain)(len(form_ids_to_fetch))
xform_map = {
form.form_id: form
for form in XFormInstance.objects.get_forms_with_attachments_meta(
form_ids_to_fetch)
}
forms_missing_transactions = list(updated_xform_ids - form_ids)
for form_id in forms_missing_transactions:
# Add in any transactions that aren't already present
form = updated_xforms_map[form_id]
case_updates = [
update for update in get_case_updates(form)
if update.id == self.case.case_id
]
types = [
CaseTransaction.type_from_action_type_slug(a.action_type_slug)
for case_update in case_updates for a in case_update.actions
]
modified_on = case_updates[0].guess_modified_on()
new_transaction = CaseTransaction.form_transaction(
self.case, form, modified_on, types)
transactions.append(new_transaction)
def get_form(form_id):
if form_id in updated_xforms_map:
return updated_xforms_map[form_id]
try:
return xform_map[form_id]
except KeyError:
raise XFormNotFound(form_id)
for case_transaction in transactions:
if case_transaction.form_id:
try:
case_transaction.cached_form = get_form(
case_transaction.form_id)
except XFormNotFound:
logging.error('Form not found during rebuild: %s',
case_transaction.form_id)
def apply_action_intents(self, primary_intent, deprecation_intent=None):
# for now we only allow commtrack actions to be processed this way so just assert that's the case
assert primary_intent.action_type == CASE_ACTION_COMMTRACK
transaction = CaseTransaction.ledger_transaction(self.case, primary_intent.form)
if deprecation_intent:
assert transaction.is_saved()
elif transaction not in self.case.get_tracked_models_to_create(CaseTransaction):
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
transaction.sync_log_id = None
self.case.track_create(transaction)
def create_form_for_test(domain, case_id=None, attachments=None, save=True, state=XFormInstanceSQL.NORMAL):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:param case_id: create case with ID if supplied
:param attachments: additional attachments dict
:param save: if False return the unsaved form
:return: form object
"""
from corehq.form_processor.utils import get_simple_form_xml
form_id = uuid4().hex
user_id = 'user1'
utcnow = datetime.utcnow()
form_xml = get_simple_form_xml(form_id, case_id)
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain,
state=state
)
attachments = attachments or {}
attachment_tuples = map(
lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items()
)
attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml'))
FormProcessorSQL.store_attachments(form, attachment_tuples)
cases = []
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
case.track_create(CaseTransaction.form_transaction(case, form))
cases = [case]
if save:
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return form
def _rebuild_case_from_transactions(case, detail, updated_xforms=None):
transactions = get_case_transactions(case.case_id, updated_xforms=updated_xforms)
strategy = SqlCaseUpdateStrategy(case)
rebuild_transaction = CaseTransaction.rebuild_transaction(case, detail)
unarchived_form_id = None
if detail.type == CaseTransaction.TYPE_REBUILD_FORM_ARCHIVED and not detail.archived:
# we're rebuilding because a form was un-archived
unarchived_form_id = detail.form_id
strategy.rebuild_from_transactions(
transactions, rebuild_transaction, unarchived_form_id=unarchived_form_id
)
return case
class SqlCaseUpdateStrategy(UpdateStrategy):
case_implementation_class = CommCareCase
def apply_action_intents(self, primary_intent, deprecation_intent=None):
# for now we only allow commtrack actions to be processed this way so just assert that's the case
if primary_intent:
if primary_intent.action_type != CASE_ACTION_COMMTRACK:
raise StockProcessingError(
'intent not of expected type: {}'.format(
primary_intent.action_type))
if not deprecation_intent:
# don't create / update transaction if we're processing an edited form
transaction = CaseTransaction.ledger_transaction(
self.case, primary_intent.form)
if transaction not in self.case.get_tracked_models_to_create(
CaseTransaction):
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
transaction.sync_log_id = None
self.case.track_create(transaction)
elif deprecation_intent:
# edited form no longer has a ledger block to update existing transaction
# to remove leger type
transaction = self.case.get_transaction_by_form_id(
deprecation_intent.form.orig_id)
if not transaction.is_saved():
raise StockProcessingError('Deprecated transaction not saved')
transaction.sync_log_id = None
transaction.type -= CaseTransaction.TYPE_LEDGER
self.case.track_update(transaction)
def update_from_case_update(self, case_update, xformdoc, other_forms=None):
self._apply_case_update(case_update, xformdoc)
self.add_transaction_for_form(self.case, case_update, xformdoc)
@staticmethod
def add_transaction_for_form(case, case_update, form):
types = [
CaseTransaction.type_from_action_type_slug(a.action_type_slug)
for a in case_update.actions
]
transaction = CaseTransaction.form_transaction(
case, form, case_update.guess_modified_on(), types)
if transaction not in case.get_tracked_models_to_create(
CaseTransaction):
case.track_create(transaction)
def _rebuild_case_from_transactions(case, detail, updated_xforms=None):
transactions = CaseAccessorSQL.get_case_transactions_by_case_id(
case,
updated_xforms=updated_xforms)
strategy = SqlCaseUpdateStrategy(case)
rebuild_transaction = CaseTransaction.rebuild_transaction(case, detail)
if updated_xforms:
rebuild_transaction.server_date = updated_xforms[0].edited_on
unarchived_form_id = None
if detail.type == CaseTransaction.TYPE_REBUILD_FORM_ARCHIVED and not detail.archived:
# we're rebuilding because a form was un-archived
unarchived_form_id = detail.form_id
strategy.rebuild_from_transactions(
transactions, rebuild_transaction, unarchived_form_id=unarchived_form_id
)
return case, rebuild_transaction
def reconcile_transactions(self):
transactions = self.case.transactions
sorted_transactions = sorted(
transactions,
key=_transaction_sort_key_function(self.case)
)
if sorted_transactions:
if not sorted_transactions[0].is_case_create:
error = "Case {0} first transaction not create transaction: {1}"
raise ReconciliationError(
error.format(self.case.case_id, sorted_transactions[0])
)
CaseAccessorSQL.fetch_case_transaction_forms(self.case, sorted_transactions)
rebuild_detail = RebuildWithReason(reason="client_date_reconciliation")
rebuild_transaction = CaseTransaction.rebuild_transaction(self.case, rebuild_detail)
self.rebuild_from_transactions(sorted_transactions, rebuild_transaction)
def create_case(case) -> CommCareCaseSQL:
form = XFormInstanceSQL(
form_id=uuid4().hex,
xmlns='http://commcarehq.org/formdesigner/form-processor',
received_on=case.server_modified_on,
user_id=case.owner_id,
domain=case.domain,
)
transaction = CaseTransaction(
type=CaseTransaction.TYPE_FORM,
form_id=form.form_id,
case=case,
server_date=case.server_modified_on,
)
with patch.object(FormProcessorSQL, "publish_changes_to_kafka"):
case.track_create(transaction)
processed_forms = ProcessedForms(form, [])
FormProcessorSQL.save_processed_models(processed_forms, [case])
def get_case(self, case_json=None):
utcnow = datetime.utcnow()
case_json = self.props if case_json is None else case_json
intent_case = CommCareCase(
domain=SOURCE_DOMAIN,
type=self.CASE_TYPE,
case_json=case_json,
case_id=uuid.uuid4().hex,
user_id="local_user1",
opened_on=utcnow,
modified_on=utcnow,
)
intent_case.track_create(CaseTransaction(form_id="form123", type=CaseTransaction.TYPE_FORM))
def _mock_subcases(*args, **kwargs):
return self.subcases
intent_case.get_subcases = _mock_subcases
return intent_case
class TestCaseESAccessors(BaseESAccessorsTest):
es_index_infos = [CASE_INDEX_INFO, USER_INDEX_INFO]
def setUp(self):
super(TestCaseESAccessors, self).setUp()
self.owner_id = 'batman'
self.user_id = 'robin'
self.case_type = 'heroes'
def _send_case_to_es(self,
domain=None,
owner_id=None,
user_id=None,
case_type=None,
opened_on=None,
closed_on=None,
modified_on=None):
case = CommCareCase(case_id=uuid.uuid4().hex,
domain=domain or self.domain,
owner_id=owner_id or self.owner_id,
modified_by=user_id or self.user_id,
type=case_type or self.case_type,
opened_on=opened_on or datetime.now(),
opened_by=user_id or self.user_id,
closed_on=closed_on,
modified_on=modified_on or datetime.now(),
closed_by=user_id or self.user_id,
server_modified_on=datetime.utcnow(),
closed=bool(closed_on))
case.track_create(
CaseTransaction(
type=CaseTransaction.TYPE_FORM,
form_id=uuid.uuid4().hex,
case=case,
server_date=opened_on,
))
def _create_case(domain=None, form_id=None, case_type=None, user_id=None, closed=False):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:return: case_id
"""
domain = domain or DOMAIN
form_id = form_id or uuid.uuid4().hex
case_id = uuid.uuid4().hex
user_id = user_id or 'user1'
utcnow = datetime.utcnow()
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain
)
cases = []
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type=case_type or '',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
closed=closed or False
)
case.track_create(CaseTransaction.form_transaction(case, form))
cases = [case]
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return CaseAccessorSQL.get_case(case_id)
def test_ignores_before_rebuild_transaction(self):
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-11"):
new_old_xform = self._create_form()
with freeze_time("2018-10-08"):
new_old_trans = self._create_case_transaction(case, new_old_xform)
with freeze_time("2018-10-11"):
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
self.assertFalse(case.check_transaction_order())
with freeze_time("2018-10-13"):
new_rebuild_xform = self._create_form()
rebuild_detail = RebuildWithReason(reason="shadow's golden coin")
rebuild_transaction = CaseTransaction.rebuild_transaction(case, rebuild_detail)
case.track_create(rebuild_transaction)
FormProcessorSQL.save_processed_models(ProcessedForms(new_rebuild_xform, []), [case])
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
def _create_case_transaction(self, case, form=None, submitted_on=None, action_types=None):
form = form or self._create_form()
submitted_on = submitted_on or datetime.utcnow()
return CaseTransaction.form_transaction(case, form, submitted_on, action_types)
