def _perfom_post_save_actions(form, save=True):
interface = FormProcessorInterface(form.domain)
cache = interface.casedb_cache(
domain=form.domain, lock=False, deleted_ok=True, xforms=[form],
load_src="reprocess_form_post_save",
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
case_models = case_stock_result.case_models
if interface.use_sql_domain:
forms = ProcessedForms(form, None)
stock_result = case_stock_result.stock_result
try:
FormProcessorSQL.publish_changes_to_kafka(forms, case_models, stock_result)
except Exception:
error_message = "Error publishing to kafka"
return ReprocessingResult(form, None, None, error_message)
try:
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
except PostSaveError:
error_message = "Error performing post save operations"
return ReprocessingResult(form, None, None, error_message)
return ReprocessingResult(form, case_models, None, None)
def test_save_processed_models_deprecated(self):
existing_form, new_form = _simulate_form_edit()
FormProcessorSQL.save_processed_models(
ProcessedForms(new_form, existing_form))
self._validate_deprecation(existing_form, new_form)
def test_reconcile_transactions_within_fudge_factor(self, soft_assert_mock):
""" tests a transanction with an early client date and late server date """
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-11 06:00"):
new_old_xform = self._create_form()
with freeze_time("2018-10-10 18:00"):
new_old_trans = self._create_case_transaction(case, new_old_xform)
with freeze_time("2018-10-11 06:00"):
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
with freeze_time("2018-10-11"):
new_old_xform = self._create_form()
new_old_trans = self._create_case_transaction(case, new_old_xform)
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertTrue(update_strategy.reconcile_transactions_if_necessary())
self._check_for_reconciliation_error_soft_assert(soft_assert_mock)
CaseAccessorSQL.save_case(case)
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
self._check_for_reconciliation_error_soft_assert(soft_assert_mock)
def rebuild_cases(domain, cases, logger):
detail = RebuildWithReason(reason='undo UUID clash')
for case_id in cases:
try:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)
logger.info('Case %s rebuilt' % case_id)
except:
logger.error("Exception rebuilding case %s".format(case_id))
logger.exception("message")
def rebuild_cases(domain, cases, logger):
detail = RebuildWithReason(reason='undo UUID clash')
for case_id in cases:
try:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)
logger.info('Case %s rebuilt' % case_id)
except:
logger.error("Exception rebuilding case %s".format(case_id))
logger.exception("message")
def create_form_for_test(domain,
case_id=None,
attachments=None,
save=True,
state=XFormInstanceSQL.NORMAL,
received_on=None,
user_id='user1',
edited_on=None):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:param case_id: create case with ID if supplied
:param attachments: additional attachments dict
:param save: if False return the unsaved form
:return: form object
"""
from corehq.form_processor.utils import get_simple_form_xml
form_id = uuid4().hex
utcnow = received_on or datetime.utcnow()
form_xml = get_simple_form_xml(form_id, case_id)
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain,
state=state,
edited_on=edited_on,
)
attachments = attachments or {}
attachment_tuples = [
Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type)
for a in attachments.items()
]
attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml'))
FormProcessorSQL.store_attachments(form, attachment_tuples)
cases = []
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
case.track_create(CaseTransaction.form_transaction(case, form, utcnow))
cases = [case]
def create_form_for_test(domain, case_id=None, attachments=None, save=True, state=XFormInstanceSQL.NORMAL):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:param case_id: create case with ID if supplied
:param attachments: additional attachments dict
:param save: if False return the unsaved form
:return: form object
"""
from corehq.form_processor.utils import get_simple_form_xml
form_id = uuid4().hex
user_id = 'user1'
utcnow = datetime.utcnow()
form_xml = get_simple_form_xml(form_id, case_id)
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain,
state=state
)
attachments = attachments or {}
attachment_tuples = map(
lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items()
)
attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml'))
FormProcessorSQL.store_attachments(form, attachment_tuples)
cases = []
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
case.track_create(CaseTransaction.form_transaction(case, form))
cases = [case]
if save:
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return form
def test_reconcile_not_necessary(self):
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-11"):
new_old_xform = self._create_form()
new_old_trans = self._create_case_transaction(case, new_old_xform)
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
def test_first_transaction_not_create(self):
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-08"):
new_old_xform = self._create_form()
new_old_trans = self._create_case_transaction(case, new_old_xform)
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
self.assertTrue(case.check_transaction_order())
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertRaises(ReconciliationError, update_strategy.reconcile_transactions)
def rebuild_case(sql_case, detail=None):
"""Rebuild SQL case
This does not save the case. This function should be wrapped in a
case lock if the case will be saved afterward.
"""
if detail is None:
detail = RebuildWithReason(reason=COUCH_SQL_REBUILD_REASON)
new_case = FormProcessorSQL.hard_rebuild_case(
sql_case.domain, sql_case.case_id, detail, lock=False, save=False)
return new_case
def _create_case(self, case_type=None, user_id=None, case_id=None):
case_id = case_id or uuid.uuid4().hex
user_id = user_id or 'mr_wednesday'
utcnow = datetime.utcnow()
case = CommCareCaseSQL(
case_id=case_id,
domain=self.DOMAIN,
type=case_type or '',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=utcnow,
server_modified_on=utcnow
)
form = self._create_form(user_id, utcnow)
case.track_create(self._create_case_transaction(case, form, utcnow, action_types=[128]))
FormProcessorSQL.save_processed_models(ProcessedForms(form, []), [case])
return CaseAccessorSQL.get_case(case_id)
def _create_case(domain=None, form_id=None, case_type=None, user_id=None, closed=False):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:return: case_id
"""
domain = domain or DOMAIN
form_id = form_id or uuid.uuid4().hex
case_id = uuid.uuid4().hex
user_id = user_id or 'user1'
utcnow = datetime.utcnow()
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain
)
cases = []
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type=case_type or '',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
closed=closed or False
)
case.track_create(CaseTransaction.form_transaction(case, form))
cases = [case]
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return CaseAccessorSQL.get_case(case_id)
def _create_case(domain=None,
form_id=None,
case_type=None,
user_id=None,
closed=False,
case_id=None):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:return: CommCareCaseSQL
"""
domain = domain or DOMAIN
form_id = form_id or uuid.uuid4().hex
case_id = case_id or uuid.uuid4().hex
user_id = user_id or 'user1'
utcnow = datetime.utcnow()
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain)
case = CommCareCaseSQL(case_id=case_id,
domain=domain,
type=case_type or '',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
closed=closed or False)
case.track_create(CaseTransaction.form_transaction(case, form, utcnow))
cases = [case]
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return CaseAccessorSQL.get_case(case_id)
def _save_migrated_models(sql_form, case_stock_result):
"""
See SubmissionPost.save_processed_models for ~what this should do.
However, note that that function does some things that this one shouldn't,
e.g. process ownership cleanliness flags.
"""
forms_tuple = ProcessedForms(sql_form, None)
stock_result = case_stock_result.stock_result if case_stock_result else None
if stock_result:
assert stock_result.populated
return FormProcessorSQL.save_processed_models(
forms_tuple,
cases=case_stock_result.case_models if case_stock_result else None,
stock_result=stock_result,
publish_to_kafka=False)
def test_ignores_before_rebuild_transaction(self):
with freeze_time("2018-10-10"):
case = self._create_case()
with freeze_time("2018-10-11"):
new_old_xform = self._create_form()
with freeze_time("2018-10-08"):
new_old_trans = self._create_case_transaction(case, new_old_xform)
with freeze_time("2018-10-11"):
case.track_create(new_old_trans)
FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])
self.assertFalse(case.check_transaction_order())
with freeze_time("2018-10-13"):
new_rebuild_xform = self._create_form()
rebuild_detail = RebuildWithReason(reason="shadow's golden coin")
rebuild_transaction = CaseTransaction.rebuild_transaction(case, rebuild_detail)
case.track_create(rebuild_transaction)
FormProcessorSQL.save_processed_models(ProcessedForms(new_rebuild_xform, []), [case])
case = CaseAccessorSQL.get_case(case.case_id)
update_strategy = SqlCaseUpdateStrategy(case)
self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
def _save_migrated_models(sql_form, case_stock_result=None):
"""
See SubmissionPost.save_processed_models for ~what this should do.
However, note that that function does some things that this one shouldn't,
e.g. process ownership cleanliness flags.
"""
forms_tuple = ProcessedForms(sql_form, None)
stock_result = case_stock_result.stock_result if case_stock_result else None
if stock_result:
assert stock_result.populated
return FormProcessorSQL.save_processed_models(
forms_tuple,
cases=case_stock_result.case_models if case_stock_result else None,
stock_result=stock_result,
publish_to_kafka=False
)
def rebuild_cases(cases_to_rebuild_by_domain, logger):
detail = RebuildWithReason(reason='undo UUID clash')
for domain, case_ids in six.iteritems(cases_to_rebuild_by_domain):
for case_id in case_ids:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)
logger.log('Case %s rebuilt' % case_id)
def rebuild_cases(cases_to_rebuild_by_domain, logger):
detail = RebuildWithReason(reason='undo UUID clash')
for domain, case_ids in six.iteritems(cases_to_rebuild_by_domain):
for case_id in case_ids:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)
logger.log('Case %s rebuilt' % case_id)
def reprocess_form(form, save=True, lock_form=True):
if lock_form:
# track load if locking; otherise it will be tracked elsewhere
form_load_counter("reprocess_form", form.domain)()
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
load_src="reprocess_form",
)
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
error_message = '{}: {}'.format(type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id, detail, lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
else:
if save:
interface.processor.save_processed_models([form], cases, stock_result)
from casexml.apps.stock.models import StockTransaction
ledgers = [
model
for model in stock_result.models_to_save
if isinstance(model, StockTransaction)
]
for ledger in ledgers:
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
return ReprocessingResult(form, cases, ledgers, None)
def reprocess_xform_error(form):
"""
Attempt to re-process an error form. This was created specifically to address
the issue of out of order forms and child cases (form creates child case before
parent case has been created).
See http://manage.dimagi.com/default.asp?250459
:param form_id: ID of the error form to process
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.submission_post import SubmissionPost
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL, FormAccessorSQL, LedgerAccessorSQL
from corehq.blobs.mixin import bulk_atomic_blobs
from couchforms.models import XFormInstance
from casexml.apps.case.signals import case_post_save
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
if not form:
raise Exception('Form with ID {} not found'.format(form.form_id))
if not form.is_error:
raise Exception('Form was not an error form: {}={}'.format(form.form_id, form.doc_type))
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
form.initial_processing_complete = True
form.problem = None
cache = FormProcessorInterface(form.domain).casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form]
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
if case_stock_result:
stock_result = case_stock_result.stock_result
if stock_result:
assert stock_result.populated
cases = case_stock_result.case_models
if should_use_sql_backend(form.domain):
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
LedgerAccessorSQL.save_ledger_values(stock_result.models_to_save)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None),
cases,
stock_result
)
else:
with bulk_atomic_blobs([form] + cases):
XFormInstance.save(form) # use this save to that we don't overwrite the doc_type
XFormInstance.get_db().bulk_save(cases)
if stock_result:
stock_result.commit()
case_stock_result.stock_result.finalize()
case_stock_result.case_result.commit_dirtiness_flags()
for case in cases:
case_post_save.send(case.__class__, case=case)
return form
def test_save_processed_models_deprecated(self):
existing_form, new_form = _simulate_form_edit()
FormProcessorSQL.save_processed_models(ProcessedForms(new_form, existing_form))
self._validate_deprecation(existing_form, new_form)
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
def rebuild_case(sql_case, detail=None):
if detail is None:
detail = RebuildWithReason(reason=COUCH_SQL_REBUILD_REASON)
return FormProcessorSQL.hard_rebuild_case(sql_case.domain, sql_case.case_id, detail)
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type=case_type or '',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
closed=closed or False
)
case.track_create(CaseTransaction.form_transaction(case, form))
cases = [case]
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return CaseAccessorSQL.get_case(case_id)
def _create_case_with_index(referenced_case_id, identifier='parent', referenced_type='mother',
relationship_id=CommCareCaseIndexSQL.CHILD, case_is_deleted=False,
case_type='child'):
case = _create_case(case_type=case_type)
case.deleted = case_is_deleted
index = CommCareCaseIndexSQL(
case=case,
identifier=identifier,
referenced_type=referenced_type,
referenced_id=referenced_case_id,
relationship_id=relationship_id
def rebuild_cases(cases_to_rebuild_by_domain):
detail = RebuildWithReason(reason='undo UUID clash')
for domain, case_ids in cases_to_rebuild_by_domain.iteritems():
for case_id in case_ids:
FormProcessorSQL.hard_rebuild_case(domain, case_id, detail)
