def _create_models_for_stock_report_helper(self, form,
stock_report_helper):
processing_result = StockProcessingResult(
form, stock_report_helpers=[stock_report_helper])
processing_result.populate_models()
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save)
def _create_ledger(domain,
entry_id,
balance,
case_id=None,
section_id='stock'):
user_id = 'user1'
utcnow = datetime.utcnow()
case_id = case_id or uuid.uuid4().hex
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
CaseAccessorSQL.save_case(case)
ledger = LedgerValue(domain=domain,
case_id=case_id,
section_id=section_id,
entry_id=entry_id,
balance=balance,
last_modified=utcnow)
LedgerAccessorSQL.save_ledger_values([ledger])
return ledger
def _create_ledger(domain, entry_id, balance, case_id=None, section_id='stock'):
user_id = 'user1'
utcnow = datetime.utcnow()
case_id = case_id or uuid.uuid4().hex
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
CaseAccessorSQL.save_case(case)
ledger = LedgerValue(
domain=domain,
case_id=case_id,
section_id=section_id,
entry_id=entry_id,
balance=balance,
last_modified=utcnow
)
LedgerAccessorSQL.save_ledger_values([ledger])
return ledger
def save_processed_models(cls, processed_forms, cases=None, stock_result=None, publish_to_kafka=True):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db for ledger_value in stock_result.models_to_save
}
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, stock_result)
if publish_to_kafka:
cls._publish_changes(processed_forms, cases, stock_result)
def _create_models_for_stock_report_helper(self, form, stock_report_helper):
processing_result = StockProcessingResult(form, stock_report_helpers=[stock_report_helper])
processing_result.populate_models()
if should_use_sql_backend(self.domain):
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save)
else:
processing_result.commit()
def _create_models_for_stock_report_helper(self, form, stock_report_helper):
processing_result = StockProcessingResult(form, stock_report_helpers=[stock_report_helper])
processing_result.populate_models()
if should_use_sql_backend(self.domain_name):
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save)
else:
processing_result.commit()
def hard_rebuild_ledgers(case_id, section_id=None, entry_id=None):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id, section_id, entry_id)
if not transactions:
LedgerAccessorSQL.delete_ledger_values(case_id, section_id, entry_id)
return
ledger_value = LedgerAccessorSQL.get_ledger_value(case_id, section_id, entry_id)
ledger_value = LedgerProcessorSQL._rebuild_ledger_value_from_transactions(ledger_value, transactions)
LedgerAccessorSQL.save_ledger_values([ledger_value])
publish_ledger_v2_saved(ledger_value)
def _get_daily_consumption_for_ledger(ledger):
from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json
daily_consumption = get_consumption_for_ledger_json(ledger)
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
ledger_value = LedgerAccessorSQL.get_ledger_value(ledger['case_id'],
ledger['section_id'],
ledger['entry_id'])
ledger_value.daily_consumption = daily_consumption
LedgerAccessorSQL.save_ledger_values([ledger_value])
return daily_consumption
def hard_rebuild_ledgers(domain, case_id, section_id, entry_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id, section_id, entry_id)
if not transactions:
LedgerAccessorSQL.delete_ledger_values(case_id, section_id, entry_id)
publish_ledger_v2_deleted(domain, case_id, section_id, entry_id)
return
ledger_value = LedgerAccessorSQL.get_ledger_value(case_id, section_id, entry_id)
ledger_value = LedgerProcessorSQL._rebuild_ledger_value_from_transactions(ledger_value, transactions)
LedgerAccessorSQL.save_ledger_values([ledger_value])
publish_ledger_v2_saved(ledger_value)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db for ledger_value in stock_result.models_to_save
}
all_models = filter(None, chain(
processed_forms,
cases or [],
stock_result.models_to_save if stock_result else [],
))
try:
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, stock_result)
if cases:
sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
if sort_submissions:
for case in cases:
if SqlCaseUpdateStrategy(case).reconcile_transactions_if_necessary():
CaseAccessorSQL.save_case(case)
except DatabaseError:
for model in all_models:
setattr(model, model._meta.pk.attname, None)
for tracked in model.create_models:
setattr(tracked, tracked._meta.pk.attname, None)
raise
try:
cls.publish_changes_to_kafka(processed_forms, cases, stock_result)
except Exception as e:
raise KafkaPublishingError(e)
def save_processed_models(cls,
processed_forms,
cases=None,
stock_result=None):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db
for ledger_value in stock_result.models_to_save
}
all_models = filter(
None,
chain(
processed_forms,
cases or [],
stock_result.models_to_save if stock_result else [],
))
try:
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
XFormInstance.objects.update_form(
processed_forms.deprecated, publish_changes=False)
XFormInstance.objects.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
case.save(with_tracked_models=True)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(
ledgers_to_save, stock_result)
if cases:
sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
if sort_submissions:
for case in cases:
if SqlCaseUpdateStrategy(
case).reconcile_transactions_if_necessary():
case.save(with_tracked_models=True)
def process_form_unarchived(self, form):
from corehq.apps.commtrack.processing import process_stock
result = process_stock([form])
result.populate_models()
LedgerAccessorSQL.save_ledger_values(result.models_to_save)
for ledger_value in result.models_to_save:
publish_ledger_v2_saved(ledger_value)
refs_to_rebuild = {ledger_value.ledger_reference for ledger_value in result.models_to_save}
for ref in refs_to_rebuild:
self.rebuild_ledger_state(**ref._asdict())
result.finalize()
def _get_daily_consumption_for_ledger(ledger):
from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json
daily_consumption = get_consumption_for_ledger_json(ledger)
if should_use_sql_backend(ledger['domain']):
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
ledger_value = LedgerAccessorSQL.get_ledger_value(
ledger['case_id'], ledger['section_id'], ledger['entry_id'])
ledger_value.daily_consumption = daily_consumption
LedgerAccessorSQL.save_ledger_values([ledger_value])
else:
from corehq.apps.commtrack.models import StockState
StockState.objects.filter(pk=ledger['_id']).update(
daily_consumption=daily_consumption)
return daily_consumption
def process_form_unarchived(self, form):
from corehq.apps.commtrack.processing import process_stock
result = process_stock([form])
result.populate_models()
LedgerAccessorSQL.save_ledger_values(result.models_to_save)
for ledger_value in result.models_to_save:
publish_ledger_v2_saved(ledger_value)
refs_to_rebuild = {
ledger_value.ledger_reference for ledger_value in result.models_to_save
}
for ref in refs_to_rebuild:
self.rebuild_ledger_state(**ref._asdict())
result.finalize()
def _get_daily_consumption_for_ledger(ledger):
from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json
daily_consumption = get_consumption_for_ledger_json(ledger)
if should_use_sql_backend(ledger['domain']):
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
ledger_value = LedgerAccessorSQL.get_ledger_value(
ledger['case_id'], ledger['section_id'], ledger['entry_id']
)
ledger_value.daily_consumption = daily_consumption
LedgerAccessorSQL.save_ledger_values([ledger_value])
else:
from corehq.apps.commtrack.models import StockState
StockState.objects.filter(pk=ledger['_id']).update(daily_consumption=daily_consumption)
return daily_consumption
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
with transaction.atomic():
logging.debug('Beginning atomic commit\n')
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.save_deprecated_form(processed_forms.deprecated)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, processed_forms.deprecated)
cls._publish_changes(processed_forms, cases, stock_result)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None):
with transaction.atomic():
logging.debug('Beginning atomic commit\n')
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.save_deprecated_form(processed_forms.deprecated)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, processed_forms.deprecated)
cls._publish_changes(processed_forms, cases, stock_result)
def save_processed_models(cls, processed_forms, cases=None, stock_result=None, publish_to_kafka=True):
db_names = {processed_forms.submitted.db}
if processed_forms.deprecated:
db_names |= {processed_forms.deprecated.db}
if cases:
db_names |= {case.db for case in cases}
if stock_result:
db_names |= {
ledger_value.db for ledger_value in stock_result.models_to_save
}
with ExitStack() as stack:
for db_name in db_names:
stack.enter_context(transaction.atomic(db_name))
# Save deprecated form first to avoid ID conflicts
if processed_forms.deprecated:
FormAccessorSQL.update_form(processed_forms.deprecated, publish_changes=False)
FormAccessorSQL.save_new_form(processed_forms.submitted)
if cases:
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
ledgers_to_save = stock_result.models_to_save
LedgerAccessorSQL.save_ledger_values(ledgers_to_save, stock_result)
if cases:
sort_submissions = toggles.SORT_OUT_OF_ORDER_FORM_SUBMISSIONS_SQL.enabled(
processed_forms.submitted.domain, toggles.NAMESPACE_DOMAIN)
if sort_submissions:
for case in cases:
if SqlCaseUpdateStrategy(case).reconcile_transactions_if_necessary():
CaseAccessorSQL.save_case(case)
if publish_to_kafka:
try:
cls.publish_changes_to_kafka(processed_forms, cases, stock_result)
except Exception as e:
raise KafkaPublishingError(e)
def reprocess_form(form, save=True, lock_form=True):
if lock_form:
# track load if locking; otherise it will be tracked elsewhere
form_load_counter("reprocess_form", form.domain)()
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form],
load_src="reprocess_form",
)
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
error_message = '{}: {}'.format(type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {ledger.ledger_reference for ledger in ledgers if ledger.is_saved()}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL.publish_changes_to_kafka(ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id, detail, lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s', ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
else:
if save:
interface.processor.save_processed_models([form], cases, stock_result)
from casexml.apps.stock.models import StockTransaction
ledgers = [
model
for model in stock_result.models_to_save
if isinstance(model, StockTransaction)
]
for ledger in ledgers:
interface.ledger_processor.rebuild_ledger_state(**ledger.ledger_reference._asdict())
save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result)
return ReprocessingResult(form, cases, ledgers, None)
def reprocess_xform_error(form):
"""
Attempt to re-process an error form. This was created specifically to address
the issue of out of order forms and child cases (form creates child case before
parent case has been created).
See http://manage.dimagi.com/default.asp?250459
:param form_id: ID of the error form to process
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.submission_post import SubmissionPost
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL, FormAccessorSQL, LedgerAccessorSQL
from corehq.blobs.mixin import bulk_atomic_blobs
from couchforms.models import XFormInstance
from casexml.apps.case.signals import case_post_save
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
if not form:
raise Exception('Form with ID {} not found'.format(form.form_id))
if not form.is_error:
raise Exception('Form was not an error form: {}={}'.format(form.form_id, form.doc_type))
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
form.initial_processing_complete = True
form.problem = None
cache = FormProcessorInterface(form.domain).casedb_cache(
domain=form.domain, lock=True, deleted_ok=True, xforms=[form]
)
with cache as casedb:
case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb)
if case_stock_result:
stock_result = case_stock_result.stock_result
if stock_result:
assert stock_result.populated
cases = case_stock_result.case_models
if should_use_sql_backend(form.domain):
for case in cases:
CaseAccessorSQL.save_case(case)
if stock_result:
LedgerAccessorSQL.save_ledger_values(stock_result.models_to_save)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None),
cases,
stock_result
)
else:
with bulk_atomic_blobs([form] + cases):
XFormInstance.save(form) # use this save to that we don't overwrite the doc_type
XFormInstance.get_db().bulk_save(cases)
if stock_result:
stock_result.commit()
case_stock_result.stock_result.finalize()
case_stock_result.case_result.commit_dirtiness_flags()
for case in cases:
case_post_save.send(case.__class__, case=case)
return form
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
