def resave_form(domain, form):
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.change_publishers import publish_form_saved
from couchforms.models import XFormInstance
if should_use_sql_backend(domain):
publish_form_saved(form)
else:
XFormInstance.get_db().save_doc(form.to_json())
def resave_form(domain, form):
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.change_publishers import publish_form_saved
from couchforms.models import XFormInstance
if should_use_sql_backend(domain):
publish_form_saved(form)
else:
XFormInstance.get_db().save_doc(form.to_json())
def publish_changes_to_kafka(processed_forms, cases, stock_result):
publish_form_saved(processed_forms.submitted)
cases = cases or []
for case in cases:
publish_case_saved(case)
if stock_result:
for ledger in stock_result.models_to_save:
publish_ledger_v2_saved(ledger)
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
def publish_changes_to_kafka(processed_forms, cases, stock_result):
publish_form_saved(processed_forms.submitted)
cases = cases or []
for case in cases:
publish_case_saved(case)
if stock_result:
for ledger in stock_result.models_to_save:
publish_ledger_v2_saved(ledger)
def _publish_changes(processed_forms, cases, stock_result):
# todo: form deprecations?
publish_form_saved(processed_forms.submitted)
if processed_forms.submitted.is_duplicate:
# for duplicate forms, also publish changes for the original form since the fact that
# we're getting a duplicate indicates that we may not have fully processed/published it
# the first time
republish_all_changes_for_form(processed_forms.submitted.domain,
processed_forms.submitted.orig_id)
cases = cases or []
for case in cases:
publish_case_saved(case)
if stock_result:
for ledger in stock_result.models_to_save:
publish_ledger_v2_saved(ledger)
def _publish_changes(processed_forms, cases, stock_result):
# todo: form deprecations?
publish_form_saved(processed_forms.submitted)
if processed_forms.submitted.is_duplicate:
# for duplicate forms, also publish changes for the original form since the fact that
# we're getting a duplicate indicates that we may not have fully processd/published it
# the first time
republish_all_changes_for_form(
processed_forms.submitted.domain, processed_forms.submitted.orig_id)
cases = cases or []
for case in cases:
publish_case_saved(case)
if stock_result:
for ledger in stock_result.models_to_save:
publish_ledger_v2_saved(ledger)
def handle(self, form_ids_file, **options):
cases_rebuilt = 0
errored_form_ids = set()
with open(form_ids_file, 'r') as f:
lines = f.readlines()
form_ids = [l.strip() for l in lines]
for form_id in with_progress_bar(form_ids):
try:
form = get_form(form_id)
publish_form_saved(form)
cases_rebuilt += rebuild_case_changes(form)
except Exception:
errored_form_ids.add(form_id)
logger.info("Rebuilt {} cases from {} forms. {} errors".format(
cases_rebuilt, len(form_ids), len(errored_form_ids)))
if errored_form_ids:
logger.error("errors in forms:\n{}".format("\n".join(errored_form_ids)))
with open('form_rebuild_errors.txt', 'w+') as f:
print("\n".join(errored_form_ids), file=f)
class FormProcessorSQL(object):
@classmethod
def store_attachments(cls, xform, attachments):
xform_attachments = []
for attachment in attachments:
xform_attachment = XFormAttachmentSQL(
name=attachment.name,
attachment_id=uuid.uuid4(),
content_type=attachment.content_type,
)
xform_attachment.write_content(attachment.content)
if xform_attachment.is_image:
try:
img_size = Image.open(attachment.content_as_file()).size
xform_attachment.properties = dict(width=img_size[0],
height=img_size[1])
except IOError:
xform_attachment.content_type = 'application/octet-stream'
xform_attachments.append(xform_attachment)
xform.unsaved_attachments = xform_attachments
@classmethod
def new_xform(cls, form_data):
form_id = extract_meta_instance_id(form_data) or unicode(uuid.uuid4())
return XFormInstanceSQL(
# other properties can be set post-wrap
form_id=form_id,
xmlns=form_data.get('@xmlns'),
received_on=datetime.datetime.utcnow(),
user_id=extract_meta_user_id(form_data),
)
@classmethod
def is_duplicate(cls, xform_id, domain=None):
return FormAccessorSQL.form_exists(xform_id, domain=domain)
@classmethod
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
class FormProcessorSQL(object):
@classmethod
def store_attachments(cls, xform, attachments):
xform.attachments_list = attachments
@classmethod
def copy_attachments(cls, from_form, to_form):
to_form.copy_attachments(from_form)
@classmethod
def copy_form_operations(cls, from_form, to_form):
for op in from_form.history:
op.id = None
op.form = to_form
to_form.track_create(op)
@classmethod
def new_xform(cls, form_data):
form_id = extract_meta_instance_id(form_data) or str(uuid.uuid4())
return XFormInstanceSQL(
# other properties can be set post-wrap
form_id=form_id,
xmlns=form_data.get('@xmlns'),
received_on=datetime.datetime.utcnow(),
user_id=extract_meta_user_id(form_data),
)
@classmethod
def is_duplicate(cls, xform_id, domain=None):
return FormAccessorSQL.form_exists(xform_id, domain=domain)
@classmethod
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
def _reprocess_form(form, save=True):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
form.initial_processing_complete = True
form.problem = None
interface = FormProcessorInterface(form.domain)
accessors = FormAccessors(form.domain)
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases([form],
casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e:
error_message = '{}: {}'.format(type(e).__name__, unicode(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
accessors.update_form_problem_and_state(form)
return ReprocessingResult(form, [], [])
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes('unfiltered', cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases = _filter_already_processed_cases(form, cases)
cases_needing_rebuild = _get_case_ids_needing_rebuild(form, cases)
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
ledgers = _filter_already_processed_ledgers(
form, stock_result.models_to_save)
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
LedgerAccessorSQL.save_ledger_values(ledgers)
if save:
FormAccessorSQL.update_form_problem_and_state(form)
publish_form_saved(form)
_log_changes('filtered', cases, ledgers, [])
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
save and case_post_save.send(case.__class__, case=case)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild upated ledgers
interface.ledger_processor.hard_rebuild_ledgers(
**ledger.ledger_reference._asdict())
else:
def unarchive_form(form, user_id=None):
from corehq.form_processor.change_publishers import publish_form_saved
FormAccessorSQL._archive_unarchive_form(form, user_id, False)
form.state = XFormInstanceSQL.NORMAL
publish_form_saved(form)
def archive_form(form, user_id=None):
from corehq.form_processor.change_publishers import publish_form_saved
FormAccessorSQL._archive_unarchive_form(form, user_id, True)
form.state = XFormInstanceSQL.ARCHIVED
publish_form_saved(form)
def unarchive_form(form, user_id=None):
from corehq.form_processor.change_publishers import publish_form_saved
FormAccessorSQL._archive_unarchive_form(form, user_id, False)
form.state = XFormInstanceSQL.NORMAL
publish_form_saved(form)
def archive_form(form, user_id=None):
from corehq.form_processor.change_publishers import publish_form_saved
FormAccessorSQL._archive_unarchive_form(form, user_id, True)
form.state = XFormInstanceSQL.ARCHIVED
publish_form_saved(form)
def _publish_changes(processed_forms, cases):
# todo: form deprecations?
publish_form_saved(processed_forms.submitted)
cases = cases or []
for case in cases:
publish_case_saved(case)
def resave_form(domain, form):
from corehq.form_processor.change_publishers import publish_form_saved
publish_form_saved(form)
