def _get_new_form_json(xml, xform_id):
form_json = convert_xform_to_json(xml)
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_json)
# this is actually in-place because of how jsonobject works
scrub_meta(XFormInstance.wrap({'form': form_json, '_id': xform_id}))
return form_json
def _migrate_form_and_associated_models(self,
couch_form,
form_is_processed=True):
"""
Copies `couch_form` into a new sql form
"""
if form_is_processed:
form_data = couch_form.form
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_data)
xmlns = form_data.get("@xmlns", "")
user_id = extract_meta_user_id(form_data)
else:
xmlns = couch_form.xmlns
user_id = couch_form.user_id
sql_form = XFormInstanceSQL(
form_id=couch_form.form_id,
domain=self.domain,
xmlns=xmlns,
user_id=user_id,
)
_copy_form_properties(sql_form, couch_form)
_migrate_form_attachments(sql_form, couch_form)
_migrate_form_operations(sql_form, couch_form)
if couch_form.doc_type != 'SubmissionErrorLog':
self._save_diffs(couch_form, sql_form)
case_stock_result = self._get_case_stock_result(
sql_form, couch_form) if form_is_processed else None
_save_migrated_models(sql_form, case_stock_result)
def _get_all_stock_report_helpers_from_form(xform):
"""
Given an instance of an AbstractXFormInstance, extract the ledger actions and convert
them to StockReportHelper objects.
"""
form_xml = xform.get_xml_element()
commtrack_node_names = ('{%s}balance' % COMMTRACK_REPORT_XMLNS,
'{%s}transfer' % COMMTRACK_REPORT_XMLNS)
def _extract_ledger_nodes_from_xml(node):
"""
Goes through a parsed XML document and recursively pulls out any ledger XML blocks.
"""
for child in node:
if child.tag in commtrack_node_names:
yield child
else:
for e in _extract_ledger_nodes_from_xml(child):
yield e
for elem in _extract_ledger_nodes_from_xml(form_xml):
report_type, ledger_json = convert_xml_to_json(elem, last_xmlns=COMMTRACK_REPORT_XMLNS)
# apply the same datetime & string conversions
# that would be applied to XFormInstance.form
adjust_datetimes(ledger_json)
ledger_json = XFormInstance({'form': ledger_json}).form
yield _ledger_json_to_stock_report_helper(xform, report_type, ledger_json)
def _get_new_form_json(xml, xform_id):
form_json = convert_xform_to_json(xml)
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_json)
# this is actually in-place because of how jsonobject works
scrub_meta(XFormInstance.wrap({'form': form_json, '_id': xform_id}))
return form_json
def get_all_stock_report_helpers_from_form(xform):
"""
Given an instance of an AbstractXFormInstance, extract the ledger actions and convert
them to StockReportHelper objects.
"""
form_xml = xform.get_xml_element()
commtrack_node_names = ('{%s}balance' % COMMTRACK_REPORT_XMLNS,
'{%s}transfer' % COMMTRACK_REPORT_XMLNS)
def _extract_ledger_nodes_from_xml(node):
"""
Goes through a parsed XML document and recursively pulls out any ledger XML blocks.
"""
for child in node:
if child.tag in commtrack_node_names:
yield child
else:
for e in _extract_ledger_nodes_from_xml(child):
yield e
for elem in _extract_ledger_nodes_from_xml(form_xml):
report_type, ledger_json = convert_xml_to_json(
elem, last_xmlns=COMMTRACK_REPORT_XMLNS)
# apply the same datetime & string conversions
# that would be applied to XFormInstance.form
adjust_datetimes(ledger_json)
ledger_json = XFormInstance({'form': ledger_json}).form
yield _ledger_json_to_stock_report_helper(xform, report_type,
ledger_json)
def test_strip_tz(self):
if phone_timezones_should_be_processed():
self.assertEqual(
adjust_datetimes({'datetime': '2013-03-09T06:30:09.007+03'}),
{'datetime': '2013-03-09T03:30:09.007000Z'})
else:
self.assertEqual(
adjust_datetimes({'datetime': '2013-03-09T06:30:09.007+03'}),
{'datetime': '2013-03-09T06:30:09.007000Z'})
def _migrate_form_and_associated_models(self,
couch_form,
form_is_processed=True):
"""
Copies `couch_form` into a new sql form
"""
sql_form = None
try:
if form_is_processed:
form_data = couch_form.form
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_data)
xmlns = form_data.get("@xmlns", "")
user_id = extract_meta_user_id(form_data)
else:
xmlns = couch_form.xmlns
user_id = couch_form.user_id
if xmlns == SYSTEM_ACTION_XMLNS:
for form_id, case_ids in do_system_action(couch_form):
self.case_diff_queue.update(case_ids, form_id)
sql_form = XFormInstanceSQL(
form_id=couch_form.form_id,
domain=self.domain,
xmlns=xmlns,
user_id=user_id,
)
_copy_form_properties(sql_form, couch_form)
_migrate_form_attachments(sql_form, couch_form)
_migrate_form_operations(sql_form, couch_form)
case_stock_result = (self._get_case_stock_result(
sql_form, couch_form) if form_is_processed else None)
_save_migrated_models(sql_form, case_stock_result)
except IntegrityError as err:
exc_info = sys.exc_info()
try:
sql_form = FormAccessorSQL.get_form(couch_form.form_id)
except XFormNotFound:
proc = "" if form_is_processed else " unprocessed"
log.error("Error migrating%s form %s",
proc,
couch_form.form_id,
exc_info=exc_info)
if self.stop_on_error:
raise err from None
except Exception as err:
proc = "" if form_is_processed else " unprocessed"
log.exception("Error migrating%s form %s", proc,
couch_form.form_id)
try:
sql_form = FormAccessorSQL.get_form(couch_form.form_id)
except XFormNotFound:
pass
if self.stop_on_error:
raise err from None
finally:
if couch_form.doc_type != 'SubmissionErrorLog':
self._save_diffs(couch_form, sql_form)
def test_strip_tz(self):
if phone_timezones_should_be_processed():
self.assertEqual(
adjust_datetimes({'datetime': '2013-03-09T06:30:09.007+03'}),
{'datetime': '2013-03-09T03:30:09.007000Z'}
)
else:
self.assertEqual(
adjust_datetimes({'datetime': '2013-03-09T06:30:09.007+03'}),
{'datetime': '2013-03-09T06:30:09.007000Z'}
)
def _create_new_xform(domain,
instance_xml,
attachments=None,
auth_context=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
:param domain:
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
interface = FormProcessorInterface(domain)
assert attachments is not None
form_data = convert_xform_to_json(instance_xml)
if not form_data.get('@xmlns'):
raise MissingXMLNSError("Form is missing a required field: XMLNS")
adjust_datetimes(form_data)
xform = interface.new_xform(form_data)
xform.domain = domain
xform.auth_context = auth_context
# Maps all attachments to uniform format and adds form.xml to list before storing
attachments = map(
lambda a: Attachment(
name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items())
attachments.append(
Attachment(name='form.xml',
raw_content=instance_xml,
content_type='text/xml'))
interface.store_attachments(xform, attachments)
result = LockedFormProcessingResult(xform)
with ReleaseOnError(result.lock):
if interface.is_duplicate(xform.form_id):
raise DuplicateError(xform)
return result
def _migrate_form(domain, couch_form):
"""
This copies the couch form into a new sql form but does not save it.
See form_processor.parsers.form._create_new_xform
and SubmissionPost._set_submission_properties for what this should do.
"""
interface = FormProcessorInterface(domain)
form_data = couch_form.form
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_data)
sql_form = interface.new_xform(form_data)
sql_form.form_id = couch_form.form_id # some legacy forms don't have ID's so are assigned random ones
if sql_form.xmlns is None:
sql_form.xmlns = ''
return _copy_form_properties(domain, sql_form, couch_form)
def _migrate_form(domain, couch_form):
"""
This copies the couch form into a new sql form but does not save it.
See form_processor.parsers.form._create_new_xform
and SubmissionPost._set_submission_properties for what this should do.
"""
interface = FormProcessorInterface(domain)
form_data = couch_form.form
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_data)
sql_form = interface.new_xform(form_data)
sql_form.form_id = couch_form.form_id # some legacy forms don't have ID's so are assigned random ones
if sql_form.xmlns is None:
sql_form.xmlns = ''
return _copy_form_properties(domain, sql_form, couch_form)
def _create_new_xform(domain, instance_xml, attachments=None):
"""
create but do not save an XFormInstance from an xform payload (xml_string)
optionally set the doc _id to a predefined value (_id)
return doc _id of the created doc
`process` is transformation to apply to the form right before saving
This is to avoid having to save multiple times
If xml_string is bad xml
- raise couchforms.XMLSyntaxError
:param domain:
"""
from corehq.form_processor.interfaces.processor import FormProcessorInterface
interface = FormProcessorInterface(domain)
assert attachments is not None
form_data = convert_xform_to_json(instance_xml)
if not form_data.get('@xmlns'):
raise MissingXMLNSError("Form is missing a required field: XMLNS")
adjust_datetimes(form_data)
xform = interface.new_xform(form_data)
xform.domain = domain
# Maps all attachments to uniform format and adds form.xml to list before storing
attachments = map(
lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items()
)
attachments.append(Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml'))
interface.store_attachments(xform, attachments)
result = LockedFormProcessingResult(xform)
with ReleaseOnError(result.lock):
if interface.is_duplicate(xform.form_id):
raise DuplicateError(xform)
return result
def unpack_commtrack(xform):
form_xml = xform.get_xml_element()
commtrack_node_names = ('{%s}balance' % COMMTRACK_REPORT_XMLNS,
'{%s}transfer' % COMMTRACK_REPORT_XMLNS)
def commtrack_nodes(node):
for child in node:
if child.tag in commtrack_node_names:
yield child
else:
for e in commtrack_nodes(child):
yield e
for elem in commtrack_nodes(form_xml):
report_type, ledger_json = convert_xml_to_json(
elem, last_xmlns=COMMTRACK_REPORT_XMLNS)
# apply the same datetime & string conversions
# that would be applied to XFormInstance.form
adjust_datetimes(ledger_json)
ledger_json = XFormInstance({'form': ledger_json}).form
yield ledger_json_to_stock_report_helper(
xform, report_type, ledger_json)
def test_date_no_change(self):
self.assertEqual(adjust_datetimes({'date': '2015-04-03'}),
{'date': '2015-04-03'})
def test_match_no_parse(self):
fake_datetime = '2015-07-14 2015-06-07 '
self.assertEqual(adjust_datetimes({'fake_datetime': fake_datetime}),
{'fake_datetime': fake_datetime})
def test_no_tz(self):
self.assertEqual(
adjust_datetimes({'datetime': '2013-03-09T06:30:09.007'}),
{'datetime': '2013-03-09T06:30:09.007000Z'})
def test_date_no_change(self):
self.assertEqual(adjust_datetimes({'date': '2015-04-03'}),
{'date': '2015-04-03'})
def test_match_no_parse(self):
fake_datetime = '2015-07-14 2015-06-07 '
self.assertEqual(
adjust_datetimes({'fake_datetime': fake_datetime}),
{'fake_datetime': fake_datetime}
)
def test_no_tz(self):
self.assertEqual(
adjust_datetimes({'datetime': '2013-03-09T06:30:09.007'}),
{'datetime': '2013-03-09T06:30:09.007000Z'}
)
