def test_extract_form_attachment_info(self):
image_1_name = "1234.jpg"
image_2_name = "5678.jpg"
form = {
"name": "foo",
"color": "bar",
"image_1": image_1_name,
"my_group": {
"image_2": image_2_name
}
}
with mock.patch.object(XFormInstanceSQL, 'form_data') as form_data_mock:
form_data_mock.__get__ = mock.MagicMock(return_value=form)
couch_xform = XFormInstance(
received_on=datetime.datetime.now(),
form=form,
_attachments={
image_1_name: {
"content_type": "image/jpeg",
"length": 1024,
},
image_2_name: {
"content_type": "image/jpeg",
"length": 2048,
},
"form.xml": {
"content_type": "text/xml",
"length": 2048,
}
}
)
sql_xform = XFormInstanceSQL(
received_on=datetime.datetime.now(),
)
sql_xform.unsaved_attachments = [
XFormAttachmentSQL(
name=image_1_name,
content_type="image/jpeg",
content_length=1024,
),
XFormAttachmentSQL(
name=image_2_name,
content_type="image/jpeg",
content_length=1024,
),
XFormAttachmentSQL(
name="form.xml",
content_type="text/xml",
content_length=1024,
),
]
for xform in (couch_xform, sql_xform):
form_info = _extract_form_attachment_info(xform, {"my_group-image_2", "image_1"})
attachments = {a['name']: a for a in form_info['attachments']}
self.assertTrue(image_1_name in attachments)
self.assertTrue(image_2_name in attachments)
self.assertEqual(attachments[image_1_name]['question_id'], "image_1")
self.assertEqual(attachments[image_2_name]['question_id'], "my_group-image_2")
def test_extract_form_attachment_info(self):
image_1_name = "1234.jpg"
image_2_name = "5678.jpg"
form = {
"name": "foo",
"color": "bar",
"image_1": image_1_name,
"my_group": {
"image_2": image_2_name
}
}
attachments = {
image_1_name: {
"content_type": "image/jpeg",
"content_length": 1024,
},
image_2_name: {
"content_type": "image/jpeg",
"content_length": 2048,
},
"form.xml": {
"content_type": "text/xml",
"content_length": 2048,
}
}
with mock.patch.object(XFormInstanceSQL, 'form_data') as form_data_mock:
form_data_mock.__get__ = mock.MagicMock(return_value=form)
couch_xform = XFormInstance(
received_on=datetime.datetime.now(),
form=form,
)
for name, meta in attachments.items():
couch_xform.deferred_put_attachment("content", name, **meta)
sql_xform = XFormInstanceSQL(received_on=datetime.datetime.now())
sql_xform.attachments_list = [BlobMeta(name=name, **meta)
for name, meta in attachments.items()]
for xform in (couch_xform, sql_xform):
print(type(xform).__name__)
form_info = _extract_form_attachment_info(xform, {"my_group-image_2", "image_1"})
attachments = {a['name']: a for a in form_info['attachments']}
self.assertTrue(image_1_name in attachments)
self.assertTrue(image_2_name in attachments)
self.assertEqual(attachments[image_1_name]['question_id'], "image_1")
self.assertEqual(attachments[image_2_name]['question_id'], "my_group-image_2")
def new_xform(cls, form_data):
form_id = extract_meta_instance_id(form_data) or str(uuid.uuid4())
return XFormInstanceSQL(
# other properties can be set post-wrap
form_id=form_id,
xmlns=form_data.get('@xmlns'),
received_on=datetime.datetime.utcnow(),
user_id=extract_meta_user_id(form_data),
)
def test_update_from_openmrs(self):
"""
payloads from OpenMRS should not be forwarded back to OpenMRS
"""
payload = XFormInstanceSQL(
domain=DOMAIN,
xmlns=XMLNS_OPENMRS,
)
repeater = OpenmrsRepeater()
self.assertFalse(repeater.allowed_to_forward(payload))
def _make_form_instance(cls, form_id):
return XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=datetime.utcnow(),
user_id='a-user',
domain=cls.domain,
state=XFormInstanceSQL.NORMAL,
)
def _migrate_form_and_associated_models(self,
couch_form,
form_is_processed=True):
"""
Copies `couch_form` into a new sql form
"""
sql_form = None
try:
if form_is_processed:
form_data = couch_form.form
with force_phone_timezones_should_be_processed():
adjust_datetimes(form_data)
xmlns = form_data.get("@xmlns", "")
user_id = extract_meta_user_id(form_data)
else:
xmlns = couch_form.xmlns
user_id = couch_form.user_id
if xmlns == SYSTEM_ACTION_XMLNS:
for form_id, case_ids in do_system_action(couch_form):
self.case_diff_queue.update(case_ids, form_id)
sql_form = XFormInstanceSQL(
form_id=couch_form.form_id,
domain=self.domain,
xmlns=xmlns,
user_id=user_id,
)
_copy_form_properties(sql_form, couch_form)
_migrate_form_attachments(sql_form, couch_form)
_migrate_form_operations(sql_form, couch_form)
case_stock_result = (self._get_case_stock_result(
sql_form, couch_form) if form_is_processed else None)
_save_migrated_models(sql_form, case_stock_result)
except IntegrityError:
exc_info = sys.exc_info()
try:
sql_form = FormAccessorSQL.get_form(couch_form.form_id)
except XFormNotFound:
sql_form = None
proc = "" if form_is_processed else " unprocessed"
log.error("Error migrating%s form %s",
proc,
couch_form.form_id,
exc_info=exc_info)
except Exception:
proc = "" if form_is_processed else " unprocessed"
log.exception("Error migrating%s form %s", proc,
couch_form.form_id)
try:
sql_form = FormAccessorSQL.get_form(couch_form.form_id)
except XFormNotFound:
sql_form = None
finally:
if couch_form.doc_type != 'SubmissionErrorLog':
self._save_diffs(couch_form, sql_form)
def _migrate_unprocessed_form(self, couch_form_json):
from corehq.apps.tzmigration.timezonemigration import json_diff
self.log_debug('Processing doc: {}({})'.format(couch_form_json['doc_type'], couch_form_json['_id']))
couch_form = _wrap_form(couch_form_json)
sql_form = XFormInstanceSQL(
form_id=couch_form.form_id,
xmlns=couch_form.xmlns,
user_id=couch_form.user_id,
)
_copy_form_properties(self.domain, sql_form, couch_form)
_migrate_form_attachments(sql_form, couch_form)
_migrate_form_operations(sql_form, couch_form)
if couch_form.doc_type != 'SubmissionErrorLog':
diffs = json_diff(couch_form.to_json(), sql_form.to_json(), track_list_indices=False)
self.diff_db.add_diffs(
couch_form.doc_type, couch_form.form_id,
filter_form_diffs(couch_form.doc_type, diffs)
)
_save_migrated_models(sql_form)
def process_patch(patch_form):
sql_form = XFormInstanceSQL(
form_id=patch_form.form_id,
domain=patch_form.domain,
xmlns=patch_form.xmlns,
user_id=patch_form.user_id,
received_on=patch_form.received_on,
)
add_form_xml(sql_form, patch_form)
add_patch_operation(sql_form)
case_stock_result = get_case_and_ledger_updates(patch_form.domain, sql_form)
save_sql_form(sql_form, case_stock_result)
def create_form_for_test(domain,
case_id=None,
attachments=None,
save=True,
state=XFormInstanceSQL.NORMAL):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:param case_id: create case with ID if supplied
:param attachments: additional attachments dict
:param save: if False return the unsaved form
:return: form object
"""
from corehq.form_processor.utils import get_simple_form_xml
form_id = uuid4().hex
user_id = 'user1'
utcnow = datetime.utcnow()
form_xml = get_simple_form_xml(form_id, case_id)
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain,
state=state)
attachments = attachments or {}
attachment_tuples = map(
lambda a: Attachment(
name=a[0], raw_content=a[1], content_type=a[1].content_type),
attachments.items())
attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml'))
FormProcessorSQL.store_attachments(form, attachment_tuples)
cases = []
if case_id:
case = CommCareCaseSQL(
case_id=case_id,
domain=domain,
type='',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
)
case.track_create(CaseTransaction.form_transaction(case, form))
cases = [case]
def _drop_sql_form_ids(couch_ids, domain):
from corehq.sql_db.util import split_list_by_db_partition
get_missing_forms = """
SELECT couch.form_id
FROM (SELECT unnest(%s) AS form_id) AS couch
LEFT JOIN form_processor_xforminstancesql sql USING (form_id)
WHERE sql.form_id IS NULL
"""
for dbname, form_ids in split_list_by_db_partition(couch_ids):
with XFormInstanceSQL.get_cursor_for_partition_db(
dbname, readonly=True) as cursor:
cursor.execute(get_missing_forms, [form_ids])
yield from (form_id for form_id, in cursor.fetchall())
def submission_error_form_instance(cls, domain, instance, message):
xform = XFormInstanceSQL(
domain=domain,
form_id=uuid.uuid4().hex,
received_on=datetime.datetime.utcnow(),
problem=message,
state=XFormInstanceSQL.SUBMISSION_ERROR_LOG
)
cls.store_attachments(xform, [Attachment(
name=ATTACHMENT_NAME,
raw_content=instance,
content_type='text/xml',
)])
return xform
def _migrate_unprocessed_form(self, couch_form_json):
self.log_debug('Processing doc: {}({})'.format(couch_form_json['doc_type'], couch_form_json['_id']))
couch_form = _wrap_form(couch_form_json)
sql_form = XFormInstanceSQL(
form_id=couch_form.form_id,
xmlns=couch_form.xmlns,
user_id=couch_form.user_id,
)
_copy_form_properties(self.domain, sql_form, couch_form)
_migrate_form_attachments(sql_form, couch_form)
_migrate_form_operations(sql_form, couch_form)
if couch_form.doc_type != 'SubmissionErrorLog':
self._save_diffs(couch_form, sql_form)
_save_migrated_models(sql_form)
def create_case(case) -> CommCareCaseSQL:
form = XFormInstanceSQL(
form_id=uuid4().hex,
xmlns='http://commcarehq.org/formdesigner/form-processor',
received_on=case.server_modified_on,
user_id=case.owner_id,
domain=case.domain,
)
transaction = CaseTransaction(
type=CaseTransaction.TYPE_FORM,
form_id=form.form_id,
case=case,
server_date=case.server_modified_on,
)
with patch.object(FormProcessorSQL, "publish_changes_to_kafka"):
case.track_create(transaction)
processed_forms = ProcessedForms(form, [])
FormProcessorSQL.save_processed_models(processed_forms, [case])
def _migrate_unprocessed_form(self, couch_form_json):
log.debug('Processing doc: {}({})'.format(couch_form_json['doc_type'], couch_form_json['_id']))
try:
couch_form = _wrap_form(couch_form_json)
sql_form = XFormInstanceSQL(
form_id=couch_form.form_id,
xmlns=couch_form.xmlns,
user_id=couch_form.user_id,
)
_copy_form_properties(self.domain, sql_form, couch_form)
_migrate_form_attachments(sql_form, couch_form)
_migrate_form_operations(sql_form, couch_form)
if couch_form.doc_type != 'SubmissionErrorLog':
self._save_diffs(couch_form, sql_form)
_save_migrated_models(sql_form)
self.processed_docs += 1
self._log_unprocessed_forms_processed_count(throttled=True)
except Exception:
log.exception("Error migrating form %s", couch_form_json["_id"])
def get_databases():
sql_dbs = [
_SQLFormDb(XFormInstanceSQL._meta.db_table,
lambda id_: XFormInstanceSQL.get_obj_by_id(id_),
XFormInstanceSQL.__name__),
_SQLDb(
CommCareCaseSQL._meta.db_table,
lambda id_: CommCareCaseSQLRawDocSerializer(
CommCareCaseSQL.get_obj_by_id(id_)).data,
CommCareCaseSQL.__name__),
_SQLDb(SQLLocation._meta.db_table,
lambda id_: SQLLocation.objects.get(location_id=id_).to_json(),
SQLLocation.__name__),
]
all_dbs = OrderedDict()
couchdbs_by_name = couch_config.all_dbs_by_db_name
for dbname in sorted(couchdbs_by_name):
all_dbs[dbname] = _CouchDb(couchdbs_by_name[dbname])
for db in sql_dbs:
all_dbs[db.dbname] = db
return all_dbs
def _update_forms(db_name, form_ids):
with XFormInstanceSQL.get_cursor_for_partition_db(db_name) as cursor:
cursor.execute(
"""
WITH max_dates as (
SELECT form_id, max(modified_on) as modified_on FROM (
SELECT form_id,
CASE WHEN deleted_on is not NULL THEN deleted_on
WHEN edited_on is not NULL AND edited_on > received_on THEN edited_on
ELSE received_on END as modified_on
FROM form_processor_xforminstancesql
WHERE form_processor_xforminstancesql.form_id in %(form_ids)s
union
select form_id, max(date) as modified_on from form_processor_xformoperationsql
where form_processor_xformoperationsql.form_id in %(form_ids)s
group by form_id
) as d group by form_id
)
UPDATE form_processor_xforminstancesql SET server_modified_on = max_dates.modified_on
FROM max_dates
WHERE form_processor_xforminstancesql.form_id = max_dates.form_id
""", {'form_ids': form_ids})
def _create_case(domain=None,
form_id=None,
case_type=None,
user_id=None,
closed=False,
case_id=None):
"""
Create the models directly so that these tests aren't dependent on any
other apps. Not testing form processing here anyway.
:return: CommCareCaseSQL
"""
domain = domain or DOMAIN
form_id = form_id or uuid.uuid4().hex
case_id = case_id or uuid.uuid4().hex
user_id = user_id or 'user1'
utcnow = datetime.utcnow()
form = XFormInstanceSQL(
form_id=form_id,
xmlns='http://openrosa.org/formdesigner/form-processor',
received_on=utcnow,
user_id=user_id,
domain=domain)
case = CommCareCaseSQL(case_id=case_id,
domain=domain,
type=case_type or '',
owner_id=user_id,
opened_on=utcnow,
modified_on=utcnow,
modified_by=user_id,
server_modified_on=utcnow,
closed=closed or False)
case.track_create(CaseTransaction.form_transaction(case, form, utcnow))
cases = [case]
FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases)
return CaseAccessorSQL.get_case(case_id)
def drop_sql_ids(self, couch_ids):
"""Filter the given couch ids, removing ids that are in SQL"""
for dbname, form_ids in split_list_by_db_partition(couch_ids):
with XFormInstanceSQL.get_cursor_for_partition_db(dbname, readonly=True) as cursor:
cursor.execute(self.sql, [form_ids])
yield from (form_id for form_id, in cursor.fetchall())
def __init__(self, dbname, getter, doc_type):
self.dbname = dbname
self._getter = getter
self.doc_type = doc_type
def get(self, record_id):
try:
return self._getter(record_id)
except (XFormNotFound, CaseNotFound, ObjectDoesNotExist):
raise ResourceNotFound("missing")
_SQL_DBS = OrderedDict((db.dbname, db) for db in [
_Db(
XFormInstanceSQL._meta.db_table,
lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_)).data,
XFormInstanceSQL.__name__
),
_Db(
CommCareCaseSQL._meta.db_table,
lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_)).data,
CommCareCaseSQL.__name__
),
_Db(
SQLLocation._meta.db_table,
lambda id_: SQLLocation.objects.get(location_id=id_).to_json(),
SQLLocation.__name__
),
])
def get_missing_form_ids(db, db_form_ids):
with XFormInstanceSQL.get_cursor_for_partition_db(
db, readonly=True) as cursor:
cursor.execute(sql, [db_form_ids])
return [r[0] for r in cursor.fetchall()]
def __init__(self, dbname, getter, doc_type):
self.dbname = dbname
self._getter = getter
self.doc_type = doc_type
def get(self, record_id):
try:
return self._getter(record_id)
except (XFormNotFound, CaseNotFound, ObjectDoesNotExist):
raise ResourceNotFound("missing")
_SQL_DBS = OrderedDict((db.dbname, db) for db in [
_Db(
XFormInstanceSQL._meta.db_table, lambda id_:
XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_)
).data, XFormInstanceSQL.__name__),
_Db(
CommCareCaseSQL._meta.db_table, lambda id_:
CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_)
).data, CommCareCaseSQL.__name__),
_Db(SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get(
location_id=id_).to_json(), SQLLocation.__name__),
])
def get_db_from_db_name(db_name):
if db_name in _SQL_DBS:
return _SQL_DBS[db_name]
elif db_name == couch_config.get_db(None).dbname: # primary db
return couch_config.get_db(None)
def __init__(self, dbname, getter, doc_type):
self.dbname = dbname
self._getter = getter
self.doc_type = doc_type
def get(self, record_id):
try:
return self._getter(record_id)
except (XFormNotFound, CaseNotFound, ObjectDoesNotExist):
raise ResourceNotFound("missing")
_SQL_DBS = OrderedDict((db.dbname, db) for db in [
_Db(
XFormInstanceSQL._meta.db_table,
lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_)).data,
XFormInstanceSQL.__name__
),
_Db(
CommCareCaseSQL._meta.db_table,
lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_)).data,
CommCareCaseSQL.__name__
),
_Db(
SQLLocation._meta.db_table,
lambda id_: SQLLocation.objects.get(location_id=id_).to_json(),
SQLLocation.__name__
),
])