def test_extract_form_attachment_info(self): image_1_name = "1234.jpg" image_2_name = "5678.jpg" form = { "name": "foo", "color": "bar", "image_1": image_1_name, "my_group": { "image_2": image_2_name } } with mock.patch.object(XFormInstanceSQL, 'form_data') as form_data_mock: form_data_mock.__get__ = mock.MagicMock(return_value=form) couch_xform = XFormInstance( received_on=datetime.datetime.now(), form=form, _attachments={ image_1_name: { "content_type": "image/jpeg", "length": 1024, }, image_2_name: { "content_type": "image/jpeg", "length": 2048, }, "form.xml": { "content_type": "text/xml", "length": 2048, } } ) sql_xform = XFormInstanceSQL( received_on=datetime.datetime.now(), ) sql_xform.unsaved_attachments = [ XFormAttachmentSQL( name=image_1_name, content_type="image/jpeg", content_length=1024, ), XFormAttachmentSQL( name=image_2_name, content_type="image/jpeg", content_length=1024, ), XFormAttachmentSQL( name="form.xml", content_type="text/xml", content_length=1024, ), ] for xform in (couch_xform, sql_xform): form_info = _extract_form_attachment_info(xform, {"my_group-image_2", "image_1"}) attachments = {a['name']: a for a in form_info['attachments']} self.assertTrue(image_1_name in attachments) self.assertTrue(image_2_name in attachments) self.assertEqual(attachments[image_1_name]['question_id'], "image_1") self.assertEqual(attachments[image_2_name]['question_id'], "my_group-image_2")
def test_extract_form_attachment_info(self): image_1_name = "1234.jpg" image_2_name = "5678.jpg" form = { "name": "foo", "color": "bar", "image_1": image_1_name, "my_group": { "image_2": image_2_name } } attachments = { image_1_name: { "content_type": "image/jpeg", "content_length": 1024, }, image_2_name: { "content_type": "image/jpeg", "content_length": 2048, }, "form.xml": { "content_type": "text/xml", "content_length": 2048, } } with mock.patch.object(XFormInstanceSQL, 'form_data') as form_data_mock: form_data_mock.__get__ = mock.MagicMock(return_value=form) couch_xform = XFormInstance( received_on=datetime.datetime.now(), form=form, ) for name, meta in attachments.items(): couch_xform.deferred_put_attachment("content", name, **meta) sql_xform = XFormInstanceSQL(received_on=datetime.datetime.now()) sql_xform.attachments_list = [BlobMeta(name=name, **meta) for name, meta in attachments.items()] for xform in (couch_xform, sql_xform): print(type(xform).__name__) form_info = _extract_form_attachment_info(xform, {"my_group-image_2", "image_1"}) attachments = {a['name']: a for a in form_info['attachments']} self.assertTrue(image_1_name in attachments) self.assertTrue(image_2_name in attachments) self.assertEqual(attachments[image_1_name]['question_id'], "image_1") self.assertEqual(attachments[image_2_name]['question_id'], "my_group-image_2")
def new_xform(cls, form_data): form_id = extract_meta_instance_id(form_data) or str(uuid.uuid4()) return XFormInstanceSQL( # other properties can be set post-wrap form_id=form_id, xmlns=form_data.get('@xmlns'), received_on=datetime.datetime.utcnow(), user_id=extract_meta_user_id(form_data), )
def test_update_from_openmrs(self): """ payloads from OpenMRS should not be forwarded back to OpenMRS """ payload = XFormInstanceSQL( domain=DOMAIN, xmlns=XMLNS_OPENMRS, ) repeater = OpenmrsRepeater() self.assertFalse(repeater.allowed_to_forward(payload))
def _make_form_instance(cls, form_id): return XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=datetime.utcnow(), user_id='a-user', domain=cls.domain, state=XFormInstanceSQL.NORMAL, )
def _migrate_form_and_associated_models(self, couch_form, form_is_processed=True): """ Copies `couch_form` into a new sql form """ sql_form = None try: if form_is_processed: form_data = couch_form.form with force_phone_timezones_should_be_processed(): adjust_datetimes(form_data) xmlns = form_data.get("@xmlns", "") user_id = extract_meta_user_id(form_data) else: xmlns = couch_form.xmlns user_id = couch_form.user_id if xmlns == SYSTEM_ACTION_XMLNS: for form_id, case_ids in do_system_action(couch_form): self.case_diff_queue.update(case_ids, form_id) sql_form = XFormInstanceSQL( form_id=couch_form.form_id, domain=self.domain, xmlns=xmlns, user_id=user_id, ) _copy_form_properties(sql_form, couch_form) _migrate_form_attachments(sql_form, couch_form) _migrate_form_operations(sql_form, couch_form) case_stock_result = (self._get_case_stock_result( sql_form, couch_form) if form_is_processed else None) _save_migrated_models(sql_form, case_stock_result) except IntegrityError: exc_info = sys.exc_info() try: sql_form = FormAccessorSQL.get_form(couch_form.form_id) except XFormNotFound: sql_form = None proc = "" if form_is_processed else " unprocessed" log.error("Error migrating%s form %s", proc, couch_form.form_id, exc_info=exc_info) except Exception: proc = "" if form_is_processed else " unprocessed" log.exception("Error migrating%s form %s", proc, couch_form.form_id) try: sql_form = FormAccessorSQL.get_form(couch_form.form_id) except XFormNotFound: sql_form = None finally: if couch_form.doc_type != 'SubmissionErrorLog': self._save_diffs(couch_form, sql_form)
def _migrate_unprocessed_form(self, couch_form_json): from corehq.apps.tzmigration.timezonemigration import json_diff self.log_debug('Processing doc: {}({})'.format(couch_form_json['doc_type'], couch_form_json['_id'])) couch_form = _wrap_form(couch_form_json) sql_form = XFormInstanceSQL( form_id=couch_form.form_id, xmlns=couch_form.xmlns, user_id=couch_form.user_id, ) _copy_form_properties(self.domain, sql_form, couch_form) _migrate_form_attachments(sql_form, couch_form) _migrate_form_operations(sql_form, couch_form) if couch_form.doc_type != 'SubmissionErrorLog': diffs = json_diff(couch_form.to_json(), sql_form.to_json(), track_list_indices=False) self.diff_db.add_diffs( couch_form.doc_type, couch_form.form_id, filter_form_diffs(couch_form.doc_type, diffs) ) _save_migrated_models(sql_form)
def process_patch(patch_form): sql_form = XFormInstanceSQL( form_id=patch_form.form_id, domain=patch_form.domain, xmlns=patch_form.xmlns, user_id=patch_form.user_id, received_on=patch_form.received_on, ) add_form_xml(sql_form, patch_form) add_patch_operation(sql_form) case_stock_result = get_case_and_ledger_updates(patch_form.domain, sql_form) save_sql_form(sql_form, case_stock_result)
def create_form_for_test(domain, case_id=None, attachments=None, save=True, state=XFormInstanceSQL.NORMAL): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :param case_id: create case with ID if supplied :param attachments: additional attachments dict :param save: if False return the unsaved form :return: form object """ from corehq.form_processor.utils import get_simple_form_xml form_id = uuid4().hex user_id = 'user1' utcnow = datetime.utcnow() form_xml = get_simple_form_xml(form_id, case_id) form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain, state=state) attachments = attachments or {} attachment_tuples = map( lambda a: Attachment( name=a[0], raw_content=a[1], content_type=a[1].content_type), attachments.items()) attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml')) FormProcessorSQL.store_attachments(form, attachment_tuples) cases = [] if case_id: case = CommCareCaseSQL( case_id=case_id, domain=domain, type='', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, ) case.track_create(CaseTransaction.form_transaction(case, form)) cases = [case]
def _drop_sql_form_ids(couch_ids, domain): from corehq.sql_db.util import split_list_by_db_partition get_missing_forms = """ SELECT couch.form_id FROM (SELECT unnest(%s) AS form_id) AS couch LEFT JOIN form_processor_xforminstancesql sql USING (form_id) WHERE sql.form_id IS NULL """ for dbname, form_ids in split_list_by_db_partition(couch_ids): with XFormInstanceSQL.get_cursor_for_partition_db( dbname, readonly=True) as cursor: cursor.execute(get_missing_forms, [form_ids]) yield from (form_id for form_id, in cursor.fetchall())
def submission_error_form_instance(cls, domain, instance, message): xform = XFormInstanceSQL( domain=domain, form_id=uuid.uuid4().hex, received_on=datetime.datetime.utcnow(), problem=message, state=XFormInstanceSQL.SUBMISSION_ERROR_LOG ) cls.store_attachments(xform, [Attachment( name=ATTACHMENT_NAME, raw_content=instance, content_type='text/xml', )]) return xform
def _migrate_unprocessed_form(self, couch_form_json): self.log_debug('Processing doc: {}({})'.format(couch_form_json['doc_type'], couch_form_json['_id'])) couch_form = _wrap_form(couch_form_json) sql_form = XFormInstanceSQL( form_id=couch_form.form_id, xmlns=couch_form.xmlns, user_id=couch_form.user_id, ) _copy_form_properties(self.domain, sql_form, couch_form) _migrate_form_attachments(sql_form, couch_form) _migrate_form_operations(sql_form, couch_form) if couch_form.doc_type != 'SubmissionErrorLog': self._save_diffs(couch_form, sql_form) _save_migrated_models(sql_form)
def create_case(case) -> CommCareCaseSQL: form = XFormInstanceSQL( form_id=uuid4().hex, xmlns='http://commcarehq.org/formdesigner/form-processor', received_on=case.server_modified_on, user_id=case.owner_id, domain=case.domain, ) transaction = CaseTransaction( type=CaseTransaction.TYPE_FORM, form_id=form.form_id, case=case, server_date=case.server_modified_on, ) with patch.object(FormProcessorSQL, "publish_changes_to_kafka"): case.track_create(transaction) processed_forms = ProcessedForms(form, []) FormProcessorSQL.save_processed_models(processed_forms, [case])
def _migrate_unprocessed_form(self, couch_form_json): log.debug('Processing doc: {}({})'.format(couch_form_json['doc_type'], couch_form_json['_id'])) try: couch_form = _wrap_form(couch_form_json) sql_form = XFormInstanceSQL( form_id=couch_form.form_id, xmlns=couch_form.xmlns, user_id=couch_form.user_id, ) _copy_form_properties(self.domain, sql_form, couch_form) _migrate_form_attachments(sql_form, couch_form) _migrate_form_operations(sql_form, couch_form) if couch_form.doc_type != 'SubmissionErrorLog': self._save_diffs(couch_form, sql_form) _save_migrated_models(sql_form) self.processed_docs += 1 self._log_unprocessed_forms_processed_count(throttled=True) except Exception: log.exception("Error migrating form %s", couch_form_json["_id"])
def get_databases(): sql_dbs = [ _SQLFormDb(XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQL.get_obj_by_id(id_), XFormInstanceSQL.__name__), _SQLDb( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer( CommCareCaseSQL.get_obj_by_id(id_)).data, CommCareCaseSQL.__name__), _SQLDb(SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get(location_id=id_).to_json(), SQLLocation.__name__), ] all_dbs = OrderedDict() couchdbs_by_name = couch_config.all_dbs_by_db_name for dbname in sorted(couchdbs_by_name): all_dbs[dbname] = _CouchDb(couchdbs_by_name[dbname]) for db in sql_dbs: all_dbs[db.dbname] = db return all_dbs
def _update_forms(db_name, form_ids): with XFormInstanceSQL.get_cursor_for_partition_db(db_name) as cursor: cursor.execute( """ WITH max_dates as ( SELECT form_id, max(modified_on) as modified_on FROM ( SELECT form_id, CASE WHEN deleted_on is not NULL THEN deleted_on WHEN edited_on is not NULL AND edited_on > received_on THEN edited_on ELSE received_on END as modified_on FROM form_processor_xforminstancesql WHERE form_processor_xforminstancesql.form_id in %(form_ids)s union select form_id, max(date) as modified_on from form_processor_xformoperationsql where form_processor_xformoperationsql.form_id in %(form_ids)s group by form_id ) as d group by form_id ) UPDATE form_processor_xforminstancesql SET server_modified_on = max_dates.modified_on FROM max_dates WHERE form_processor_xforminstancesql.form_id = max_dates.form_id """, {'form_ids': form_ids})
def _create_case(domain=None, form_id=None, case_type=None, user_id=None, closed=False, case_id=None): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :return: CommCareCaseSQL """ domain = domain or DOMAIN form_id = form_id or uuid.uuid4().hex case_id = case_id or uuid.uuid4().hex user_id = user_id or 'user1' utcnow = datetime.utcnow() form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain) case = CommCareCaseSQL(case_id=case_id, domain=domain, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, closed=closed or False) case.track_create(CaseTransaction.form_transaction(case, form, utcnow)) cases = [case] FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases) return CaseAccessorSQL.get_case(case_id)
def drop_sql_ids(self, couch_ids): """Filter the given couch ids, removing ids that are in SQL""" for dbname, form_ids in split_list_by_db_partition(couch_ids): with XFormInstanceSQL.get_cursor_for_partition_db(dbname, readonly=True) as cursor: cursor.execute(self.sql, [form_ids]) yield from (form_id for form_id, in cursor.fetchall())
def __init__(self, dbname, getter, doc_type): self.dbname = dbname self._getter = getter self.doc_type = doc_type def get(self, record_id): try: return self._getter(record_id) except (XFormNotFound, CaseNotFound, ObjectDoesNotExist): raise ResourceNotFound("missing") _SQL_DBS = OrderedDict((db.dbname, db) for db in [ _Db( XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_)).data, XFormInstanceSQL.__name__ ), _Db( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_)).data, CommCareCaseSQL.__name__ ), _Db( SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get(location_id=id_).to_json(), SQLLocation.__name__ ), ])
def get_missing_form_ids(db, db_form_ids): with XFormInstanceSQL.get_cursor_for_partition_db( db, readonly=True) as cursor: cursor.execute(sql, [db_form_ids]) return [r[0] for r in cursor.fetchall()]
def __init__(self, dbname, getter, doc_type): self.dbname = dbname self._getter = getter self.doc_type = doc_type def get(self, record_id): try: return self._getter(record_id) except (XFormNotFound, CaseNotFound, ObjectDoesNotExist): raise ResourceNotFound("missing") _SQL_DBS = OrderedDict((db.dbname, db) for db in [ _Db( XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_) ).data, XFormInstanceSQL.__name__), _Db( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_) ).data, CommCareCaseSQL.__name__), _Db(SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get( location_id=id_).to_json(), SQLLocation.__name__), ]) def get_db_from_db_name(db_name): if db_name in _SQL_DBS: return _SQL_DBS[db_name] elif db_name == couch_config.get_db(None).dbname: # primary db return couch_config.get_db(None)