def test_archived_forms(self): xmlns1 = 'https://xmlns1' xmlns2 = 'https://xmlns2' forms = self._send_forms([(xmlns1, None), (xmlns2, None)]) # archive update = forms[0].to_json() update['doc_type'] = 'xformarchived' send_to_elasticsearch('forms', transform_xform_for_elasticsearch(update)) self.es.indices.refresh(XFORM_INDEX_INFO.index) # archived form should not be included by default response = self._assert_auth_get_resource(self.list_endpoint) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 1) api_form = api_forms[0] self.assertEqual(api_form['form']['@xmlns'], xmlns2) # archived form should be included response = self._assert_auth_get_resource( '%s?%s' % (self.list_endpoint, urlencode({'include_archived': 'true'}))) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 2)
def _send_form_to_es( self, domain=None, completion_time=None, received_on=None, attachment_dict=None, **metadata_kwargs): attachment_dict = attachment_dict or {} metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) if attachment_dict: form_pair.wrapped_form.external_blobs = {name: BlobMetaRef(**meta) for name, meta in six.iteritems(attachment_dict)} form_pair.json_form['external_blobs'] = attachment_dict es_form = transform_xform_for_elasticsearch(form_pair.json_form) send_to_elasticsearch('forms', es_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def _send_forms(self, forms): # list of form tuples [(xmlns, received_on)] to_ret = [] for xmlns, received_on in forms: backend_form = XFormInstance( xmlns=xmlns or 'fake-xmlns', domain=self.domain.name, received_on=received_on or datetime.utcnow(), edited_on=datetime.utcnow(), form={ '#type': 'fake-type', '@xmlns': xmlns or 'fake-xmlns', 'meta': { 'userID': 'metadata-user-id' }, }, auth_context={ 'user_id': 'auth-user-id', 'domain': self.domain.name, 'authenticated': True, }, ) backend_form.save() to_ret.append(backend_form) self.addCleanup(backend_form.delete) send_to_elasticsearch( 'forms', transform_xform_for_elasticsearch(backend_form.to_json())) self.es.indices.refresh(XFORM_INDEX_INFO.index) return to_ret
def _send_form_to_es(self, domain=None, completion_time=None, received_on=None, attachment_dict=None, **metadata_kwargs): attachment_dict = attachment_dict or {} metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) if attachment_dict: form_pair.wrapped_form.external_blobs = { name: BlobMetaRef(**meta) for name, meta in attachment_dict.items() } form_pair.json_form['external_blobs'] = attachment_dict es_form = transform_xform_for_elasticsearch(form_pair.json_form) send_to_elasticsearch('forms', es_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def form_to_es_form(xform_instance): from corehq.pillows.xform import transform_xform_for_elasticsearch, xform_pillow_filter from corehq.apps.api.models import ESXFormInstance json_form = xform_instance.to_json() if not xform_pillow_filter(json_form): es_form = transform_xform_for_elasticsearch(json_form) return ESXFormInstance(es_form)
def testXFormPillowSingleCaseProcess(self): """ Test that xform pillow can process and cleanup a single xform with a case submission """ xform = XFORM_SINGLE_CASE changed = transform_xform_for_elasticsearch(xform) self.assertIsNone(changed['form']['case'].get('@date_modified')) self.assertIsNotNone(xform['form']['case']['@date_modified'])
def _send_forms_to_es(self, forms): for form in forms: es_form = transform_xform_for_elasticsearch( FormDocumentStore(form.domain, form.xmlns).get_document(form.form_id)) send_to_elasticsearch('forms', es_form) self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index) self.forms_to_delete_from_es.update(form.form_id for form in forms)
def test_transform_xform_for_elasticsearch_location_missing(self): doc_dict = { 'domain': 'demo', 'form': { 'meta': { } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['geo_point'], None)
def _submit_form(self, xmlns, form_name, app_id, build_id): xform_source = self.get_xml('xform_template').decode('utf-8').format( xmlns=xmlns, name=form_name, id=uuid.uuid4().hex) result = submit_form_locally(xform_source, DOMAIN, app_id=app_id, build_id=build_id) send_to_elasticsearch( 'forms', transform_xform_for_elasticsearch(result.xform.to_json())) return result.xform
def transform_xform_for_report_forms_index(doc_dict): doc_ret = transform_xform_for_elasticsearch(doc_dict) convert_property_dict( doc_ret['form'], REPORT_XFORM_INDEX_INFO.mapping['properties']['form'], override_root_keys=['case']) if 'computed_' in doc_ret: convert_property_dict(doc_ret['computed_'], {}) return doc_ret
def test_transform_xform_for_elasticsearch_location_missing(self): doc_dict = { 'domain': 'demo', 'received_on': '2013-10-12T11:59:41Z', 'form': { 'meta': {} } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['geo_point'], None)
def test_transform_xform_for_elasticsearch_location_bad(self): doc_dict = { 'domain': 'demo', 'form': { 'meta': { 'location': 'not valid' } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['geo_point'], None)
def transform_xform_for_report_forms_index(doc_dict): doc_ret = transform_xform_for_elasticsearch(doc_dict) convert_property_dict( doc_ret['form'], REPORT_XFORM_INDEX_INFO.mapping['properties']['form'], override_root_keys=['case'] ) if 'computed_' in doc_ret: convert_property_dict(doc_ret['computed_'], {}) return doc_ret
def test_transform_xform_for_elasticsearch_location(self): doc_dict = { 'domain': 'demo', 'form': { 'meta': { 'location': '42.7 -21 0 0' } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['geo_point'], {'lat': Decimal('42.7'), 'lon': Decimal('-21')})
def testXFormPillowListCaseProcess(self): """ Test that xform pillow can process and cleanup a single xform with a list of cases in it """ xform = XFORM_MULTI_CASES changed = transform_xform_for_elasticsearch(xform) changed_cases = extract_case_blocks(changed) orig_cases = extract_case_blocks(xform) [self.assertIsNotNone(x['@date_modified']) for x in orig_cases] [self.assertIsNone(x.get('@date_modified')) for x in changed_cases]
def test_transform_xform_for_elasticsearch_app_versions(self): doc_dict = { 'domain': 'demo', 'form': { 'meta': { 'appVersion': 'version "2.27.2"(414569). App v56. 2.27. Build 414569' } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['commcare_version'], '2.27.2') self.assertEqual(doc_ret['form']['meta']['app_build_version'], 56)
def test_transform_xform_for_elasticsearch_app_versions_none(self): doc_dict = { 'domain': 'demo', 'form': { 'meta': { 'appVersion': 'not an app version' } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['commcare_version'], None) self.assertEqual(doc_ret['form']['meta']['app_build_version'], None)
def _setup_es_for_data(cls): cls.es = get_es_new() cls.es_indices = [XFORM_INDEX_INFO] for index_info in cls.es_indices: initialize_index_and_mapping(cls.es, index_info) for form in cls.forms + cls.forms_b: es_form = transform_xform_for_elasticsearch(form) send_to_elasticsearch('forms', es_form) for index_info in cls.es_indices: cls.es.indices.refresh(index_info.index)
def test_transform_xform_for_elasticsearch_location(self): doc_dict = { 'domain': 'demo', 'received_on': '2013-10-12T11:59:41Z', 'form': { 'meta': { 'location': '42.7 -21 0 0' } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['geo_point'], {'lat': 42.7, 'lon': -21})
def test_transform_xform_for_elasticsearch_location_bad(self): doc_dict = { 'domain': 'demo', 'received_on': '2013-10-12T11:59:41Z', 'form': { 'meta': { 'location': 'not valid' } } } doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertEqual(doc_ret['form']['meta']['geo_point'], None)
def test_get_list(self): """ Any form in the appropriate domain should be in the list from the API. """ # The actual infrastructure involves saving to CouchDB, having PillowTop # read the changes and write it to ElasticSearch. # In order to test just the API code, we set up a fake XFormES (this should # really be a parameter to the XFormInstanceResource constructor) # and write the translated form directly; we are not trying to test # the ptop infrastructure. # the pillow is set to offline mode - elasticsearch not needed to validate fake_xform_es = FakeXFormES() v0_4.MOCK_XFORM_ES = fake_xform_es backend_form = XFormInstance( xmlns='fake-xmlns', domain=self.domain.name, received_on=datetime.utcnow(), edited_on=datetime.utcnow(), form={ '#type': 'fake-type', '@xmlns': 'fake-xmlns', 'meta': { 'userID': 'metadata-user-id' }, }, auth_context={ 'user_id': 'auth-user-id', 'domain': self.domain.name, 'authenticated': True, }, ) backend_form.save() self.addCleanup(backend_form.delete) translated_doc = transform_xform_for_elasticsearch( backend_form.to_json()) fake_xform_es.add_doc(translated_doc['_id'], translated_doc) response = self._assert_auth_get_resource(self.list_endpoint) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 1) api_form = api_forms[0] self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns) self.assertEqual(api_form['received_on'], json_format_datetime(backend_form.received_on)) self.assertEqual(api_form['metadata']['userID'], 'metadata-user-id') self.assertEqual(api_form['edited_by_user_id'], 'auth-user-id')
def test_transform_xform_base_case_xmlns(self): doc_dict = { 'domain': 'demo', 'form': { "case": { "@case_id": "123", "@xmlns": "ZZZ" }, } } # previously raised an error doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertIsNotNone(doc_ret)
def test_transform_xform_base_case_dates(self): doc_dict = { 'domain': 'demo', 'form': { "case": { "@case_id": "123", "@date_modified": "13:54Z", }, } } # previously raised an error doc_ret = transform_xform_for_elasticsearch(doc_dict) self.assertIsNotNone(doc_ret)
def form_to_es_form(xform_instance, include_attachments=False): # include_attachments is only relevant for SQL domains; they're always # included for Couch domains from corehq.pillows.xform import transform_xform_for_elasticsearch, xform_pillow_filter from corehq.apps.api.models import ESXFormInstance from corehq.form_processor.models import XFormInstanceSQL if include_attachments and isinstance(xform_instance, XFormInstanceSQL): json_form = xform_instance.to_json(include_attachments=True) else: json_form = xform_instance.to_json() if not xform_pillow_filter(json_form): es_form = transform_xform_for_elasticsearch(json_form) return ESXFormInstance(es_form)
def test_get_list(self): """ Any form in the appropriate domain should be in the list from the API. """ # The actual infrastructure involves saving to CouchDB, having PillowTop # read the changes and write it to ElasticSearch. # In order to test just the API code, we set up a fake XFormES (this should # really be a parameter to the XFormInstanceResource constructor) # and write the translated form directly; we are not trying to test # the ptop infrastructure. # the pillow is set to offline mode - elasticsearch not needed to validate fake_xform_es = FakeXFormES() v0_4.MOCK_XFORM_ES = fake_xform_es backend_form = XFormInstance( xmlns='fake-xmlns', domain=self.domain.name, received_on=datetime.utcnow(), edited_on=datetime.utcnow(), form={ '#type': 'fake-type', '@xmlns': 'fake-xmlns', 'meta': {'userID': 'metadata-user-id'}, }, auth_context={ 'user_id': 'auth-user-id', 'domain': self.domain.name, 'authenticated': True, }, ) backend_form.save() self.addCleanup(backend_form.delete) translated_doc = transform_xform_for_elasticsearch(backend_form.to_json()) fake_xform_es.add_doc(translated_doc['_id'], translated_doc) response = self._assert_auth_get_resource(self.list_endpoint) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 1) api_form = api_forms[0] self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns) self.assertEqual(api_form['received_on'], json_format_datetime(backend_form.received_on)) self.assertEqual(api_form['metadata']['userID'], 'metadata-user-id') self.assertEqual(api_form['edited_by_user_id'], 'auth-user-id')
def setUpClass(cls): super(ExportsFormsAnalyticsTest, cls).setUpClass() from casexml.apps.case.tests.util import delete_all_xforms from corehq.apps.app_manager.models import Application, Module, Form delete_all_xforms() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): cls.es = get_es_new() initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) cls.domain = 'exports_forms_analytics_domain' cls.app_id_1 = 'a' + uuid.uuid4().hex cls.app_id_2 = 'b' + uuid.uuid4().hex cls.xmlns_1 = 'my://crazy.xmlns/' cls.xmlns_2 = 'my://crazy.xmlns/app' cls.apps = [ Application(_id=cls.app_id_2, domain=cls.domain, modules=[Module(forms=[Form(xmlns=cls.xmlns_2)])]) ] for app in cls.apps: app.save() cls.forms = [ create_form_for_test(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1, save=False), create_form_for_test(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1, save=False), create_form_for_test(domain=cls.domain, app_id=cls.app_id_2, xmlns=cls.xmlns_2, save=False), ] cls.error_forms = [ create_form_for_test(domain=cls.domain, state=XFormInstance.ERROR, save=False) ] cls.all_forms = cls.forms + cls.error_forms for form in cls.all_forms: elastic_form = transform_xform_for_elasticsearch(form.to_json()) send_to_elasticsearch('forms', elastic_form) cls.es.indices.refresh(XFORM_INDEX_INFO.alias)
def test_fetching_xform_cases(self): # Create an xform that touches a case case_id = uuid.uuid4().hex form = submit_case_blocks( CaseBlock.deprecated_init( case_id=case_id, create=True, ).as_text(), self.domain.name)[0] send_to_elasticsearch( 'forms', transform_xform_for_elasticsearch(form.to_json())) self.es.indices.refresh(XFORM_INDEX_INFO.index) # Fetch the xform through the API response = self._assert_auth_get_resource( self.single_endpoint(form.form_id) + "?cases__full=true") self.assertEqual(response.status_code, 200) cases = json.loads(response.content)['cases'] # Confirm that the case appears in the resource self.assertEqual(len(cases), 1) self.assertEqual(cases[0]['id'], case_id)
def test_fetching_xform_cases(self): fake_xform_es = FakeXFormES(ESXFormInstance) v0_4.MOCK_XFORM_ES = fake_xform_es # Create an xform that touches a case case_id = uuid.uuid4().hex form = submit_case_blocks( CaseBlock( case_id=case_id, create=True, ).as_text(), self.domain.name )[0] fake_xform_es.add_doc(form.form_id, transform_xform_for_elasticsearch(form.to_json())) # Fetch the xform through the API response = self._assert_auth_get_resource(self.single_endpoint(form.form_id) + "?cases__full=true") self.assertEqual(response.status_code, 200) cases = json.loads(response.content)['cases'] # Confirm that the case appears in the resource self.assertEqual(len(cases), 1) self.assertEqual(cases[0]['id'], case_id)
def _submit_form(self, xmlns, form_name, app_id, build_id): xform_source = self.get_xml('xform_template').decode('utf-8').format( xmlns=xmlns, name=form_name, id=uuid.uuid4().hex) result = submit_form_locally(xform_source, DOMAIN, app_id=app_id, build_id=build_id) send_to_elasticsearch('forms', transform_xform_for_elasticsearch(result.xform.to_json())) return result.xform
def transform_doc(doc): doc = deepcopy(doc) doc['xmlns.exact'] = doc.get('xmlns', '') doc['form.meta.timeEnd'] = parser.parse(doc['form']['meta']['timeEnd']) return transform_xform_for_elasticsearch(doc)
def setUpClass(cls): super().setUpClass() cls.es = get_es_new() initialize_index_and_mapping(cls.es, USER_INDEX_INFO) initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) cls.date_start, cls.date_end = get_start_and_end_dates_of_month( 2021, 11) cls.domain = Domain.get_or_create_with_name('test-partner-analytics', is_active=True) # Data for Mobile Workers Tests DomainUserHistory.objects.create(domain=cls.domain.name, record_date=cls.date_end, num_users=3) DomainUserHistory.objects.create(domain=cls.domain.name, record_date=cls.date_start - datetime.timedelta(days=1), num_users=6) DomainUserHistory.objects.create(domain=cls.domain.name, record_date=cls.date_end + datetime.timedelta(days=1), num_users=9) # Data for Web Users tests cls.users = [ WebUser.create(cls.domain.name, '*****@*****.**', 'testpwd', None, None, date=cls.date_start + datetime.timedelta(days=5)), WebUser.create(cls.domain.name, '*****@*****.**', 'testpwd', None, None, date=cls.date_start - datetime.timedelta(days=1)), WebUser.create(cls.domain.name, '*****@*****.**', 'testpwd', None, None, date=cls.date_end + datetime.timedelta(days=1)), WebUser.create(cls.domain.name, '*****@*****.**', 'testpwd', None, None, date=cls.date_start + datetime.timedelta(days=6)), ] for user in cls.users: elastic_user = transform_user_for_elasticsearch(user.to_json()) send_to_elasticsearch('users', elastic_user) invitations = [ Invitation.objects.create( domain=cls.domain.name, email='*****@*****.**', invited_by='*****@*****.**', invited_on=cls.date_start + datetime.timedelta(days=5), ), Invitation.objects.create( domain=cls.domain.name, email='*****@*****.**', invited_by='*****@*****.**', invited_on=cls.date_start - datetime.timedelta(days=1), ), Invitation.objects.create( domain=cls.domain.name, email='*****@*****.**', invited_by='*****@*****.**', invited_on=cls.date_end + datetime.timedelta(days=1), ), ] for invitation in invitations: invitation.is_accepted = True invitation.save() # Data for forms tests forms = [ create_form_for_test(cls.domain.name, received_on=cls.date_start - datetime.timedelta(days=1)), create_form_for_test(cls.domain.name, received_on=cls.date_end + datetime.timedelta(days=1)), create_form_for_test(cls.domain.name, received_on=cls.date_start + datetime.timedelta(days=3)), create_form_for_test(cls.domain.name, received_on=cls.date_start + datetime.timedelta(days=12)), ] for form in forms: elastic_form = transform_xform_for_elasticsearch(form.to_json()) send_to_elasticsearch('forms', elastic_form) cls.es.indices.refresh(USER_INDEX_INFO.alias) cls.es.indices.refresh(XFORM_INDEX_INFO.alias)