def setUpClass(cls): super(TestExportInstanceDBAccessors, cls).setUpClass() cls.form_instance_deid = FormExportInstance(domain=cls.domain, name='Forms', is_deidentified=True) cls.form_instance_wrong = FormExportInstance( domain='wrong-domain', name='Forms', ) cls.form_instance_daily_saved = FormExportInstance( domain='wrong-domain', is_daily_saved_export=True, ) cls.case_instance_deid = CaseExportInstance(domain=cls.domain, name='Cases', is_deidentified=True) cls.case_instance = CaseExportInstance(domain=cls.domain, name='Cases', is_deidentified=False) cls.case_instance_daily_saved = CaseExportInstance( domain='wrong-domain', is_daily_saved_export=True, ) cls.instances = [ cls.form_instance_deid, cls.form_instance_wrong, cls.form_instance_daily_saved, cls.case_instance, cls.case_instance_deid, cls.case_instance_daily_saved, ] for instance in cls.instances: instance.save()
def setUpClass(cls): super(TestExportInstanceDBAccessors, cls).setUpClass() cls.form_instance = FormExportInstance( domain=cls.domain, name='Forms', is_deidentified=False ) cls.form_instance_deid = FormExportInstance( domain=cls.domain, name='Forms', is_deidentified=True ) cls.form_instance_wrong = FormExportInstance( domain='wrong-domain', name='Forms', ) cls.form_instance_daily_saved = FormExportInstance( domain='wrong-domain', is_daily_saved_export=True, auto_rebuild_enabled=True, last_accessed=datetime.utcnow() ) cls.case_instance_deid = CaseExportInstance( domain=cls.domain, name='Cases', is_deidentified=True ) cls.case_instance = CaseExportInstance( domain=cls.domain, name='Cases', is_deidentified=False ) cls.case_instance_daily_saved = CaseExportInstance( domain='wrong-domain', is_daily_saved_export=True, auto_rebuild_enabled=True, last_accessed=(datetime.utcnow() - timedelta(days=4)) ) cls.instances = [ cls.form_instance, cls.form_instance_deid, cls.form_instance_wrong, cls.form_instance_daily_saved, cls.case_instance, cls.case_instance_deid, cls.case_instance_daily_saved, ] for instance in cls.instances: instance.save()
def get_instance(cls, domain_name): return CaseExportInstance( domain=domain_name, is_odata_config=True, transform_dates=False, tables=[ TableConfiguration( selected=True, columns=[ ExportColumn( label='closed', selected=True, # this is what exports generate for a base level property item=ExportItem(path=[PathNode(name='closed')])), ExportColumn( label='date_modified', selected=True, item=ExportItem( path=[PathNode(name='date_modified')])), ExportColumn(label='selected_property_1', selected=True), ExportColumn(label='selected_property_2', selected=True), ExportColumn(label='unselected_property'), ], ), ])
def test_ignore_case_link_label(self): export_with_case_link = CaseExportInstance( is_odata_config=True, tables=[ TableConfiguration( selected=True, columns=[ ExportColumn( label='my_case_link', item=ExportItem( path=[ PathNode(name='_id') ], transform=CASE_ID_TO_LINK, ), selected=True, ) ] ) ] ) export_with_case_link.save() self.addCleanup(export_with_case_link.delete) cleaned_export = CaseExportInstance.get(export_with_case_link.get_id) self.assertEqual(cleaned_export.tables[0].columns[0].label, 'my_case_link')
def setUp(cls): cls.new_exports = [ FormExportInstance(), CaseExportInstance(), ] for export in cls.new_exports: export.save()
def test_populated_metadata_document(self): odata_config = CaseExportInstance( _id='my_config_id', domain=self.domain.name, is_odata_config=True, tables=[ TableConfiguration( selected=True, columns=[ ExportColumn( label='closed', selected=True, # this is what exports generate for a base level property item=ExportItem(path=[PathNode(name='closed')])), ExportColumn( label='date_modified', selected=True, item=ExportItem( path=[PathNode(name='date_modified')])), ExportColumn(label='selected_property_1', selected=True), ExportColumn(label='selected_property_2', selected=True), ExportColumn(label='unselected_property'), ], ), ]) odata_config.save() self.addCleanup(odata_config.delete) non_odata_config = CaseExportInstance(domain=self.domain.name) non_odata_config.save() self.addCleanup(non_odata_config.delete) config_in_other_domain = CaseExportInstance(domain='other_domain', is_odata_config=True) config_in_other_domain.save() self.addCleanup(config_in_other_domain.delete) correct_credentials = self._get_correct_credentials() response = self._execute_query(correct_credentials) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') self.assertEqual(response['OData-Version'], '4.0') self.assertXmlEqual( self.get_xml('populated_case_odata_metadata_document_from_config', override_path=PATH_TO_TEST_DATA), response.content)
def setUpClass(cls): super().setUpClass() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): cls.es = get_es_new() initialize_index_and_mapping(cls.es, CASE_INDEX_INFO) cls.domain = uuid.uuid4().hex now = datetime.utcnow() cases = [ new_case(domain=cls.domain, foo="apple", bar="banana", server_modified_on=now - timedelta(hours=3)), new_case(domain=cls.domain, foo="orange", bar="pear", server_modified_on=now - timedelta(hours=2)), ] for case in cases: send_to_elasticsearch('cases', case.to_json()) cls.es.indices.refresh(CASE_INDEX_INFO.index) cls.export_instance = CaseExportInstance( export_format=Format.UNZIPPED_CSV, domain=cls.domain, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Foo column", item=ExportItem(path=[PathNode(name="foo")]), selected=True, ), ExportColumn( label="Bar column", item=ExportItem(path=[PathNode(name="bar")]), selected=True, ) ]) ]) cls.export_instance.save() cls.incremental_export = IncrementalExport.objects.create( domain=cls.domain, name='test_export', export_instance_id=cls.export_instance.get_id, connection_settings=ConnectionSettings.objects.create( domain=cls.domain, name='test conn', url='http://somewhere', auth_type=BASIC_AUTH, ))
def setUpClass(cls): cls.exports = [ FormExportInstance(is_daily_saved_export=True), FormExportInstance(is_daily_saved_export=False), CaseExportInstance(is_daily_saved_export=True), ] for export in cls.exports: export.save()
def test_populated_metadata_document(self): odata_config_1 = CaseExportInstance( _id='odata_config_1', domain=self.domain.name, is_odata_config=True, tables=[TableConfiguration(columns=[])]) odata_config_1.save() self.addCleanup(odata_config_1.delete) odata_config_2 = CaseExportInstance( _id='odata_config_2', domain=self.domain.name, is_odata_config=True, tables=[ TableConfiguration(columns=[ ExportColumn(label='selected_property_1', selected=True), ExportColumn(label='selected_property_2', selected=True), ExportColumn(label='unselected_property'), ], ), ]) odata_config_2.save() self.addCleanup(odata_config_2.delete) non_odata_config = CaseExportInstance(domain=self.domain.name) non_odata_config.save() self.addCleanup(non_odata_config.delete) config_in_other_domain = CaseExportInstance(domain='other_domain', is_odata_config=True) config_in_other_domain.save() self.addCleanup(config_in_other_domain.delete) correct_credentials = self._get_correct_credentials() with flag_enabled('ODATA'): with patch( 'corehq.apps.api.odata.views.get_odata_case_configs_by_domain', return_value=sorted(get_odata_case_configs_by_domain( self.domain.name), key=lambda _config: _config.get_id)): response = self._execute_query(correct_credentials) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') self.assertEqual(response['OData-Version'], '4.0') self.assertXmlEqual( self.get_xml('populated_case_odata_metadata_document_from_config', override_path=PATH_TO_TEST_DATA), response.content)
def test_populated_metadata_document(self): non_odata_config = CaseExportInstance(domain=self.domain.name) non_odata_config.save() self.addCleanup(non_odata_config.delete) config_in_other_domain = CaseExportInstance(domain='other_domain', is_odata_config=True) config_in_other_domain.save() self.addCleanup(config_in_other_domain.delete) correct_credentials = self._get_correct_credentials() response = self._execute_query(correct_credentials) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') self.assertEqual(response['OData-Version'], '4.0') self.assertXmlEqual( self.get_xml('populated_case_odata_metadata_document_from_config', override_path=PATH_TO_TEST_DATA), response.content)
def setUpClass(cls): super(TestDailySavedExports, cls).setUpClass() cls.exports = [ FormExportInstance(is_daily_saved_export=True), FormExportInstance(is_daily_saved_export=False), CaseExportInstance(is_daily_saved_export=True), ] for export in cls.exports: export.save()
def test_selected_false(self): export_json = get_export_json( CaseExportInstance(export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration(label="My table", selected=False, path=[], columns=[]) ])) self.assertEqual(export_json, {})
def test_populated_service_document(self): odata_config_1 = CaseExportInstance(domain=self.domain.name, is_odata_config=True) odata_config_1.save() self.addCleanup(odata_config_1.delete) odata_config_2 = CaseExportInstance(domain=self.domain.name, is_odata_config=True) odata_config_2.save() self.addCleanup(odata_config_2.delete) non_odata_config = CaseExportInstance(domain=self.domain.name) non_odata_config.save() self.addCleanup(non_odata_config.delete) config_in_other_domain = CaseExportInstance(domain='other_domain', is_odata_config=True) config_in_other_domain.save() self.addCleanup(config_in_other_domain.delete) correct_credentials = self._get_correct_credentials() with flag_enabled('ODATA'): response = self._execute_query(correct_credentials) self.assertEqual(response.status_code, 200) self.assertEqual(response['OData-Version'], '4.0') response_content = json.loads(response.content.decode('utf-8')) self.assertCountEqual(response_content, ['@odata.context', 'value']) self.assertEqual( response_content['@odata.context'], 'http://localhost:8000/a/test_domain/api/v0.5/odata/cases/$metadata' ) self.assertCountEqual(response_content['value'], [ { 'url': odata_config_1.get_id, 'kind': 'EntitySet', 'name': odata_config_1.get_id, }, { 'url': odata_config_2.get_id, 'kind': 'EntitySet', 'name': odata_config_2.get_id, }, ])
def test_get_export_file(self): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")] ), selected=True, ), ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")] ), selected=True, ) ] )] ), ], [] # No filters ) with export_file as export: self.assertEqual( json.loads(export.read()), { u'My table': { u'headers': [ u'Foo column', u'Bar column'], u'rows': [ [u'apple', u'banana'], [u'apple', u'banana'], [u'apple', u'banana'], ], } } )
def test_missing_value_is_null(self): self.assertEqual( ODataCaseSerializer.serialize_cases_using_config( [{}], CaseExportInstance(tables=[ TableConfiguration(columns=[ ExportColumn( label='owner-name-label', item=ExportItem(path=[PathNode( name='owner_name')]), selected=True, ) ]) ])), [{ 'owner-name-label': '---' }])
def test_config_in_different_domain(self): export_config_in_other_domain = CaseExportInstance( _id='config_id', tables=[TableConfiguration(columns=[])], case_type='my_case_type', domain='different_domain') export_config_in_other_domain.save() self.addCleanup(export_config_in_other_domain.delete) correct_credentials = self._get_correct_credentials() response = self.client.get( self._odata_feed_url_by_domain_and_config_id( self.domain.name, export_config_in_other_domain.get_id), HTTP_AUTHORIZATION='Basic ' + correct_credentials, ) self.assertEqual(response.status_code, 404)
def test_user_permissions(self): self.web_user.set_role(self.domain.name, 'none') self.web_user.save() self.addCleanup(self._setup_user_permissions) export_config = CaseExportInstance( _id='config_id', tables=[TableConfiguration(columns=[])], case_type='my_case_type', domain=self.domain.name, ) export_config.save() self.addCleanup(export_config.delete) correct_credentials = self._get_correct_credentials() response = self._execute_query(correct_credentials) self.assertEqual(response.status_code, 403)
def test_unselected_column_excluded(self): self.assertEqual( ODataCaseSerializer.serialize_cases_using_config( [{ 'owner_name': 'owner-name-value', 'properties': {} }], CaseExportInstance(tables=[ TableConfiguration(columns=[ ExportColumn( label='owner-name-label', item=ExportItem(path=[PathNode( name='owner_name')]), selected=False, ) ]) ])), [{}])
def test_case_name(self): self.assertEqual( ODataCaseSerializer.serialize_cases_using_config( [{ 'name': 'case-name-value' }], CaseExportInstance(tables=[ TableConfiguration(columns=[ ExportColumn( label='case-name-label', item=ExportItem(path=[PathNode(name='name')]), selected=True, ) ]) ])), [{ 'case-name-label': 'case-name-value' }])
def test_selected_false(self): export_file = get_export_file( [ CaseExportInstance(export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration(label="My table", selected=False, path=[], columns=[]) ]), ], [] # No filters ) with export_file as export: self.assertEqual(json.loads(export.read()), {})
def test_caseid_column_label(self): export_with_modified_caseid_column = CaseExportInstance( is_odata_config=True, tables=[ TableConfiguration(columns=[ ExportColumn( label='modified_case_id_column', item=ExportItem(path=[PathNode(name='_id')]), selected=True, ) ]) ]) export_with_modified_caseid_column.save() self.addCleanup(export_with_modified_caseid_column.delete) cleaned_export = CaseExportInstance.get( export_with_modified_caseid_column.get_id) self.assertEqual(cleaned_export.tables[0].columns[0].label, 'caseid')
def test_get_export_file(self): export_json = get_export_json( CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")] ), selected=True, ), ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")] ), selected=True, ) ] )] ), ) self.assertEqual( export_json, { 'My table': { 'headers': [ 'Foo column', 'Bar column'], 'rows': [ ['apple', 'banana'], ['apple', 'banana'], ['apple', 'banana'], ], } } )
def test_non_standard_case_property(self): self.assertEqual( ODataCaseSerializer.serialize_cases_using_config( [{ 'property_1': 'property-1-value' }], CaseExportInstance(tables=[ TableConfiguration(columns=[ ExportColumn( label='property-1-label', item=ExportItem(path=[PathNode( name='property_1')]), selected=True, ) ]) ])), [{ 'property-1-label': 'property-1-value' }])
def test_export_transforms(self, _): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="DEID Date Transform column", item=ExportItem( path=[PathNode(name="date")] ), selected=True, deid_transform=DEID_DATE_TRANSFORM, ) ] )] ), ], [] # No filters ) with export_file as export: export_dict = json.loads(export.read()) export_dict['My table']['rows'].sort() self.assertEqual( export_dict, { u'My table': { u'headers': [ u'DEID Date Transform column [sensitive]', ], u'rows': [ [MISSING_VALUE], [u'2016-04-07'], [u'2016-04-27'], # offset by 3 since that's the mocked random offset ], } } )
def setUp(self): super().setUp() self.export_instance = CaseExportInstance( export_format=Format.UNZIPPED_CSV, domain=self.domain, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="Foo column", item=ExportItem(path=[PathNode(name="foo")]), selected=True, ), ExportColumn( label="Bar column", item=ExportItem(path=[PathNode(name="bar")]), selected=True, ) ]) ]) self.export_instance.save() connection_settings = ConnectionSettings.objects.create( domain=self.domain, name='test conn', url='http://commcarehq.org', auth_type=BASIC_AUTH, username='******', password='******', ) self.incremental_export = IncrementalExport.objects.create( domain=self.domain, name='test_export', export_instance_id=self.export_instance.get_id, connection_settings=connection_settings, )
def test_request_succeeded(self): export_config = CaseExportInstance( _id='config_id', tables=[TableConfiguration(columns=[])], case_type='my_case_type', domain=self.domain.name, ) export_config.save() self.addCleanup(export_config.delete) correct_credentials = self._get_correct_credentials() response = self._execute_query(correct_credentials) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/json; charset=utf-8') self.assertEqual(response['OData-Version'], '4.0') self.assertEqual( json.loads(response.content.decode('utf-8')), { '@odata.context': 'http://localhost:8000/a/test_domain/api/v0.5/odata/cases/config_id/$metadata#feed', 'value': [] })
def test_export_transforms(self, export_save, _): export_json = get_export_json( CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[ TableConfiguration( label="My table", selected=True, path=[], columns=[ ExportColumn( label="DEID Date Transform column", item=ExportItem(path=[PathNode(name="date")]), selected=True, deid_transform=DEID_DATE_TRANSFORM, ) ]) ]), ) export_json['My table']['rows'].sort() self.assertEqual( export_json, { 'My table': { 'headers': [ 'DEID Date Transform column [sensitive]', ], 'rows': [ [MISSING_VALUE], ['2016-04-07'], [ '2016-04-27' ], # offset by 3 since that's the mocked random offset ], } }) self.assertTrue(export_save.called)
def test_default_case_values(self): # Confirm that CaseExportInstances set the default show_all_data flag correctly case_export = CaseExportInstance() case_export_wrapped = CaseExportInstance.wrap({}) for e in [case_export, case_export_wrapped]: self.assertTrue(e.filters.show_all_data)
def test_simple_bulk_export(self): export_file = get_export_file( [ CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( selected=True, label="My table", path=MAIN_TABLE, columns=[ ExportColumn( label="Foo column", item=ExportItem( path=[PathNode(name="foo")] ), selected=True, ), ] )] ), CaseExportInstance( export_format=Format.JSON, domain=DOMAIN, case_type=DEFAULT_CASE_TYPE, tables=[TableConfiguration( label="My table", selected=True, path=MAIN_TABLE, columns=[ ExportColumn( label="Bar column", item=ExportItem( path=[PathNode(name="bar")] ), selected=True, ) ] )] ), ], [] # No filters ) expected = { 'Export1-My table': { "A1": "Foo column", "A2": "apple", "A3": "apple", "A4": "apple", }, "Export2-My table": { "A1": "Bar column", "A2": "banana", "A3": "banana", "A4": "banana", }, } with export_file as export: wb = load_workbook(export) self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"]) for sheet in expected.keys(): for cell in expected[sheet].keys(): self.assertEqual( wb[sheet][cell].value, expected[sheet][cell], 'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format( sheet, cell, expected[sheet][cell], wb[sheet][cell].value ) )