def test_ignore_case_link_label(self):
export_with_case_link = CaseExportInstance(
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='my_case_link',
item=ExportItem(
path=[
PathNode(name='_id')
],
transform=CASE_ID_TO_LINK,
),
selected=True,
)
]
)
]
)
export_with_case_link.save()
self.addCleanup(export_with_case_link.delete)
cleaned_export = CaseExportInstance.get(export_with_case_link.get_id)
self.assertEqual(cleaned_export.tables[0].columns[0].label, 'my_case_link')
def test_config_in_different_domain(self):
export_config_in_other_domain = CaseExportInstance(
_id='config_id',
tables=[TableConfiguration(columns=[])],
case_type='my_case_type',
domain='different_domain')
export_config_in_other_domain.save()
self.addCleanup(export_config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self.client.get(
self._odata_feed_url_by_domain_and_config_id(
self.domain.name, export_config_in_other_domain.get_id),
HTTP_AUTHORIZATION='Basic ' + correct_credentials,
)
self.assertEqual(response.status_code, 404)
def test_caseid_column_label(self):
export_with_modified_caseid_column = CaseExportInstance(
is_odata_config=True,
tables=[
TableConfiguration(columns=[
ExportColumn(
label='modified_case_id_column',
item=ExportItem(path=[PathNode(name='_id')]),
selected=True,
)
])
])
export_with_modified_caseid_column.save()
self.addCleanup(export_with_modified_caseid_column.delete)
cleaned_export = CaseExportInstance.get(
export_with_modified_caseid_column.get_id)
self.assertEqual(cleaned_export.tables[0].columns[0].label, 'caseid')
def test_user_permissions(self):
self.web_user.set_role(self.domain.name, 'none')
self.web_user.save()
self.addCleanup(self._setup_user_permissions)
export_config = CaseExportInstance(
_id='config_id',
tables=[TableConfiguration(columns=[])],
case_type='my_case_type',
domain=self.domain.name,
)
export_config.save()
self.addCleanup(export_config.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 403)
def test_populated_metadata_document(self):
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
def test_request_succeeded(self):
export_config = CaseExportInstance(
_id='config_id',
tables=[TableConfiguration(columns=[])],
case_type='my_case_type',
domain=self.domain.name,
)
export_config.save()
self.addCleanup(export_config.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(response['OData-Version'], '4.0')
self.assertEqual(
json.loads(response.content.decode('utf-8')), {
'@odata.context':
'http://localhost:8000/a/test_domain/api/v0.5/odata/cases/config_id/$metadata#feed',
'value': []
})
def test_populated_metadata_document(self):
odata_config_1 = CaseExportInstance(
_id='odata_config_1',
domain=self.domain.name,
is_odata_config=True,
tables=[TableConfiguration(columns=[])])
odata_config_1.save()
self.addCleanup(odata_config_1.delete)
odata_config_2 = CaseExportInstance(
_id='odata_config_2',
domain=self.domain.name,
is_odata_config=True,
tables=[
TableConfiguration(columns=[
ExportColumn(label='selected_property_1', selected=True),
ExportColumn(label='selected_property_2', selected=True),
ExportColumn(label='unselected_property'),
], ),
])
odata_config_2.save()
self.addCleanup(odata_config_2.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
with flag_enabled('ODATA'):
with patch(
'corehq.apps.api.odata.views.get_odata_case_configs_by_domain',
return_value=sorted(get_odata_case_configs_by_domain(
self.domain.name),
key=lambda _config: _config.get_id)):
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
def test_populated_service_document(self):
odata_config_1 = CaseExportInstance(domain=self.domain.name, is_odata_config=True)
odata_config_1.save()
self.addCleanup(odata_config_1.delete)
odata_config_2 = CaseExportInstance(domain=self.domain.name, is_odata_config=True)
odata_config_2.save()
self.addCleanup(odata_config_2.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain', is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
with flag_enabled('ODATA'):
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['OData-Version'], '4.0')
response_content = json.loads(response.content.decode('utf-8'))
self.assertCountEqual(response_content, ['@odata.context', 'value'])
self.assertEqual(
response_content['@odata.context'],
'http://localhost:8000/a/test_domain/api/v0.5/odata/cases/$metadata'
)
self.assertCountEqual(response_content['value'], [
{
'url': odata_config_1.get_id,
'kind': 'EntitySet',
'name': odata_config_1.get_id,
},
{
'url': odata_config_2.get_id,
'kind': 'EntitySet',
'name': odata_config_2.get_id,
},
])
def test_populated_metadata_document(self):
odata_config = CaseExportInstance(
_id='my_config_id',
domain=self.domain.name,
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='closed',
selected=True,
# this is what exports generate for a base level property
item=ExportItem(path=[PathNode(name='closed')])),
ExportColumn(
label='date_modified',
selected=True,
item=ExportItem(
path=[PathNode(name='date_modified')])),
ExportColumn(label='selected_property_1',
selected=True),
ExportColumn(label='selected_property_2',
selected=True),
ExportColumn(label='unselected_property'),
],
),
])
odata_config.save()
self.addCleanup(odata_config.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
class TestIncrementalExport(TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
with trap_extra_setup(ConnectionError,
msg="cannot connect to elasicsearch"):
cls.es = get_es_new()
initialize_index_and_mapping(cls.es, CASE_INDEX_INFO)
initialize_index_and_mapping(cls.es, USER_INDEX_INFO)
cls.domain = uuid.uuid4().hex
create_domain(cls.domain)
cls.now = datetime.utcnow()
cases = [
new_case(domain=cls.domain,
foo="apple",
bar="banana",
server_modified_on=cls.now - timedelta(hours=3)),
new_case(domain=cls.domain,
foo="orange",
bar="pear",
server_modified_on=cls.now - timedelta(hours=2)),
]
for case in cases:
send_to_elasticsearch('cases', case.to_json())
cls.es.indices.refresh(CASE_INDEX_INFO.index)
@classmethod
def tearDownClass(cls):
ensure_index_deleted(CASE_INDEX_INFO.index)
super().tearDownClass()
def setUp(self):
super().setUp()
self.export_instance = CaseExportInstance(
export_format=Format.UNZIPPED_CSV,
domain=self.domain,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(path=[PathNode(name="foo")]),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(path=[PathNode(name="bar")]),
selected=True,
)
])
])
self.export_instance.save()
self.incremental_export = IncrementalExport.objects.create(
domain=self.domain,
name='test_export',
export_instance_id=self.export_instance.get_id,
connection_settings=ConnectionSettings.objects.create(
domain=self.domain,
name='test conn',
url='http://somewhere',
auth_type=BASIC_AUTH,
))
def tearDown(self):
self.incremental_export.delete()
self.export_instance.delete()
super().tearDown()
def _cleanup_case(self, case_id):
def _clean():
self.es.delete(CASE_INDEX_INFO.index, CASE_INDEX_INFO.type,
case_id)
self.es.indices.refresh(CASE_INDEX_INFO.index)
return _clean
def test_initial(self):
checkpoint = _generate_incremental_export(self.incremental_export)
data = checkpoint.get_blob().read().decode('utf-8-sig')
expected = "Foo column,Bar column\r\napple,banana\r\norange,pear\r\n"
self.assertEqual(data, expected)
self.assertEqual(checkpoint.doc_count, 2)
return checkpoint
def test_initial_failure(self):
# calling it twice should result in the same output since the checkpoints were not
# marked as success
self.test_initial()
self.test_initial()
def test_incremental_success(self):
checkpoint = self.test_initial()
checkpoint.status = IncrementalExportStatus.SUCCESS
checkpoint.save()
case = new_case(domain=self.domain,
foo="peach",
bar="plumb",
server_modified_on=datetime.utcnow())
send_to_elasticsearch('cases', case.to_json())
self.es.indices.refresh(CASE_INDEX_INFO.index)
self.addCleanup(self._cleanup_case(case.case_id))
checkpoint = _generate_incremental_export(
self.incremental_export, last_doc_date=checkpoint.last_doc_date)
data = checkpoint.get_blob().read().decode('utf-8-sig')
expected = "Foo column,Bar column\r\npeach,plumb\r\n"
self.assertEqual(data, expected)
self.assertEqual(checkpoint.doc_count, 1)
checkpoint = _generate_incremental_export(
self.incremental_export,
last_doc_date=self.now - timedelta(hours=2, minutes=1))
data = checkpoint.get_blob().read().decode("utf-8-sig")
expected = "Foo column,Bar column\r\norange,pear\r\npeach,plumb\r\n"
self.assertEqual(data, expected)
self.assertEqual(checkpoint.doc_count, 2)
self.assertEqual(self.incremental_export.checkpoints.count(), 3)
def test_sending_success(self):
self._test_sending(200, IncrementalExportStatus.SUCCESS)
def test_sending_fail(self):
self._test_sending(401, IncrementalExportStatus.FAILURE)
def _test_sending(self, status_code, expected_status):
checkpoint = self.test_initial()
with requests_mock.Mocker() as m:
m.post('http://somewhere/', status_code=status_code)
_send_incremental_export(self.incremental_export, checkpoint)
checkpoint.refresh_from_db()
self.assertEqual(checkpoint.status, expected_status)
self.assertEqual(checkpoint.request_log.response_status,
status_code)
def test_owner_filter(self):
setup_locations_and_types(
self.domain, ['state', 'health-department', 'team', 'sub-team'],
[], [('State1', [
('HealthDepartment1', [
('Team1', [
('SubTeam1', []),
('SubTeam2', []),
]),
('Team2', []),
]),
])])
team1 = SQLLocation.objects.filter(domain=self.domain,
name='Team1').first()
health_department = SQLLocation.objects.filter(
domain=self.domain, name='HealthDepartment1').first()
self.addCleanup(delete_all_locations)
user = CommCareUser.create(self.domain,
'm2',
'abc',
None,
None,
location=team1)
send_to_elasticsearch('users', user.to_json())
self.es.indices.refresh(USER_INDEX_INFO.index)
self.addCleanup(delete_all_users)
cases = [
new_case(
domain=self.domain,
foo="peach",
bar="plumb",
server_modified_on=datetime.utcnow() + timedelta(hours=-1),
owner_id='123',
),
new_case(
domain=self.domain,
foo="orange",
bar="melon",
server_modified_on=datetime.utcnow(),
owner_id=user.
user_id, # this user is part of the team1 location.
),
new_case(
domain=self.domain,
foo="grape",
bar="pineapple",
server_modified_on=datetime.utcnow(),
),
]
for case in cases:
send_to_elasticsearch("cases", case.to_json())
self.addCleanup(self._cleanup_case(case.case_id))
self.es.indices.refresh(CASE_INDEX_INFO.index)
self.export_instance.filters.show_project_data = False
self.export_instance.filters.locations = [
health_department.location_id
]
self.export_instance.filters.users = ['123']
self.export_instance.save()
checkpoint = _generate_incremental_export(self.incremental_export)
data = checkpoint.get_blob().read().decode("utf-8-sig")
expected = "Foo column,Bar column\r\npeach,plumb\r\norange,melon\r\n"
self.assertEqual(data, expected)