def setUpClass(cls):
super(TestExportInstanceDBAccessors, cls).setUpClass()
cls.form_instance_deid = FormExportInstance(domain=cls.domain,
name='Forms',
is_deidentified=True)
cls.form_instance_wrong = FormExportInstance(
domain='wrong-domain',
name='Forms',
)
cls.form_instance_daily_saved = FormExportInstance(
domain='wrong-domain',
is_daily_saved_export=True,
)
cls.case_instance_deid = CaseExportInstance(domain=cls.domain,
name='Cases',
is_deidentified=True)
cls.case_instance = CaseExportInstance(domain=cls.domain,
name='Cases',
is_deidentified=False)
cls.case_instance_daily_saved = CaseExportInstance(
domain='wrong-domain',
is_daily_saved_export=True,
)
cls.instances = [
cls.form_instance_deid,
cls.form_instance_wrong,
cls.form_instance_daily_saved,
cls.case_instance,
cls.case_instance_deid,
cls.case_instance_daily_saved,
]
for instance in cls.instances:
instance.save()
def setUpClass(cls):
super(TestExportInstanceDBAccessors, cls).setUpClass()
cls.form_instance = FormExportInstance(
domain=cls.domain,
name='Forms',
is_deidentified=False
)
cls.form_instance_deid = FormExportInstance(
domain=cls.domain,
name='Forms',
is_deidentified=True
)
cls.form_instance_wrong = FormExportInstance(
domain='wrong-domain',
name='Forms',
)
cls.form_instance_daily_saved = FormExportInstance(
domain='wrong-domain',
is_daily_saved_export=True,
auto_rebuild_enabled=True,
last_accessed=datetime.utcnow()
)
cls.case_instance_deid = CaseExportInstance(
domain=cls.domain,
name='Cases',
is_deidentified=True
)
cls.case_instance = CaseExportInstance(
domain=cls.domain,
name='Cases',
is_deidentified=False
)
cls.case_instance_daily_saved = CaseExportInstance(
domain='wrong-domain',
is_daily_saved_export=True,
auto_rebuild_enabled=True,
last_accessed=(datetime.utcnow() - timedelta(days=4))
)
cls.instances = [
cls.form_instance,
cls.form_instance_deid,
cls.form_instance_wrong,
cls.form_instance_daily_saved,
cls.case_instance,
cls.case_instance_deid,
cls.case_instance_daily_saved,
]
for instance in cls.instances:
instance.save()
def get_instance(cls, domain_name):
return CaseExportInstance(
domain=domain_name,
is_odata_config=True,
transform_dates=False,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='closed',
selected=True,
# this is what exports generate for a base level property
item=ExportItem(path=[PathNode(name='closed')])),
ExportColumn(
label='date_modified',
selected=True,
item=ExportItem(
path=[PathNode(name='date_modified')])),
ExportColumn(label='selected_property_1',
selected=True),
ExportColumn(label='selected_property_2',
selected=True),
ExportColumn(label='unselected_property'),
],
),
])
def test_ignore_case_link_label(self):
export_with_case_link = CaseExportInstance(
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='my_case_link',
item=ExportItem(
path=[
PathNode(name='_id')
],
transform=CASE_ID_TO_LINK,
),
selected=True,
)
]
)
]
)
export_with_case_link.save()
self.addCleanup(export_with_case_link.delete)
cleaned_export = CaseExportInstance.get(export_with_case_link.get_id)
self.assertEqual(cleaned_export.tables[0].columns[0].label, 'my_case_link')
def setUp(cls):
cls.new_exports = [
FormExportInstance(),
CaseExportInstance(),
]
for export in cls.new_exports:
export.save()
def test_populated_metadata_document(self):
odata_config = CaseExportInstance(
_id='my_config_id',
domain=self.domain.name,
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='closed',
selected=True,
# this is what exports generate for a base level property
item=ExportItem(path=[PathNode(name='closed')])),
ExportColumn(
label='date_modified',
selected=True,
item=ExportItem(
path=[PathNode(name='date_modified')])),
ExportColumn(label='selected_property_1',
selected=True),
ExportColumn(label='selected_property_2',
selected=True),
ExportColumn(label='unselected_property'),
],
),
])
odata_config.save()
self.addCleanup(odata_config.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
def setUpClass(cls):
super().setUpClass()
with trap_extra_setup(ConnectionError,
msg="cannot connect to elasicsearch"):
cls.es = get_es_new()
initialize_index_and_mapping(cls.es, CASE_INDEX_INFO)
cls.domain = uuid.uuid4().hex
now = datetime.utcnow()
cases = [
new_case(domain=cls.domain,
foo="apple",
bar="banana",
server_modified_on=now - timedelta(hours=3)),
new_case(domain=cls.domain,
foo="orange",
bar="pear",
server_modified_on=now - timedelta(hours=2)),
]
for case in cases:
send_to_elasticsearch('cases', case.to_json())
cls.es.indices.refresh(CASE_INDEX_INFO.index)
cls.export_instance = CaseExportInstance(
export_format=Format.UNZIPPED_CSV,
domain=cls.domain,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(path=[PathNode(name="foo")]),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(path=[PathNode(name="bar")]),
selected=True,
)
])
])
cls.export_instance.save()
cls.incremental_export = IncrementalExport.objects.create(
domain=cls.domain,
name='test_export',
export_instance_id=cls.export_instance.get_id,
connection_settings=ConnectionSettings.objects.create(
domain=cls.domain,
name='test conn',
url='http://somewhere',
auth_type=BASIC_AUTH,
))
def setUpClass(cls):
cls.exports = [
FormExportInstance(is_daily_saved_export=True),
FormExportInstance(is_daily_saved_export=False),
CaseExportInstance(is_daily_saved_export=True),
]
for export in cls.exports:
export.save()
def test_populated_metadata_document(self):
odata_config_1 = CaseExportInstance(
_id='odata_config_1',
domain=self.domain.name,
is_odata_config=True,
tables=[TableConfiguration(columns=[])])
odata_config_1.save()
self.addCleanup(odata_config_1.delete)
odata_config_2 = CaseExportInstance(
_id='odata_config_2',
domain=self.domain.name,
is_odata_config=True,
tables=[
TableConfiguration(columns=[
ExportColumn(label='selected_property_1', selected=True),
ExportColumn(label='selected_property_2', selected=True),
ExportColumn(label='unselected_property'),
], ),
])
odata_config_2.save()
self.addCleanup(odata_config_2.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
with flag_enabled('ODATA'):
with patch(
'corehq.apps.api.odata.views.get_odata_case_configs_by_domain',
return_value=sorted(get_odata_case_configs_by_domain(
self.domain.name),
key=lambda _config: _config.get_id)):
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
def test_populated_metadata_document(self):
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
def setUpClass(cls):
super(TestDailySavedExports, cls).setUpClass()
cls.exports = [
FormExportInstance(is_daily_saved_export=True),
FormExportInstance(is_daily_saved_export=False),
CaseExportInstance(is_daily_saved_export=True),
]
for export in cls.exports:
export.save()
def test_selected_false(self):
export_json = get_export_json(
CaseExportInstance(export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(label="My table",
selected=False,
path=[],
columns=[])
]))
self.assertEqual(export_json, {})
def test_populated_service_document(self):
odata_config_1 = CaseExportInstance(domain=self.domain.name, is_odata_config=True)
odata_config_1.save()
self.addCleanup(odata_config_1.delete)
odata_config_2 = CaseExportInstance(domain=self.domain.name, is_odata_config=True)
odata_config_2.save()
self.addCleanup(odata_config_2.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain', is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
with flag_enabled('ODATA'):
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['OData-Version'], '4.0')
response_content = json.loads(response.content.decode('utf-8'))
self.assertCountEqual(response_content, ['@odata.context', 'value'])
self.assertEqual(
response_content['@odata.context'],
'http://localhost:8000/a/test_domain/api/v0.5/odata/cases/$metadata'
)
self.assertCountEqual(response_content['value'], [
{
'url': odata_config_1.get_id,
'kind': 'EntitySet',
'name': odata_config_1.get_id,
},
{
'url': odata_config_2.get_id,
'kind': 'EntitySet',
'name': odata_config_2.get_id,
},
])
def test_get_export_file(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
],
[] # No filters
)
with export_file as export:
self.assertEqual(
json.loads(export.read()),
{
u'My table': {
u'headers': [
u'Foo column',
u'Bar column'],
u'rows': [
[u'apple', u'banana'],
[u'apple', u'banana'],
[u'apple', u'banana'],
],
}
}
)
def test_missing_value_is_null(self):
self.assertEqual(
ODataCaseSerializer.serialize_cases_using_config(
[{}],
CaseExportInstance(tables=[
TableConfiguration(columns=[
ExportColumn(
label='owner-name-label',
item=ExportItem(path=[PathNode(
name='owner_name')]),
selected=True,
)
])
])), [{
'owner-name-label': '---'
}])
def test_config_in_different_domain(self):
export_config_in_other_domain = CaseExportInstance(
_id='config_id',
tables=[TableConfiguration(columns=[])],
case_type='my_case_type',
domain='different_domain')
export_config_in_other_domain.save()
self.addCleanup(export_config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self.client.get(
self._odata_feed_url_by_domain_and_config_id(
self.domain.name, export_config_in_other_domain.get_id),
HTTP_AUTHORIZATION='Basic ' + correct_credentials,
)
self.assertEqual(response.status_code, 404)
def test_user_permissions(self):
self.web_user.set_role(self.domain.name, 'none')
self.web_user.save()
self.addCleanup(self._setup_user_permissions)
export_config = CaseExportInstance(
_id='config_id',
tables=[TableConfiguration(columns=[])],
case_type='my_case_type',
domain=self.domain.name,
)
export_config.save()
self.addCleanup(export_config.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 403)
def test_unselected_column_excluded(self):
self.assertEqual(
ODataCaseSerializer.serialize_cases_using_config(
[{
'owner_name': 'owner-name-value',
'properties': {}
}],
CaseExportInstance(tables=[
TableConfiguration(columns=[
ExportColumn(
label='owner-name-label',
item=ExportItem(path=[PathNode(
name='owner_name')]),
selected=False,
)
])
])), [{}])
def test_case_name(self):
self.assertEqual(
ODataCaseSerializer.serialize_cases_using_config(
[{
'name': 'case-name-value'
}],
CaseExportInstance(tables=[
TableConfiguration(columns=[
ExportColumn(
label='case-name-label',
item=ExportItem(path=[PathNode(name='name')]),
selected=True,
)
])
])), [{
'case-name-label': 'case-name-value'
}])
def test_selected_false(self):
export_file = get_export_file(
[
CaseExportInstance(export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(label="My table",
selected=False,
path=[],
columns=[])
]),
],
[] # No filters
)
with export_file as export:
self.assertEqual(json.loads(export.read()), {})
def test_caseid_column_label(self):
export_with_modified_caseid_column = CaseExportInstance(
is_odata_config=True,
tables=[
TableConfiguration(columns=[
ExportColumn(
label='modified_case_id_column',
item=ExportItem(path=[PathNode(name='_id')]),
selected=True,
)
])
])
export_with_modified_caseid_column.save()
self.addCleanup(export_with_modified_caseid_column.delete)
cleaned_export = CaseExportInstance.get(
export_with_modified_caseid_column.get_id)
self.assertEqual(cleaned_export.tables[0].columns[0].label, 'caseid')
def test_get_export_file(self):
export_json = get_export_json(
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
)
self.assertEqual(
export_json,
{
'My table': {
'headers': [
'Foo column',
'Bar column'],
'rows': [
['apple', 'banana'],
['apple', 'banana'],
['apple', 'banana'],
],
}
}
)
def test_non_standard_case_property(self):
self.assertEqual(
ODataCaseSerializer.serialize_cases_using_config(
[{
'property_1': 'property-1-value'
}],
CaseExportInstance(tables=[
TableConfiguration(columns=[
ExportColumn(
label='property-1-label',
item=ExportItem(path=[PathNode(
name='property_1')]),
selected=True,
)
])
])), [{
'property-1-label': 'property-1-value'
}])
def test_export_transforms(self, _):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="DEID Date Transform column",
item=ExportItem(
path=[PathNode(name="date")]
),
selected=True,
deid_transform=DEID_DATE_TRANSFORM,
)
]
)]
),
],
[] # No filters
)
with export_file as export:
export_dict = json.loads(export.read())
export_dict['My table']['rows'].sort()
self.assertEqual(
export_dict,
{
u'My table': {
u'headers': [
u'DEID Date Transform column [sensitive]',
],
u'rows': [
[MISSING_VALUE],
[u'2016-04-07'],
[u'2016-04-27'], # offset by 3 since that's the mocked random offset
],
}
}
)
def setUp(self):
super().setUp()
self.export_instance = CaseExportInstance(
export_format=Format.UNZIPPED_CSV,
domain=self.domain,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(path=[PathNode(name="foo")]),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(path=[PathNode(name="bar")]),
selected=True,
)
])
])
self.export_instance.save()
connection_settings = ConnectionSettings.objects.create(
domain=self.domain,
name='test conn',
url='http://commcarehq.org',
auth_type=BASIC_AUTH,
username='******',
password='******',
)
self.incremental_export = IncrementalExport.objects.create(
domain=self.domain,
name='test_export',
export_instance_id=self.export_instance.get_id,
connection_settings=connection_settings,
)
def test_request_succeeded(self):
export_config = CaseExportInstance(
_id='config_id',
tables=[TableConfiguration(columns=[])],
case_type='my_case_type',
domain=self.domain.name,
)
export_config.save()
self.addCleanup(export_config.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'],
'application/json; charset=utf-8')
self.assertEqual(response['OData-Version'], '4.0')
self.assertEqual(
json.loads(response.content.decode('utf-8')), {
'@odata.context':
'http://localhost:8000/a/test_domain/api/v0.5/odata/cases/config_id/$metadata#feed',
'value': []
})
def test_export_transforms(self, export_save, _):
export_json = get_export_json(
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="DEID Date Transform column",
item=ExportItem(path=[PathNode(name="date")]),
selected=True,
deid_transform=DEID_DATE_TRANSFORM,
)
])
]), )
export_json['My table']['rows'].sort()
self.assertEqual(
export_json,
{
'My table': {
'headers': [
'DEID Date Transform column [sensitive]',
],
'rows': [
[MISSING_VALUE],
['2016-04-07'],
[
'2016-04-27'
], # offset by 3 since that's the mocked random offset
],
}
})
self.assertTrue(export_save.called)
def test_default_case_values(self):
# Confirm that CaseExportInstances set the default show_all_data flag correctly
case_export = CaseExportInstance()
case_export_wrapped = CaseExportInstance.wrap({})
for e in [case_export, case_export_wrapped]:
self.assertTrue(e.filters.show_all_data)
def test_simple_bulk_export(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
selected=True,
label="My table",
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
]
)]
),
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=MAIN_TABLE,
columns=[
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
],
[] # No filters
)
expected = {
'Export1-My table': {
"A1": "Foo column",
"A2": "apple",
"A3": "apple",
"A4": "apple",
},
"Export2-My table": {
"A1": "Bar column",
"A2": "banana",
"A3": "banana",
"A4": "banana",
},
}
with export_file as export:
wb = load_workbook(export)
self.assertEqual(wb.get_sheet_names(), ["Export1-My table", "Export2-My table"])
for sheet in expected.keys():
for cell in expected[sheet].keys():
self.assertEqual(
wb[sheet][cell].value,
expected[sheet][cell],
'AssertionError: Sheet "{}", cell "{}" expected: "{}", got "{}"'.format(
sheet, cell, expected[sheet][cell], wb[sheet][cell].value
)
)