def test_inferred_schema_merge(self):
schema = CaseExportDataSchema(
domain='my-domain',
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[ExportItem(path=[PathNode(name='case_property')])],
)
])
inferred_schema = CaseExportDataSchema(
domain='my-domain',
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[PathNode(name='case_property')],
inferred=True,
)
],
inferred=True,
)
])
merged = ExportDataSchema._merge_schemas(schema, inferred_schema)
self.assertEqual(len(merged.group_schemas), 1)
self.assertTrue(merged.group_schemas[0].inferred)
group_schema = merged.group_schemas[0]
self.assertEqual(len(group_schema.items), 1)
self.assertTrue(group_schema.items[0].inferred)
def get_instance(cls, domain_name):
return CaseExportInstance(
domain=domain_name,
is_odata_config=True,
transform_dates=False,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='closed',
selected=True,
# this is what exports generate for a base level property
item=ExportItem(path=[PathNode(name='closed')])),
ExportColumn(
label='date_modified',
selected=True,
item=ExportItem(
path=[PathNode(name='date_modified')])),
ExportColumn(label='selected_property_1',
selected=True),
ExportColumn(label='selected_property_2',
selected=True),
ExportColumn(label='unselected_property'),
],
),
])
def setUpClass(cls):
super(TestConversionOrdering, cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[PathNode(name='form'), PathNode(name='question1')],
label='q1',
),
ExportItem(
path=[PathNode(name='form'), PathNode(name='question2')],
label='q2',
),
ExportItem(
path=[PathNode(name='form'), PathNode(name='other')],
label='other',
),
ExportItem(
path=[PathNode(name='form'), PathNode(name='question3')],
label='q3',
),
],
last_occurrences={cls.app_id: 2},
),
]
)
def test_partial_match_ordering(self):
schema = self._create_schema([
ExportItem(path=[PathNode(name='two')]),
ExportItem(path=[PathNode(name='one')]),
ExportItem(path=[PathNode(name='three')]),
])
ordered_schema = self._create_schema([
ExportItem(path=[PathNode(name='one')]),
ExportItem(path=[PathNode(name='four')]),
ExportItem(path=[PathNode(name='five')]),
ExportItem(path=[PathNode(name='six')]),
])
schema = CaseExportDataSchema._reorder_schema_from_schema(
schema,
ordered_schema,
)
self._assert_item_order(
schema,
[],
[
ExportItem(path=[PathNode(name='one')]),
ExportItem(path=[PathNode(name='two')]),
ExportItem(path=[PathNode(name='three')]),
],
)
def get_instance(cls, domain_name):
return FormExportInstance(
domain=domain_name,
is_odata_config=True,
transform_dates=False,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(label='received_on',
selected=True,
item=ExportItem(
path=[PathNode(name='received_on')])),
ExportColumn(label='started_time',
selected=True,
item=ExportItem(path=[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='timeStart'),
])),
ExportColumn(label='selected_property_1',
selected=True),
ExportColumn(label='selected_property_2',
selected=True),
ExportColumn(label='unselected_property'),
],
),
])
def test_get_case_properties_for_case_type(self):
schema = CaseExportDataSchema(
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[PathNode(name='name')],
label='name',
last_occurrences={},
),
ExportItem(
path=[PathNode(name='color')],
label='color',
last_occurrences={},
),
],
last_occurrences={},
),
],
)
with mock.patch(
'corehq.apps.export.models.new.CaseExportDataSchema.generate_schema_from_builds',
return_value=schema):
case_types = get_case_properties_for_case_type('test-domain', 'case-type')
self.assertEqual(sorted(case_types), ['color', 'name'])
def setUpClass(cls):
super().setUpClass()
with trap_extra_setup(ConnectionError,
msg="cannot connect to elasicsearch"):
cls.es = get_es_new()
initialize_index_and_mapping(cls.es, CASE_INDEX_INFO)
cls.domain = uuid.uuid4().hex
now = datetime.utcnow()
cases = [
new_case(domain=cls.domain,
foo="apple",
bar="banana",
server_modified_on=now - timedelta(hours=3)),
new_case(domain=cls.domain,
foo="orange",
bar="pear",
server_modified_on=now - timedelta(hours=2)),
]
for case in cases:
send_to_elasticsearch('cases', case.to_json())
cls.es.indices.refresh(CASE_INDEX_INFO.index)
cls.export_instance = CaseExportInstance(
export_format=Format.UNZIPPED_CSV,
domain=cls.domain,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(path=[PathNode(name="foo")]),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(path=[PathNode(name="bar")]),
selected=True,
)
])
])
cls.export_instance.save()
cls.incremental_export = IncrementalExport.objects.create(
domain=cls.domain,
name='test_export',
export_instance_id=cls.export_instance.get_id,
connection_settings=ConnectionSettings.objects.create(
domain=cls.domain,
name='test conn',
url='http://somewhere',
auth_type=BASIC_AUTH,
))
def setUpClass(cls):
cls.current_app = Application.wrap(cls.get_json('basic_application'))
cls.first_build = Application.wrap(cls.get_json('basic_application'))
cls.first_build._id = '123'
cls.first_build.copy_of = cls.current_app.get_id
cls.first_build.version = 3
cls.advanced_app = Application.new_app('domain',
"Untitled Application")
module = cls.advanced_app.add_module(
AdvancedModule.new_module('Untitled Module', None))
form = module.new_form("Untitled Form",
cls.get_xml('repeat_group_form'))
form.xmlns = 'repeat-xmlns'
form.actions.open_cases = [
AdvancedOpenCaseAction(
case_type="advanced",
case_tag="open_case_0",
name_path="/data/question3/question4",
repeat_context="/data/question3",
)
]
cls.apps = [
cls.current_app,
cls.first_build,
cls.advanced_app,
]
with drop_connected_signals(app_post_save):
for app in cls.apps:
app.save()
cls.inferred_schema = InferredSchema(
domain=cls.domain,
case_type=cls.case_type,
group_schemas=[
InferredExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property')
],
label='Inferred 1',
inferred=True),
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property_2')
],
label='Inferred 1',
inferred=True),
],
inferred=True),
])
cls.inferred_schema.save()
def test_get_export_file(self):
export_file = get_export_file(
[
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
],
[] # No filters
)
with export_file as export:
self.assertEqual(
json.loads(export.read()),
{
u'My table': {
u'headers': [
u'Foo column',
u'Bar column'],
u'rows': [
[u'apple', u'banana'],
[u'apple', u'banana'],
[u'apple', u'banana'],
],
}
}
)
def setUpClass(cls):
super(TestExportInstanceGenerationMultipleApps, cls).setUpClass()
cls.app_id = '1234'
cls.second_app_id = '5678'
cls.schema = FormExportDataSchema(group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='question1')
],
label='Question 1',
last_occurrences={
cls.app_id: 2,
cls.second_app_id: 4
},
)
],
last_occurrences={
cls.app_id: 2,
cls.second_app_id: 4,
},
),
ExportGroupSchema(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='q2')
],
label='Question 2',
last_occurrences={
cls.app_id: 3,
},
)
],
last_occurrences={
cls.app_id: 3,
},
),
], )
def setUp(self):
self.item = ExportItem(
path=[PathNode(name='data'),
PathNode(name='question1')],
label='Question One',
last_occurrences={self.app_id: 3},
)
def test_formid_column_label(self):
export_with_modified_formid_column = FormExportInstance(
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='modified_form_id_column',
item=ExportItem(
path=[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='instanceID')
]
),
selected=True,
)
]
)
]
)
export_with_modified_formid_column.save()
self.addCleanup(export_with_modified_formid_column.delete)
cleaned_export = FormExportInstance.get(export_with_modified_formid_column.get_id)
self.assertEqual(cleaned_export.tables[0].columns[0].label, 'formid')
def test_ignore_form_link_label(self):
export_with_form_link = FormExportInstance(
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='my_form_link',
item=ExportItem(
path=[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='instanceID')
],
transform=FORM_ID_TO_LINK,
),
selected=True,
)
]
)
]
)
export_with_form_link.save()
self.addCleanup(export_with_form_link.delete)
cleaned_export = FormExportInstance.get(export_with_form_link.get_id)
self.assertEqual(cleaned_export.tables[0].columns[0].label, 'my_form_link')
def setUpClass(cls):
super(TestConvertSavedExportSchemaToCaseExportInstance,
cls).setUpClass()
cls.project = create_domain(cls.domain)
cls.project.commtrack_enabled = True
cls.project.save()
cls.schema = CaseExportDataSchema(
domain=cls.domain,
case_type='wonderwoman',
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[PathNode(name='DOB')],
label='Case Propery DOB',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 3},
),
ExportGroupSchema(
path=CASE_HISTORY_TABLE,
last_occurrences={cls.app_id: 3},
),
ExportGroupSchema(
path=PARENT_CASE_TABLE,
last_occurrences={cls.app_id: 3},
),
],
)
def setUpClass(cls):
super(TestSingleNodeRepeatConversion, cls).setUpClass()
cls.schema = FormExportDataSchema(
domain=cls.domain,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[],
last_occurrences={cls.app_id: 2},
),
ExportGroupSchema(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True)
],
items=[
ExportItem(
path=[
PathNode(name='form'),
PathNode(name='repeat', is_repeat=True),
PathNode(name='single_answer')
],
label='Single Answer',
last_occurrences={cls.app_id: 2},
)
],
last_occurrences={cls.app_id: 2},
),
])
def test_populated_metadata_document(self):
odata_config = CaseExportInstance(
_id='my_config_id',
domain=self.domain.name,
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(
label='closed',
selected=True,
# this is what exports generate for a base level property
item=ExportItem(path=[PathNode(name='closed')])),
ExportColumn(
label='date_modified',
selected=True,
item=ExportItem(
path=[PathNode(name='date_modified')])),
ExportColumn(label='selected_property_1',
selected=True),
ExportColumn(label='selected_property_2',
selected=True),
ExportColumn(label='unselected_property'),
],
),
])
odata_config.save()
self.addCleanup(odata_config.delete)
non_odata_config = CaseExportInstance(domain=self.domain.name)
non_odata_config.save()
self.addCleanup(non_odata_config.delete)
config_in_other_domain = CaseExportInstance(domain='other_domain',
is_odata_config=True)
config_in_other_domain.save()
self.addCleanup(config_in_other_domain.delete)
correct_credentials = self._get_correct_credentials()
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml('populated_case_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA), response.content)
def test_get_export_file(self):
export_json = get_export_json(
CaseExportInstance(
export_format=Format.JSON,
domain=DOMAIN,
case_type=DEFAULT_CASE_TYPE,
tables=[TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(
path=[PathNode(name="foo")]
),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(
path=[PathNode(name="bar")]
),
selected=True,
)
]
)]
),
)
self.assertEqual(
export_json,
{
'My table': {
'headers': [
'Foo column',
'Bar column'],
'rows': [
['apple', 'banana'],
['apple', 'banana'],
['apple', 'banana'],
],
}
}
)
def test_get_value_with_text(self):
column = ExportColumn(item=ExportItem(
path=[PathNode(name='form'),
PathNode(name='q1')], ), )
doc = {"q1": {'#text': "answer"}}
self.assertEqual(
column.get_value('domain', 'docid', doc, [PathNode(name='form')]),
"answer",
)
def setUpClass(cls):
super(TestExportInstanceGenerationWithInferredSchema, cls).setUpClass()
cls.schema = CaseExportDataSchema(
app_id=cls.app_id,
case_type=cls.case_type,
group_schemas=[
ExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(
path=[
PathNode(name='data'),
PathNode(name='case_property')
],
label='Question 1',
last_occurrences={cls.app_id: 3},
),
],
last_occurrences={cls.app_id: 3},
),
],
)
cls.inferred_schema = InferredSchema(
case_type=cls.case_type,
group_schemas=[
InferredExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property')
],
label='Inferred 1',
inferred=True),
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property_2')
],
label='Inferred 1',
inferred=True),
],
inferred=True),
])
def test_removes_html_from_label(self):
question = {
'label': '<span style="color:#ffffff">Enter a number</span>',
'value': '/data/enter_number',
'type': 'Int',
}
item = ExportItem.create_from_question(question, 'app_id',
'app_ersion', [])
self.assertEqual(item.label, 'Enter a number')
def get_case_name_column(case_id_export_item):
label_segments = case_id_export_item.readable_path.split('.')[:-1]
label_segments.append('case_name')
return ExportColumn(
tags=[PROPERTY_TAG_CASE],
label='.'.join(label_segments),
item=ExportItem(path=case_id_export_item.path, transform=CASE_NAME_TRANSFORM),
selected=False,
is_advanced=True,
help_text=_("The name of the case that this form operated on")
)
def test_populated_metadata_document(self):
odata_config = FormExportInstance(
_id='my_config_id',
domain=self.domain.name,
is_odata_config=True,
tables=[
TableConfiguration(
selected=True,
columns=[
ExportColumn(label='received_on', selected=True,
item=ExportItem(path=[PathNode(name='received_on')])),
ExportColumn(label='started_time', selected=True,
item=ExportItem(path=[
PathNode(name='form'),
PathNode(name='meta'),
PathNode(name='timeStart'),
])),
ExportColumn(label='selected_property_1', selected=True),
ExportColumn(label='selected_property_2', selected=True),
ExportColumn(label='unselected_property'),
],
),
]
)
odata_config.save()
self.addCleanup(odata_config.delete)
correct_credentials = self._get_correct_credentials()
with flag_enabled('BI_INTEGRATION_PREVIEW', is_preview=True):
response = self._execute_query(correct_credentials)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/xml')
self.assertEqual(response['OData-Version'], '4.0')
self.assertXmlEqual(
self.get_xml(
'populated_form_odata_metadata_document_from_config',
override_path=PATH_TO_TEST_DATA
),
response.content
)
def test_get_value(self):
column = SplitUserDefinedExportColumn(
split_type=MULTISELCT_USER_DEFINED_SPLIT_TYPE,
item=ExportItem(path=[PathNode(name='form'), PathNode(name='mc')]),
user_defined_options=['one', 'two', 'three'],
)
result = column.get_value('domain', 'docid', {'form': {'mc': 'one two extra'}}, [])
self.assertEqual(result, [1, 1, None, 'extra'])
column.split_type = PLAIN_USER_DEFINED_SPLIT_TYPE
result = column.get_value('domain', 'docid', {'form': {'mc': 'one two extra'}}, [])
self.assertEqual(result, 'one two extra')
def setUp(self):
super().setUp()
self.export_instance = CaseExportInstance(
export_format=Format.UNZIPPED_CSV,
domain=self.domain,
case_type=DEFAULT_CASE_TYPE,
tables=[
TableConfiguration(
label="My table",
selected=True,
path=[],
columns=[
ExportColumn(
label="Foo column",
item=ExportItem(path=[PathNode(name="foo")]),
selected=True,
),
ExportColumn(
label="Bar column",
item=ExportItem(path=[PathNode(name="bar")]),
selected=True,
)
])
])
self.export_instance.save()
connection_settings = ConnectionSettings.objects.create(
domain=self.domain,
name='test conn',
url='http://commcarehq.org',
auth_type=BASIC_AUTH,
username='******',
password='******',
)
self.incremental_export = IncrementalExport.objects.create(
domain=self.domain,
name='test_export',
export_instance_id=self.export_instance.get_id,
connection_settings=connection_settings,
)
def setUpClass(cls):
cls.current_app = Application.wrap(cls.get_json('basic_application'))
cls.first_build = Application.wrap(cls.get_json('basic_application'))
cls.first_build._id = '123'
cls.first_build.copy_of = cls.current_app.get_id
cls.first_build.version = 3
cls.apps = [
cls.current_app,
cls.first_build,
]
with drop_connected_signals(app_post_save):
for app in cls.apps:
app.save()
cls.inferred_schema = InferredSchema(
domain=cls.domain,
case_type=cls.case_type,
group_schemas=[
InferredExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property')
],
label='Inferred 1',
inferred=True),
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property_2')
],
label='Inferred 1',
inferred=True),
],
inferred=True),
])
cls.inferred_schema.save()
def test_unselected_column_excluded(self):
self.assertEqual(
ODataFormSerializer.serialize_forms_using_config(
[{
'user_id': 'the-user-id'
}],
FormExportInstance(tables=[
TableConfiguration(columns=[
ExportColumn(
label='user-id',
item=ExportItem(path=[PathNode(name='user_id')]),
selected=False,
)
])
])), [{}])
def test_missing_value_is_null(self):
self.assertEqual(
ODataFormSerializer.serialize_forms_using_config(
[{}],
FormExportInstance(tables=[
TableConfiguration(columns=[
ExportColumn(
label='user-id',
item=ExportItem(path=[PathNode(name='user_id')]),
selected=True,
)
])
])), [{
'user-id': '---'
}])
def test_get_headers(self, _):
column = StockExportColumn(
domain=self.domain,
label="Stock",
item=ExportItem(),
)
with patch(
'corehq.apps.export.models.new.get_ledger_section_entry_combinations',
return_value=[
MockLedgerValue(section_id='123', entry_id='456'),
MockLedgerValue(section_id='abc', entry_id='def'),
]):
headers = list(column.get_headers())
self.assertEqual(headers, ['water (123)', 'water (abc)'])
def setUp(self):
self.inferred_schema = CaseInferredSchema(
domain=self.domain,
case_type=self.case_type,
group_schemas=[
InferredExportGroupSchema(
path=MAIN_TABLE,
items=[
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property')
],
label='Inferred 1',
inferred=True),
ExportItem(path=[
PathNode(name='data'),
PathNode(name='case_property_2')
],
label='Inferred 1',
inferred=True),
],
inferred=True),
])
self.inferred_schema.save()
def test_table_containing_duplicate_paths_with_differing_doc_types_can_find_either(self):
path = [PathNode(name='closed')]
prop1 = ScalarItem(path=path)
prop2 = ExportItem(path=path)
table_config = TableConfiguration(
columns=[
ExportColumn(item=prop1), ExportColumn(item=prop2)
]
)
scalarIndex, _ = table_config.get_column(path, 'ScalarItem', None)
exportIndex, _ = table_config.get_column(path, 'ExportItem', None)
self.assertEqual(scalarIndex, 0)
self.assertEqual(exportIndex, 1)
def test_wrap_export_item(self, _):
path = [PathNode(name="foo"), PathNode(name="bar")]
item = ExportItem(path=path)
wrapped = ExportItem.wrap(item.to_json())
self.assertEqual(type(wrapped), type(item))
self.assertEqual(wrapped.to_json(), item.to_json())
def test_wrap_export_item_child(self, _):
path = [PathNode(name="foo"), PathNode(name="bar")]
item = MultipleChoiceItem(path=path, options=[Option(value="foo")])
wrapped = ExportItem.wrap(item.to_json())
self.assertEqual(type(wrapped), type(item))
self.assertEqual(wrapped.to_json(), item.to_json())