def setUpClass(cls):
super(DataSourceConfigurationDbTest, cls).setUpClass()
# TODO - handle cleanup appropriately so this isn't needed
for data_source_config in DataSourceConfiguration.all():
data_source_config.delete()
DataSourceConfiguration(domain='foo', table_id='foo1', referenced_doc_type='XFormInstance').save()
DataSourceConfiguration(domain='foo', table_id='foo2', referenced_doc_type='XFormInstance').save()
DataSourceConfiguration(domain='bar', table_id='bar1', referenced_doc_type='XFormInstance').save()
def setUpClass(cls):
DataSourceConfiguration(domain='foo',
table_id='foo1',
referenced_doc_type='doc1').save()
DataSourceConfiguration(domain='foo',
table_id='foo2',
referenced_doc_type='doc2').save()
DataSourceConfiguration(domain='bar',
table_id='bar1',
referenced_doc_type='doc3').save()
def setUpClass(cls):
super(DataSourceConfigurationDbTest, cls).setUpClass()
DataSourceConfiguration(domain='foo',
table_id='foo1',
referenced_doc_type='XFormInstance').save()
DataSourceConfiguration(domain='foo',
table_id='foo2',
referenced_doc_type='XFormInstance').save()
DataSourceConfiguration(domain='bar',
table_id='bar1',
referenced_doc_type='XFormInstance').save()
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
# this is a hack to have both sql and es backends created in a class
# method. alternative would be to have these created on each test run
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_string',
"display_name": 'date_as_string',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_date',
"datatype": "date"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "my_datetime",
},
"column_id": "datetime_as_datetime",
"datatype": "datetime"
}
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def test_import_from_basic_definition(self):
spec = self.get_monthly_config_spec()
data_source = DataSourceConfiguration(
domain=spec.domain,
referenced_doc_type='CommCareCase',
table_id='some_table',
)
data_source.save()
self.addCleanup(data_source.delete)
# these just have to be valid data source objects
spec.primary_table.data_source_id = data_source._id
spec.secondary_tables[0].data_source_id = data_source._id
aggregate_table_definition = import_aggregation_models_from_spec(spec)
self.assertEqual(1, AggregateTableDefinition.objects.count())
table_def = AggregateTableDefinition.objects.get(
pk=aggregate_table_definition.pk)
self.assertEqual(data_source._id, table_def.primary_data_source_id.hex)
self.assertEqual(4, table_def.primary_columns.count())
aggregation = table_def.time_aggregation
self.assertEqual('month', aggregation.aggregation_unit)
self.assertEqual('opened_date', aggregation.start_column)
self.assertEqual('closed_date', aggregation.end_column)
self.assertEqual(1, table_def.secondary_tables.count())
secondary_table = table_def.secondary_tables.get()
self.assertEqual(data_source._id, secondary_table.data_source_id.hex)
self.assertEqual('doc_id', secondary_table.join_column_primary)
self.assertEqual('form.case.@case_id',
secondary_table.join_column_secondary)
self.assertEqual('received_on', secondary_table.time_window_column)
self.assertEqual(1, secondary_table.columns.count())
secondary_column = secondary_table.columns.get()
self.assertEqual('fu_forms_in_month', secondary_column.column_id)
def test_column_uniqueness_when_truncated(self):
problem_spec = {
"display_name": "practicing_lessons",
"property_name": "long_column",
"choices": [
"duplicate_choice_1",
"duplicate_choice_2",
],
"select_style": "multiple",
"column_id":
"a_very_long_base_selection_column_name_with_limited_room",
"type": "choice_list",
}
data_source_config = DataSourceConfiguration(
domain='test',
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=uuid.uuid4().hex,
configured_filter={},
configured_indicators=[problem_spec],
)
adapter = get_indicator_adapter(data_source_config)
adapter.rebuild_table()
# ensure we can save data to the table.
adapter.save({
'_id': uuid.uuid4().hex,
'domain': 'test',
'doc_type': 'CommCareCase',
'long_column': 'duplicate_choice_1',
})
adapter.refresh_table()
# and query it back
q = adapter.get_query_object()
self.assertEqual(1, q.count())
def test_array_type_column(self):
problem_spec = {
"column_id": "referral_health_problem",
"datatype": "array",
"type": "expression",
"expression": {
"type": "split_string",
"string_expression": {
"type": "property_name",
"property_name": "referral_health_problem",
}
},
}
data_source_config = DataSourceConfiguration(
domain='test',
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=uuid.uuid4().hex,
configured_filter={},
configured_indicators=[problem_spec],
)
adapter = get_indicator_adapter(data_source_config)
adapter.rebuild_table()
self.addCleanup(adapter.drop_table)
# ensure we can save data to the table.
adapter.save({
'_id': uuid.uuid4().hex,
'domain': 'test',
'doc_type': 'CommCareCase',
'referral_health_problem': 'bleeding convulsions',
})
# and query it back
qs = adapter.get_query_object()
self.assertEqual(1, qs.count())
self.assertEqual(qs.first().referral_health_problem, ['bleeding', 'convulsions'])
def get_form_data_source(app, form):
xform = XForm(form.source)
schema = FormExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
xform.data_node.tag_xmlns,
only_process_current_builds=True,
)
meta_properties = [
_export_column_to_ucr_indicator(c)
for c in BOTTOM_MAIN_FORM_TABLE_PROPERTIES if c.label != 'form_link'
]
dynamic_properties = _get_dynamic_indicators_from_export_schema(schema)
form_name = form.default_name()
config = DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(
xform.data_node.tag_xmlns, app.get_id),
configured_indicators=meta_properties + dynamic_properties +
_get_shared_indicators(),
)
return _deduplicate_columns_if_necessary(config)
def setUpClass(cls):
super(ReportTranslationTest, cls).setUpClass()
data_source = DataSourceConfiguration(
domain=cls.DOMAIN,
table_id="foo",
referenced_doc_type="CommCareCase",
)
data_source.save()
ReportConfiguration(domain=cls.DOMAIN,
config_id=data_source._id,
columns=[
{
"type": "field",
"field": "foo",
"column_id": "foo",
"aggregation": "simple",
"display": "My Column",
},
{
"type": "field",
"field": "bar",
"column_id": "bar",
"aggregation": "simple",
"display": {
"en": "Name",
"fra": "Nom"
},
},
]).save()
def setUp(self):
self.domain_obj = create_domain(self.domain)
es = get_es_new()
initialize_index_and_mapping(es, USER_INDEX_INFO)
self.region = LocationType.objects.create(domain=self.domain, name="region")
self.town = LocationType.objects.create(domain=self.domain, name="town", parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_location_pillow(ucr_configs=[self.data_source_config])
self.pillow.get_change_feed().get_latest_offsets()
def setUp(self):
self.domain_obj = create_domain(self.domain)
self.region = LocationType.objects.create(domain=self.domain,
name="region")
self.town = LocationType.objects.create(domain=self.domain,
name="town",
parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.data_source_config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_latest_offsets()
def setUpClass(cls):
super(TestExpandedColumn, cls).setUpClass()
cls.data_source_config = DataSourceConfiguration(
domain=cls.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=_clean_table_name(cls.domain, str(uuid.uuid4().hex)),
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": field
},
"column_id": field,
"display_name": field,
"datatype": "string"
} for field in ['my_field', 'field_name_with_CAPITAL_letters']],
)
cls.data_source_config.save()
def get_case_data_source(app, case_type):
schema = CaseExportDataSchema.generate_schema_from_builds(
app.domain,
app._id,
case_type,
only_process_current_builds=True,
)
# the first two (row number and case id) are redundant/export specific,
meta_properties_to_use = MAIN_CASE_TABLE_PROPERTIES[2:]
# anything with a transform should also be removed
meta_properties_to_use = [property_def for property_def in meta_properties_to_use if property_def.item.transform is None]
meta_indicators = [_export_column_to_ucr_indicator(c) for c in meta_properties_to_use]
dynamic_indicators = _get_dynamic_indicators_from_export_schema(schema)
# filter out any duplicately defined columns from dynamic indicators
meta_column_names = set([c['column_id'] for c in meta_indicators])
dynamic_indicators = [indicator for indicator in dynamic_indicators if
indicator['column_id'] not in meta_column_names]
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='CommCareCase',
table_id=clean_table_name(app.domain, case_type),
display_name=case_type,
configured_filter=make_case_data_source_filter(case_type),
configured_indicators=meta_indicators + dynamic_indicators + _get_shared_indicators(),
)
def _setup_config(self, doc_type, filter_):
return DataSourceConfiguration(
domain='test',
referenced_doc_type=doc_type,
table_id='blah',
configured_filter=filter_
)
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'indicator_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'city'
},
"column_id": 'indicator_col_id_city',
"display_name": 'indicator_display_name_city',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def _make_config(indicators):
return DataSourceConfiguration(domain=domain_name,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=clean_table_name(
domain_name,
str(uuid.uuid4().hex)),
configured_indicators=indicators)
def setUp(self):
super(ReportDataTest, self).setUp()
# Create report
self.domain = 'test-ucr-report-data'
self.data_source = DataSourceConfiguration(
domain=self.domain,
referenced_doc_type='CommCareCase',
table_id=uuid.uuid4().hex,
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'name'
},
"column_id": 'name',
"display_name": 'name',
"datatype": "string"
}, {
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'number',
"display_name": 'number',
"datatype": "integer"
}],
)
self.data_source.validate()
self.data_source.save()
IndicatorSqlAdapter(self.data_source).rebuild_table()
self.addCleanup(self.data_source.delete)
# initialize a report on the data
self.report_config = ReportConfiguration(
domain=self.domain,
config_id=self.data_source._id,
aggregation_columns=['doc_id'],
columns=[{
"type": "field",
"field": "name",
"column_id": "name",
"display": "Name",
"aggregation": "simple",
}, {
"type": "field",
"field": "number",
"column_id": "number",
"display": "Number",
"aggregation": "simple",
}],
filters=[],
configured_charts=[])
self.report_config.save()
self.addCleanup(self.report_config.delete)
def setUpClass(cls):
super(TestSimpleReportConfigurationResource, cls).setUpClass()
cls.report_columns = [{
"column_id": 'foo',
"display": "foo display",
"type": "field",
"field": "my_field",
"aggregation": "simple",
}, {
"column_id": 'bar',
"display": "bar display",
"type": "field",
"field": "my_field",
"aggregation": "simple",
}, {
"column_id": 'expand',
"display": "expand display",
"type": "expanded",
"field": "my_field",
"max_expansion": 10,
}]
cls.report_filters = [{
'datatype': 'integer',
'field': 'my_field',
'type': 'dynamic_choice_list',
'slug': 'my_field_filter',
}, {
'datatype': 'string',
'field': 'my_other_field',
'type': 'dynamic_choice_list',
'slug': 'my_other_field_filter',
}]
cls.report_title = "test report"
cls.data_source = DataSourceConfiguration(
domain=cls.domain.name,
referenced_doc_type="XFormInstance",
table_id=uuid.uuid4().hex,
)
cls.data_source.save()
cls.report_configuration = ReportConfiguration(
title=cls.report_title,
domain=cls.domain.name,
config_id=cls.data_source._id,
columns=cls.report_columns,
filters=cls.report_filters)
cls.report_configuration.save()
another_report_configuration = ReportConfiguration(
domain=cls.domain.name,
config_id=cls.data_source._id,
columns=[],
filters=[])
another_report_configuration.save()
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
config = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_string',
"display_name": 'date_as_string',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_date',
"datatype": "date"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "my_datetime",
},
"column_id": "datetime_as_datetime",
"datatype": "datetime"
}
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
cls.data_sources[UCR_SQL_BACKEND] = config
cls.adapters[UCR_SQL_BACKEND] = adapter
def get_case_data_source(app, case_type):
prop_map = get_case_properties(app, [case_type], defaults=list(DEFAULT_CASE_PROPERTY_DATATYPES))
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='CommCareCase',
table_id=_clean_table_name(app.domain, case_type),
display_name=case_type,
configured_filter=make_case_data_source_filter(case_type),
configured_indicators=[
make_case_property_indicator(property) for property in prop_map[case_type]
]
)
def test_filter(self):
slug = "hierarchy_filter_slug"
filter_spec = {
"type": "enikshay_location_hierarchy",
"display": "location hierarchy",
"datatype": "string",
"slug": slug,
"field": "does_not_matter",
}
data_source_config = DataSourceConfiguration(
domain=self.domain,
referenced_doc_type="",
table_id="123",
)
with patch(
"corehq.apps.userreports.reports.data_source.get_datasource_config",
MagicMock(return_value=(data_source_config, None))
):
report_config = ReportConfiguration(
config_id="123",
filters=[filter_spec]
)
report = ReportFactory().from_spec(report_config)
filter_values = get_filter_values(
report_config.ui_filters,
{slug: self.cto.location_id},
user=MagicMock(),
)
report.set_filter_values(filter_values)
expected_filter_vals = {
'{}_sto'.format(slug): self.sto.location_id,
'{}_cto'.format(slug): self.cto.location_id,
'{}_below_cto'.format(slug): self.cto.location_id,
}
expected_filter = OR([
EQ("sto", "{}_sto".format(slug)),
EQ("cto", "{}_cto".format(slug)),
EQ("below_cto", "{}_below_cto".format(slug))
])
self.assertEqual(len(report.data_source.filters), 1)
self.assertFiltersEqual(
report.data_source.filters[0],
expected_filter
)
self.assertEqual(
report.data_source.filter_values,
expected_filter_vals
)
def _create_data_source(cls):
cls.data_source = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'first_name'
},
"column_id": 'indicator_col_id_first_name',
"display_name": 'indicator_display_name_first_name',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'rank'
},
"column_id": 'indicator_col_id_rank',
"datatype": "integer"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
cls.adapter = get_indicator_adapter(cls.data_source)
def test_last_modified_date(self):
start = datetime.datetime.utcnow()
time.sleep(.01)
data_source = DataSourceConfiguration(domain='mod-test',
table_id='mod-test',
referenced_doc_type='mod-test')
data_source.save()
self.assertTrue(start < data_source.last_modified)
time.sleep(.01)
between = datetime.datetime.utcnow()
self.assertTrue(between > data_source.last_modified)
time.sleep(.01)
data_source.save()
time.sleep(.01)
self.assertTrue(between < data_source.last_modified)
self.assertTrue(datetime.datetime.utcnow() > data_source.last_modified)
def get_sample_config(domain=None):
return DataSourceConfiguration(
domain=domain or 'domain',
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=clean_table_name('domain', str(uuid.uuid4().hex)),
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'name'
},
"column_id": 'name',
"display_name": 'name',
"datatype": "string"
}],
)
def _create_report(domain, title="report", upstream_id=None, should_save=True):
data_source = DataSourceConfiguration(
domain=domain,
table_id=uuid.uuid4().hex,
referenced_doc_type='XFormInstance',
)
data_source.save()
report = ReportConfiguration(
domain=domain,
config_id=data_source._id,
title=title,
report_meta=ReportMeta(created_by_builder=True, master_id=upstream_id),
)
if should_save:
report.save()
return report
def setUp(self):
self.config = DataSourceConfiguration(
domain='domain',
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=_clean_table_name('domain', str(uuid.uuid4().hex)),
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'name'
},
"column_id": 'name',
"display_name": 'name',
"datatype": "string"
}],
)
def _build_data_source(self):
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name=self.ds_builder.data_source_name,
referenced_doc_type=self.ds_builder.source_doc_type,
# The uuid gets truncated, so it's not really universally unique.
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter=self.ds_builder.filter,
configured_indicators=self.ds_builder.indicators,
meta=DataSourceMeta(build=DataSourceBuildInformation(
source_id=self.report_source_id,
app_id=self.app._id,
app_version=self.app.version,
)))
data_source_config.validate()
data_source_config.save()
tasks.rebuild_indicators.delay(data_source_config._id)
return data_source_config._id
def get_form_data_source(app, form):
xform = XForm(form.source)
form_name = form.default_name()
questions = xform.get_questions([])
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=_clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns),
configured_indicators=[
make_form_question_indicator(q, column_id=get_column_name(q['value']))
for q in questions
] + [
make_form_meta_block_indicator(field)
for field in FORM_METADATA_PROPERTIES
],
)
def test_column_uniqueness_when_truncated(self):
problem_spec = {
"display_name":
"practicing_lessons",
"property_name":
"long_column",
"choices": [
# test for regression:
# with sqlalchemy paramstyle='pyformat' (default)
# some queries that included columns with ')' in the column name
# would fail with a very cryptic message
"duplicate_choice_1(s)",
"duplicate_choice_2",
],
"select_style":
"multiple",
"column_id":
"a_very_long_base_selection_column_name_with_limited_room",
"type":
"choice_list",
}
data_source_config = DataSourceConfiguration(
domain='test',
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=uuid.uuid4().hex,
configured_filter={},
configured_indicators=[problem_spec],
)
adapter = get_indicator_adapter(data_source_config)
adapter.rebuild_table()
# ensure we can save data to the table.
adapter.save({
'_id': uuid.uuid4().hex,
'domain': 'test',
'doc_type': 'CommCareCase',
'long_column': 'duplicate_choice_1(s)',
})
adapter.refresh_table()
# and query it back
q = adapter.get_query_object()
self.assertEqual(1, q.count())
def test_is_static_negative(self):
self.assertFalse(DataSourceConfiguration().is_static)
self.assertFalse(
DataSourceConfiguration(_id=uuid.uuid4().hex).is_static)
