def test_import_from_basic_definition(self):
spec = self.get_monthly_config_spec()
data_source = DataSourceConfiguration(
domain=spec.domain,
referenced_doc_type='CommCareCase',
table_id='some_table',
)
data_source.save()
self.addCleanup(data_source.delete)
# these just have to be valid data source objects
spec.primary_table.data_source_id = data_source._id
spec.secondary_tables[0].data_source_id = data_source._id
aggregate_table_definition = import_aggregation_models_from_spec(spec)
self.assertEqual(1, AggregateTableDefinition.objects.count())
table_def = AggregateTableDefinition.objects.get(pk=aggregate_table_definition.pk)
self.assertEqual(data_source._id, table_def.primary_data_source_id.hex)
self.assertEqual(4, table_def.primary_columns.count())
aggregation = table_def.time_aggregation
self.assertEqual('month', aggregation.aggregation_unit)
self.assertEqual('opened_date', aggregation.start_column)
self.assertEqual('closed_date', aggregation.end_column)
self.assertEqual(1, table_def.secondary_tables.count())
secondary_table = table_def.secondary_tables.get()
self.assertEqual(data_source._id, secondary_table.data_source_id.hex)
self.assertEqual('doc_id', secondary_table.join_column_primary)
self.assertEqual('form.case.@case_id', secondary_table.join_column_secondary)
self.assertEqual('received_on', secondary_table.time_window_column)
self.assertEqual(2, secondary_table.columns.count())
self.assertEqual(
set(['fu_forms_in_month', 'any_fu_forms_in_month']),
set(secondary_table.columns.values_list('column_id', flat=True))
)
def setUpClass(cls):
cls.db2_name = 'cchq_ucr_tests'
db_conn_parts = settings.SQL_REPORTING_DATABASE_URL.split('/')
db_conn_parts[-1] = cls.db2_name
cls.db2_url = '/'.join(db_conn_parts)
# setup patches
cls.connection_string_patch = patch('corehq.sql_db.connections.connection_manager.get_connection_string')
def connection_string_for_engine(engine_id):
if engine_id == 'engine-1':
return settings.SQL_REPORTING_DATABASE_URL
else:
return cls.db2_url
mock_manager = cls.connection_string_patch.start()
mock_manager.side_effect = connection_string_for_engine
# setup data sources
data_source_template = get_sample_data_source()
cls.ds_1 = DataSourceConfiguration.wrap(data_source_template.to_json())
cls.ds_1.engine_id = 'engine-1'
cls.ds_1.save()
cls.ds_2 = DataSourceConfiguration.wrap(data_source_template.to_json())
cls.ds_2.engine_id = 'engine-2'
cls.ds_2.save()
cls.db_context = temporary_database(cls.db2_name)
cls.db_context.__enter__()
cls.ds1_adapter = IndicatorSqlAdapter(cls.ds_1)
cls.ds2_adapter = IndicatorSqlAdapter(cls.ds_2)
def update_report(self):
from corehq.apps.userreports.views import delete_data_source_shared
matching_data_source = self.ds_builder.get_existing_match()
if matching_data_source:
reactivated = False
if matching_data_source._id != self.existing_report.config_id:
# If no one else is using the current data source, delete it.
data_source = DataSourceConfiguration.get(self.existing_report.config_id)
if data_source.get_report_count() <= 1:
data_source.deactivate()
self.existing_report.config_id = matching_data_source._id
elif matching_data_source.is_deactivated:
existing_sources = DataSourceConfiguration.by_domain(self.domain)
active_sources = filter(lambda config: not config.is_deactivated, existing_sources)
if len(active_sources) >= 5:
raise forms.ValidationError(_(
"Editing this report would require a new data source. The limit is 5. "
"To continue, first delete all of the reports using a particular "
"data source (or the data source itself) and try again. "
))
matching_data_source.is_deactivated = False
reactivated = True
changed = False
indicators = self.ds_builder.indicators(self._number_columns)
if matching_data_source.configured_indicators != indicators:
matching_data_source.configured_indicators = indicators
changed = True
if changed or reactivated:
matching_data_source.save()
tasks.rebuild_indicators.delay(matching_data_source._id)
else:
# We need to create a new data source
existing_sources = DataSourceConfiguration.by_domain(self.domain)
active_sources = filter(lambda config: not config.is_deactivated, existing_sources)
# Delete the old one if no other reports use it
old_data_source = DataSourceConfiguration.get(self.existing_report.config_id)
if old_data_source.get_report_count() <= 1:
old_data_source.deactivate()
# Make sure the user can create more data sources
elif len(active_sources) >= 5:
raise forms.ValidationError(_(
"Editing this report would require a new data source. The limit is 5. "
"To continue, first delete all of the reports using a particular "
"data source (or the data source itself) and try again. "
))
data_source_config_id = self._build_data_source()
self.existing_report.config_id = data_source_config_id
self.existing_report.aggregation_columns = self._report_aggregation_cols
self.existing_report.columns = self._report_columns
self.existing_report.filters = self._report_filters
self.existing_report.configured_charts = self._report_charts
self.existing_report.validate()
self.existing_report.save()
return self.existing_report
def setUpClass(cls):
data_source = DataSourceConfiguration(
domain=cls.DOMAIN,
table_id="foo",
referenced_doc_type="CommCareCase",
)
data_source.save()
ReportConfiguration(
domain=cls.DOMAIN,
config_id=data_source._id,
columns=[
{
"type": "field",
"field": "foo",
"column_id": "foo",
"aggregation": "simple",
"display": "My Column",
},
{
"type": "field",
"field": "bar",
"column_id": "bar",
"aggregation": "simple",
"display": {"en": "Name", "fra": "Nom"},
},
]
).save()
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'indicator_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'city'
},
"column_id": 'indicator_col_id_city',
"display_name": 'indicator_display_name_city',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def test_get_by_domain(self):
results = DataSourceConfiguration.by_domain('foo')
self.assertEqual(2, len(results))
for item in results:
self.assertTrue(item.table_id in ('foo1', 'foo2'))
results = DataSourceConfiguration.by_domain('not-foo')
self.assertEqual(0, len(results))
def set_default_engine_ids(apps, schema_editor):
if not settings.UNIT_TESTING:
get_preindex_plugin('userreports').sync_design_docs()
ucr_db = DataSourceConfiguration.get_db()
with IterDB(ucr_db) as iter_db:
for doc in iter_docs(ucr_db, DataSourceConfiguration.all_ids()):
if not doc.get('engine_id'):
doc['engine_id'] = DEFAULT_ENGINE_ID
iter_db.save(doc)
def set_default_engine_ids(apps, schema_editor):
if not settings.UNIT_TESTING:
sync_docs.sync(ucr_models, verbosity=2)
ucr_db = DataSourceConfiguration.get_db()
with IterDB(ucr_db) as iter_db:
for doc in iter_docs(ucr_db, DataSourceConfiguration.all_ids()):
if not doc.get("engine_id"):
doc["engine_id"] = DEFAULT_ENGINE_ID
iter_db.save(doc)
def _build_report(self, vals, field="my_field", build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
self._new_case({field: v}).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name="foo",
referenced_doc_type="CommCareCase",
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {"type": "property_name", "property_name": "type"},
"property_value": self.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {"type": "property_name", "property_name": field},
"column_id": field,
"display_name": field,
"datatype": "string",
}
],
)
data_source_config.validate()
data_source_config.save()
if build_data_source:
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=self.domain,
config_id=data_source_config._id,
title="foo",
aggregation_columns=["doc_id"],
columns=[{"type": "expanded", "field": field, "display": field, "format": "default"}],
filters=[],
configured_charts=[],
)
report_config.save()
data_source = ReportFactory.from_spec(report_config)
return data_source, data_source.column_configs[0]
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
config = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_string',
"display_name": 'date_as_string',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_date',
"datatype": "date"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "my_datetime",
},
"column_id": "datetime_as_datetime",
"datatype": "datetime"
}
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
cls.data_sources[UCR_SQL_BACKEND] = config
cls.adapters[UCR_SQL_BACKEND] = adapter
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
# this is a hack to have both sql and es backends created in a class
# method. alternative would be to have these created on each test run
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'first_name'
},
"column_id": 'indicator_col_id_first_name',
"display_name": 'indicator_display_name_first_name',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def setUpClass(cls):
cls.db2_name = 'cchq_ucr_tests'
db_conn_parts = settings.SQL_REPORTING_DATABASE_URL.split('/')
db_conn_parts[-1] = cls.db2_name
cls.db2_url = '/'.join(db_conn_parts)
# setup patches
cls.engine_id_patches = (
# unfortunately we need to patch this directly in modules that import it as well
patch('corehq.apps.userreports.sql.connection.get_engine_id'),
patch('corehq.apps.userreports.sql.adapter.get_engine_id'),
patch('corehq.apps.userreports.reports.data_source.get_engine_id'),
)
cls.connection_string_patch = patch('corehq.db.connection_manager.get_connection_string')
for engine_id_patch in cls.engine_id_patches:
mock_engine_id_method = engine_id_patch.start()
mock_engine_id_method.side_effect = lambda x: x.engine_id
def connection_string_for_engine(engine_id):
if engine_id == 'engine-1':
return settings.SQL_REPORTING_DATABASE_URL
else:
return cls.db2_url
mock_manager = cls.connection_string_patch.start()
mock_manager.side_effect = connection_string_for_engine
# setup data sources
data_source_template = get_sample_data_source()
cls.ds_1 = DataSourceConfiguration.wrap(data_source_template.to_json())
cls.ds_1.engine_id = 'engine-1'
cls.ds_1.save()
cls.ds_2 = DataSourceConfiguration.wrap(data_source_template.to_json())
cls.ds_2.engine_id = 'engine-2'
cls.ds_2.save()
# use db1 engine to create db2 http://stackoverflow.com/a/8977109/8207
cls.root_engine = create_engine(settings.SQL_REPORTING_DATABASE_URL)
conn = cls.root_engine.connect()
conn.execute('commit')
try:
conn.execute('CREATE DATABASE {}'.format(cls.db2_name))
except ProgrammingError:
# optimistically assume it failed because was already created.
pass
conn.close()
cls.ds1_adapter = IndicatorSqlAdapter(cls.ds_1)
cls.ds2_adapter = IndicatorSqlAdapter(cls.ds_2)
def __init__(self, domain, *args, **kwargs):
self.domain = domain
available_data_sources = DataSourceConfiguration.by_domain(self.domain)
super(ReportDataSourceField, self).__init__(
choices=[(src._id, src.display_name) for src in available_data_sources],
*args, **kwargs
)
def setUp(self):
self.indicator_configuration = DataSourceConfiguration.wrap({
'display_name': 'Mother Indicators',
'doc_type': 'DataSourceConfiguration',
'domain': 'test',
'referenced_doc_type': 'CommCareCase',
'table_id': 'mother_indicators',
'named_filters': {
'pregnant': {
'type': 'property_match',
'property_name': 'mother_state',
'property_value': 'pregnant',
}
},
'configured_filter': {
'type': 'and',
'filters': [
{
'property_name': 'type',
'property_value': 'ttc_mother',
'type': 'property_match',
},
{
'type': 'named',
'name': 'pregnant',
}
]
}
})
def setUp(self):
self.domain_obj = create_domain(self.domain)
es = get_es_new()
initialize_index_and_mapping(es, USER_INDEX_INFO)
self.region = LocationType.objects.create(domain=self.domain, name="region")
self.town = LocationType.objects.create(domain=self.domain, name="town", parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_location_pillow(ucr_configs=[self.data_source_config])
self.pillow.get_change_feed().get_latest_offsets()
def handle(self, **options):
data_sources = list(DataSourceConfiguration.all())
data_sources.extend(list(StaticDataSourceConfiguration.all()))
engine_ids = self._get_engine_ids(data_sources, options.get('engine_id'))
tables_to_remove_by_engine = defaultdict(list)
for engine_id in engine_ids:
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, include_object=_include_object)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_remove_by_engine[engine_id] = [
diff.table_name for diff in diffs
if diff.type == 'remove_table'
]
for engine_id, tablenames in tables_to_remove_by_engine.items():
engine = connection_manager.get_engine(engine_id)
for tablename in tablenames:
with engine.begin() as connection:
try:
result = connection.execute(
'SELECT COUNT(*), MAX(inserted_at) FROM "{tablename}"'.format(tablename=tablename)
)
except Exception:
print(tablename, "no inserted_at column, probably not UCR")
else:
print(tablename, result.fetchone())
def rebuild_indicators(indicator_config_id):
is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(indicator_config_id)
else:
config = DataSourceConfiguration.get(indicator_config_id)
# Save the start time now in case anything goes wrong. This way we'll be
# able to see if the rebuild started a long time ago without finishing.
config.meta.build.initiated = datetime.datetime.utcnow()
config.save()
adapter = IndicatorSqlAdapter(config)
adapter.rebuild_table()
couchdb = _get_db(config.referenced_doc_type)
relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type,
database=couchdb)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
except DataError as e:
logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
if not is_static:
config.meta.build.finished = True
config.save()
def _shared_context(domain):
custom_data_sources = list(CustomDataSourceConfiguration.by_domain(domain))
return {
'domain': domain,
'reports': ReportConfiguration.by_domain(domain),
'data_sources': DataSourceConfiguration.by_domain(domain) + custom_data_sources,
}
def bootstrap(self, configs=None):
# sets up the initial stuff
if configs is None:
configs = DataSourceConfiguration.all()
self.tables = [IndicatorSqlAdapter(self.get_sql_engine(), config) for config in configs]
self.bootstrapped = True
def get_datasources_for_domain(domain, referenced_doc_type=None, include_static=False, include_aggregate=False):
from corehq.apps.userreports.models import DataSourceConfiguration, StaticDataSourceConfiguration
key = [domain]
if referenced_doc_type:
key.append(referenced_doc_type)
datasources = sorted(
DataSourceConfiguration.view(
'userreports/data_sources_by_build_info',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=True
),
key=lambda config: config.display_name or '')
if include_static:
static_ds = StaticDataSourceConfiguration.by_domain(domain)
if referenced_doc_type:
static_ds = [ds for ds in static_ds if ds.referenced_doc_type == referenced_doc_type]
datasources.extend(sorted(static_ds, key=lambda config: config.display_name))
if include_aggregate:
from corehq.apps.aggregate_ucrs.models import AggregateTableDefinition
datasources.extend(AggregateTableDefinition.objects.filter(domain=domain).all())
return datasources
def setUp(self):
delete_all_locations()
self.domain_obj = create_domain(self.domain)
self.region = LocationType.objects.create(domain=self.domain, name="region")
self.town = LocationType.objects.create(domain=self.domain, name="town", parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.data_source_config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_current_offsets()
def test_stale_rebuild(self):
# rebuild indicators in another test will save this
later_config = DataSourceConfiguration.get(self.config._id)
later_config.save()
self.assertNotEqual(self.config._rev, later_config._rev)
with self.assertRaises(StaleRebuildError):
self.pillow.processors[0].rebuild_table(get_indicator_adapter(self.config))
def _build_indicators(indicator_config_id, relevant_ids):
config = _get_config_by_id(indicator_config_id)
adapter = IndicatorSqlAdapter(config)
couchdb = _get_db(config.referenced_doc_type)
redis_client = get_redis_client().client.get_client()
redis_key = _get_redis_key_for_config(config)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
redis_client.srem(redis_key, doc.get('_id'))
except Exception as e:
logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
if not is_static(indicator_config_id):
redis_client.delete(redis_key)
config.meta.build.finished = True
try:
config.save()
except ResourceConflict:
current_config = DataSourceConfiguration.get(config._id)
# check that a new build has not yet started
if config.meta.build.initiated == current_config.meta.build.initiated:
current_config.meta.build.finished = True
current_config.save()
def clean(self):
"""
Raise a validation error if there are already 5 data sources and this
report won't be able to use one of the existing ones.
"""
cleaned_data = super(DataSourceForm, self).clean()
source_type = cleaned_data.get('source_type')
report_source = cleaned_data.get('report_source')
app_id = cleaned_data.get('application')
if report_source and source_type and app_id:
app = Application.get(app_id)
ds_builder = DataSourceBuilder(self.domain, app, source_type, report_source)
existing_sources = DataSourceConfiguration.by_domain(self.domain)
if len(existing_sources) >= 5:
if not ds_builder.get_existing_match():
raise forms.ValidationError(_(
"Too many data sources!\n"
"Creating this report would cause you to go over the maximum "
"number of data sources allowed in this domain. The current "
"limit is 5. "
"To continue, delete all of the reports using a particular "
"data source (or the data source itself) and try again. "
))
return cleaned_data
def tearDownClass(cls):
for config in DataSourceConfiguration.all():
config.delete()
delete_all_report_configs()
for domain in Domain.get_all():
domain.delete()
super(ReportConfigurationDbTest, cls).tearDownClass()
def test_last_modified_date(self):
start = datetime.datetime.utcnow()
time.sleep(.01)
data_source = DataSourceConfiguration(
domain='mod-test', table_id='mod-test', referenced_doc_type='mod-test'
)
data_source.save()
self.assertTrue(start < data_source.last_modified)
time.sleep(.01)
between = datetime.datetime.utcnow()
self.assertTrue(between > data_source.last_modified)
time.sleep(.01)
data_source.save()
time.sleep(.01)
self.assertTrue(between < data_source.last_modified)
self.assertTrue(datetime.datetime.utcnow() > data_source.last_modified)
def _iteratively_build_table(config, last_id=None, resume_helper=None):
resume_helper = resume_helper or DataSourceResumeHelper(config)
indicator_config_id = config._id
relevant_ids = []
document_store = get_document_store(config.domain, config.referenced_doc_type)
for relevant_id in document_store.iter_document_ids(last_id):
relevant_ids.append(relevant_id)
if len(relevant_ids) >= ID_CHUNK_SIZE:
resume_helper.set_ids_to_resume_from(relevant_ids)
_build_indicators(config, document_store, relevant_ids, resume_helper)
relevant_ids = []
if relevant_ids:
resume_helper.set_ids_to_resume_from(relevant_ids)
_build_indicators(config, document_store, relevant_ids, resume_helper)
if not id_is_static(indicator_config_id):
resume_helper.clear_ids()
config.meta.build.finished = True
try:
config.save()
except ResourceConflict:
current_config = DataSourceConfiguration.get(config._id)
# check that a new build has not yet started
if config.meta.build.initiated == current_config.meta.build.initiated:
current_config.meta.build.finished = True
current_config.save()
def _get_all_data_sources():
from corehq.apps.userreports.models import DataSourceConfiguration
return DataSourceConfiguration.view(
'userreports/data_sources_by_build_info',
reduce=False,
include_docs=True
)
def test_missing_no_named_in_named(self):
bad_config = DataSourceConfiguration.wrap(self.indicator_configuration.to_json())
bad_config.named_expressions['broken'] = {
"type": "named",
"name": "pregnant",
}
with self.assertRaises(BadSpecError):
bad_config.validate()
def _shared_context(domain):
static_reports = list(StaticReportConfiguration.by_domain(domain))
static_data_sources = list(StaticDataSourceConfiguration.by_domain(domain))
return {
'domain': domain,
'reports': ReportConfiguration.by_domain(domain) + static_reports,
'data_sources': DataSourceConfiguration.by_domain(domain) + static_data_sources,
}
def test_domain_is_required(self):
with self.assertRaises(BadValueError):
DataSourceConfiguration(table_id='table',
referenced_doc_type='doc').save()
def setUpClass(cls):
super(TestConfigurableReportDataResource, cls).setUpClass()
case_type = "my_case_type"
cls.field_name = "my_field"
cls.case_property_values = ["foo", "foo", "bar", "baz"]
cls.cases = []
for val in cls.case_property_values:
id = uuid.uuid4().hex
case_block = CaseBlock.deprecated_init(
create=True,
case_id=id,
case_type=case_type,
update={
cls.field_name: val
},
).as_xml()
post_case_blocks([case_block], {'domain': cls.domain.name})
cls.cases.append(CommCareCase.get(id))
cls.report_columns = [{
"column_id": cls.field_name,
"type": "field",
"field": cls.field_name,
"aggregation": "simple",
}]
cls.report_filters = [{
'datatype': 'string',
'field': cls.field_name,
'type': 'dynamic_choice_list',
'slug': 'my_field_filter',
}]
cls.data_source = DataSourceConfiguration(
domain=cls.domain.name,
referenced_doc_type="CommCareCase",
table_id=uuid.uuid4().hex,
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": cls.field_name
},
"column_id": cls.field_name,
"display_name": cls.field_name,
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "opened_by"
},
"column_id": "opened_by",
"display_name": "opened_by",
"datatype": "string"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
cls.report_configuration = ReportConfiguration(
domain=cls.domain.name,
config_id=cls.data_source._id,
aggregation_columns=["doc_id"],
columns=cls.report_columns,
filters=cls.report_filters,
)
cls.report_configuration.save()
def tearDown(self):
delete_all_report_configs()
for config in DataSourceConfiguration.all():
config.delete()
super().tearDown()
def _build_report(self, vals, field='my_field', build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
update_props = {field: v} if v is not None else {}
self._new_case(update_props).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": self.case_type,
},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": field
},
"column_id": field,
"display_name": field,
"datatype": "string"
}],
)
data_source_config.validate()
data_source_config.save()
self.addCleanup(data_source_config.delete)
if build_data_source:
tasks.rebuild_indicators(data_source_config._id)
adapter = get_indicator_adapter(data_source_config)
adapter.refresh_table()
report_config = ReportConfiguration(domain=self.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[{
"type": "expanded",
"field": field,
"display": field,
"format": "default",
}],
filters=[],
configured_charts=[])
report_config.save()
self.addCleanup(report_config.delete)
data_source = ReportFactory.from_spec(report_config)
adapter = get_indicator_adapter(data_source_config)
if build_data_source:
adapter.refresh_table()
return data_source, data_source.top_level_columns[0]
def create_data_source(request, domain):
return _edit_data_source_shared(request, domain,
DataSourceConfiguration(domain=domain))
def _get_all_data_sources():
from corehq.apps.userreports.models import DataSourceConfiguration
return DataSourceConfiguration.view(
'userreports/data_sources_by_build_info',
reduce=False,
include_docs=True)
def setUp(self):
self.indicator_configuration = DataSourceConfiguration.wrap({
'display_name':
'Mother Indicators',
'doc_type':
'DataSourceConfiguration',
'domain':
'test',
'referenced_doc_type':
'CommCareCase',
'table_id':
'mother_indicators',
'named_expressions': {
'pregnant': {
'type': 'property_name',
'property_name': 'pregnant',
},
'is_evil': {
'type': 'property_name',
'property_name': 'is_evil',
},
'laugh_sound': {
'type': 'conditional',
'test': {
'type': 'boolean_expression',
'expression': {
'type': 'property_name',
'property_name': 'is_evil',
},
'operator': 'eq',
'property_value': True,
},
'expression_if_true': "mwa-ha-ha",
'expression_if_false': "hehe",
}
},
'named_filters': {},
'configured_filter': {
'type': 'boolean_expression',
'expression': {
'type': 'named',
'name': 'pregnant'
},
'operator': 'eq',
'property_value': 'yes',
},
'configured_indicators': [
{
"type": "expression",
"column_id": "laugh_sound",
"datatype": "string",
"expression": {
'type': 'named',
'name': 'laugh_sound'
}
},
{
"type": "expression",
"column_id": "characterization",
"datatype": "string",
"expression": {
'type': 'conditional',
'test': {
'type': 'boolean_expression',
'expression': {
'type': 'named',
'name': 'is_evil',
},
'operator': 'eq',
'property_value': True,
},
'expression_if_true': "evil!",
'expression_if_false': "okay",
}
},
]
})
def test_duplicate_columns(self):
bad_config = DataSourceConfiguration.wrap(self.config.to_json())
bad_config.configured_indicators.append(
bad_config.configured_indicators[-1])
with self.assertRaises(BadSpecError):
bad_config.validate()
def tearDownClass(cls):
for config in DataSourceConfiguration.all():
config.delete()
class TestLocationDataSource(TestCase):
domain = "delos_corp"
def setUp(self):
delete_all_locations()
self.domain_obj = create_domain(self.domain)
self.region = LocationType.objects.create(domain=self.domain,
name="region")
self.town = LocationType.objects.create(domain=self.domain,
name="town",
parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.data_source_config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_current_offsets()
def tearDown(self):
self.domain_obj.delete()
delete_all_locations()
self.data_source_config.delete()
def _make_loc(self, name, location_type):
return SQLLocation.objects.create(domain=self.domain,
name=name,
site_code=name,
location_type=location_type)
def assertDataSourceAccurate(self, expected_locations):
adapter = get_indicator_adapter(self.data_source_config)
query = adapter.get_query_object()
adapter.refresh_table()
data_source = query.all()
self.assertItemsEqual(expected_locations,
[row[-1] for row in data_source])
def test_location_data_source(self):
self._make_loc("Westworld", self.region)
sweetwater = self._make_loc("Sweetwater", self.town)
las_mudas = self._make_loc("Las Mudas", self.town)
rebuild_indicators(self.data_source_config._id)
self.assertDataSourceAccurate(["Westworld", "Sweetwater", "Las Mudas"])
# Insert new location
since = self.pillow.get_change_feed().get_current_offsets()
self._make_loc("Blood Arroyo", self.town)
# Change an existing location
sweetwater.name = "Pariah"
sweetwater.save()
# Process both changes together and verify that they went through
self.pillow.process_changes(since=since, forever=False)
self.assertDataSourceAccurate(
["Westworld", "Pariah", "Las Mudas", "Blood Arroyo"])
# Delete a location
since = self.pillow.get_change_feed().get_current_offsets()
las_mudas.delete()
self.pillow.process_changes(since=since, forever=False)
# No actual change - deletions are not yet processed
self.assertDataSourceAccurate(
["Westworld", "Pariah", "Las Mudas", "Blood Arroyo"])
def test_table_id_is_required(self):
with self.assertRaises(BadValueError):
DataSourceConfiguration(
domain='domain', referenced_doc_type='XFormInstance').save()
def test_doc_type_is_required(self):
with self.assertRaises(BadValueError):
DataSourceConfiguration(domain='domain', table_id='table').save()
def _get_config_by_id(indicator_config_id):
if id_is_static(indicator_config_id):
return StaticDataSourceConfiguration.by_id(indicator_config_id)
else:
return DataSourceConfiguration.get(indicator_config_id)
def get_data_source_with_related_doc_type():
folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
sample_file = os.path.join(folder, 'parent_child_data_source.json')
with open(sample_file, encoding='utf-8') as f:
structure = json.loads(f.read())
return DataSourceConfiguration.wrap(structure)
def config(self):
if self.config_id is None:
return DataSourceConfiguration(domain=self.domain)
return get_datasource_config_or_404(self.config_id, self.domain)[0]
def tearDownClass(cls):
cls.app.delete()
for config in DataSourceConfiguration.all():
config.delete()
delete_all_report_configs()
super(ReportBuilderDBTest, cls).tearDownClass()
def get_sample_data_source():
folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
sample_file = os.path.join(folder, 'sample_data_source.json')
with open(sample_file) as f:
structure = json.loads(f.read())
return DataSourceConfiguration.wrap(structure)
def test_updating_report_data_source(self):
"""
Test that changing the app or number column for a report results in an update to the data source next time
the report is saved.
"""
# Make report
builder_form = ConfigureTableReportForm(
self.domain,
"Test Report",
self.app._id,
"case",
"some_case_type",
existing_report=None,
data={
'group_by': ['closed'],
'chart':
'bar',
'user_filters':
'[]',
'default_filters':
'[]',
'columns':
'[{"property": "closed", "display_text": "closed", "calculation": "Count per Choice"}]',
})
self.assertTrue(builder_form.is_valid())
report = builder_form.create_report()
self.assertEqual(report.config.configured_indicators[0]['datatype'],
"string")
# Make an edit to the first report builder report
builder_form = ConfigureTableReportForm(
self.domain,
"Test Report",
self.app._id,
"case",
"some_case_type",
existing_report=report,
data={
'group_by': ['user_id'],
'chart':
'bar',
'user_filters':
'[]',
'default_filters':
'[]',
# Note that a "Sum" calculation on the closed case property isn't very sensical, but doing it so
# that I can have a numeric calculation without having to create real case properties for this case
# type.
'columns':
'[{"property": "closed", "display_text": "closed", "calculation": "Sum"}]',
})
self.assertTrue(builder_form.is_valid())
builder_form.update_report()
# reload report data source, because report.config is memoized
data_source = DataSourceConfiguration.get(report.config._id)
# The closed property indicator should now be decimal type because the user indicated that it was numeric
# by giving the column the "Sum" aggregation.
self.assertEqual(data_source.configured_indicators[0]['datatype'],
"decimal")
class ReportDataTest(TestCase):
dependent_apps = ['pillowtop']
def setUp(self):
super(ReportDataTest, self).setUp()
# Create report
self.domain = 'test-ucr-report-data'
self.data_source = DataSourceConfiguration(
domain=self.domain,
referenced_doc_type='CommCareCase',
table_id=uuid.uuid4().hex,
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'name'
},
"column_id": 'name',
"display_name": 'name',
"datatype": "string"
}, {
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'number',
"display_name": 'number',
"datatype": "integer"
}],
)
self.data_source.validate()
self.data_source.save()
IndicatorSqlAdapter(self.data_source).rebuild_table()
self.addCleanup(self.data_source.delete)
# initialize a report on the data
self.report_config = ReportConfiguration(
domain=self.domain,
config_id=self.data_source._id,
aggregation_columns=['doc_id'],
columns=[{
"type": "field",
"field": "name",
"column_id": "name",
"display": "Name",
"aggregation": "simple",
}, {
"type": "field",
"field": "number",
"column_id": "number",
"display": "Number",
"aggregation": "simple",
}],
filters=[],
configured_charts=[])
self.report_config.save()
self.addCleanup(self.report_config.delete)
def _add_some_rows(self, count):
rows = [ReportDataTestRow(uuid.uuid4().hex, i) for i in range(count)]
self._add_rows(rows)
return rows
def _add_rows(self, rows):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.data_source])
def _get_case(row):
return {
'_id': uuid.uuid4().hex,
'domain': self.domain,
'doc_type': 'CommCareCase',
'type': 'city',
'name': row.name,
'number': row.number,
}
for row in rows:
pillow.process_change(doc_to_change(_get_case(row)))
def test_basic_query(self):
# add a few rows to the data source
rows = self._add_some_rows(3)
# check the returned data from the report looks right
report_data_source = ReportFactory.from_spec(self.report_config)
report_data = report_data_source.get_data()
self.assertEqual(len(rows), len(report_data))
rows_by_name = {r.name: r for r in rows}
for row in report_data:
self.assertTrue(row['name'] in rows_by_name)
self.assertEqual(rows_by_name[row['name']].number, row['number'])
def test_limit(self):
count = 5
self._add_some_rows(count)
report_data_source = ReportFactory.from_spec(self.report_config)
original_data = report_data_source.get_data()
self.assertEqual(count, len(original_data))
limited_data = report_data_source.get_data(limit=3)
self.assertEqual(3, len(limited_data))
self.assertEqual(original_data[:3], limited_data)
def test_skip(self):
count = 5
self._add_some_rows(count)
report_data_source = ReportFactory.from_spec(self.report_config)
original_data = report_data_source.get_data()
self.assertEqual(count, len(original_data))
skipped = report_data_source.get_data(start=3)
self.assertEqual(count - 3, len(skipped))
self.assertEqual(original_data[3:], skipped)
def _build_report_and_view(cls):
# Create Cases
cls._new_case({'fruit': 'apple', 'num1': 4, 'num2': 6}).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=cls.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id="woop_woop",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'fruit'
},
"column_id": 'indicator_col_id_fruit',
"display_name": 'indicator_display_name_fruit',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num1'
},
"column_id": 'indicator_col_id_num1',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num2'
},
"column_id": 'indicator_col_id_num2',
"datatype": "integer"
},
],
)
data_source_config.validate()
data_source_config.save()
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=cls.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[
{
"type": "field",
"display": "report_column_display_fruit",
"field": 'indicator_col_id_fruit',
'column_id': 'report_column_col_id_fruit',
'aggregation': 'simple'
},
{
"type": "percent",
"display": "report_column_display_percent",
'column_id': 'report_column_col_id_percent',
'format': 'percent',
"denominator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num1",
"column_id": "report_column_col_id_percent_num1"
},
"numerator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num2",
"column_id": "report_column_col_id_percent_num2"
}
},
],
)
report_config.save()
view = ConfigurableReport(request=HttpRequest())
view._domain = cls.domain
view._lang = "en"
view._report_config_id = report_config._id
return report_config, view
def setUp(self):
folder = os.path.join(os.path.dirname(__file__), 'data', 'configs')
sample_file = os.path.join(folder, 'data_source_with_repeat.json')
with open(sample_file) as f:
self.config = DataSourceConfiguration.wrap(json.loads(f.read()))
def tearDownClass(cls):
for config in DataSourceConfiguration.all():
config.delete()
super(DataSourceConfigurationDbTest, cls).tearDownClass()
def get_data_sources(self):
return DataSourceConfiguration.view('userreports/active_data_sources',
reduce=False,
include_docs=True).all()
def _setup_config(self, doc_type, filter_):
return DataSourceConfiguration(domain='test',
referenced_doc_type=doc_type,
table_id='blah',
configured_filter=filter_)
def _build_report_and_view(self, request=HttpRequest()):
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id="woop_woop",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": self.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'fruit'
},
"column_id": 'indicator_col_id_fruit',
"display_name": 'indicator_display_name_fruit',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num1'
},
"column_id": 'indicator_col_id_num1',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num2'
},
"column_id": 'indicator_col_id_num2',
"datatype": "integer"
},
],
)
data_source_config.validate()
data_source_config.save()
self.addCleanup(data_source_config.delete)
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=self.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[{
"type": "field",
"display": "report_column_display_fruit",
"field": 'indicator_col_id_fruit',
'column_id': 'report_column_col_id_fruit',
'aggregation': 'simple'
}, {
"type": "percent",
"display": "report_column_display_percent",
'column_id': 'report_column_col_id_percent',
'format': 'percent',
"denominator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num1",
"column_id": "report_column_col_id_percent_num1"
},
"numerator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num2",
"column_id": "report_column_col_id_percent_num2"
}
}, {
"type": "expanded",
"display": "report_column_display_expanded_num1",
"field": 'indicator_col_id_num1',
'column_id': 'report_column_col_id_expanded_num1',
}],
configured_charts=[
{
"type": 'pie',
"value_column": 'count',
"aggregation_column": 'fruit',
"title": 'Fruits'
},
{
"type":
'multibar',
"title":
'Fruit Properties',
"x_axis_column":
'fruit',
"y_axis_columns": [{
"column_id": "report_column_col_id_expanded_num1",
"display": "Num1 values"
}]
},
])
report_config.save()
self.addCleanup(report_config.delete)
view = ConfigurableReportView(request=request)
view._domain = self.domain
view._lang = "en"
view._report_config_id = report_config._id
return report_config, view
def _delete_everything(cls):
delete_all_cases()
for config in DataSourceConfiguration.all():
config.delete()
delete_all_report_configs()
def test_get_all(self):
self.assertEqual(3, len(list(DataSourceConfiguration.all())))
def test_is_static_negative(self):
self.assertFalse(DataSourceConfiguration().is_static)
self.assertFalse(
DataSourceConfiguration(_id=uuid.uuid4().hex).is_static)
def test_report_builder_datasource_deactivation(self):
def _get_data_source(id_):
return get_datasource_config(id_, self.project.name)[0]
# Upgrade the domain
# (for the upgrade to work, there has to be an existing subscription,
# which is why we subscribe to advanced first)
self._subscribe_to_advanced()
pro_with_rb_sub = self._subscribe_to_pro_with_rb()
# Create reports and data sources
builder_report_data_source = DataSourceConfiguration(
domain=self.project.name,
is_deactivated=False,
referenced_doc_type="XFormInstance",
table_id="foo",
)
other_data_source = DataSourceConfiguration(
domain=self.project.name,
is_deactivated=False,
referenced_doc_type="XFormInstance",
table_id="bar",
)
builder_report_data_source.save()
other_data_source.save()
report_builder_report = ReportConfiguration(
domain=self.project.name,
config_id=builder_report_data_source._id,
report_meta=ReportMeta(created_by_builder=True),
)
report_builder_report.save()
# downgrade the domain
community_sub = pro_with_rb_sub.change_plan(
DefaultProductPlan.get_default_plan_version())
# Check that the builder data source is deactivated
builder_report_data_source = _get_data_source(
builder_report_data_source._id)
self.assertTrue(builder_report_data_source.is_deactivated)
# Check that the other data source has not been deactivated
other_data_source = _get_data_source(other_data_source._id)
self.assertFalse(other_data_source.is_deactivated)
# upgrade the domain
# (for the upgrade to work, there has to be an existing subscription,
# which is why we subscribe to advanced first)
community_sub.change_plan(
DefaultProductPlan.get_default_plan_version(
edition=SoftwarePlanEdition.ADVANCED))
pro_with_rb_sub = self._subscribe_to_pro_with_rb()
# check that the data source is activated
builder_report_data_source = _get_data_source(
builder_report_data_source._id)
self.assertFalse(builder_report_data_source.is_deactivated)
# delete the data sources
builder_report_data_source.delete()
other_data_source.delete()
# Delete the report
report_builder_report.delete()
# reset the subscription
pro_with_rb_sub.change_plan(
DefaultProductPlan.get_default_plan_version())
def setUp(self):
self.indicator_configuration = DataSourceConfiguration.wrap({
'display_name':
'Mother Indicators',
'doc_type':
'DataSourceConfiguration',
'domain':
'test',
'referenced_doc_type':
'CommCareCase',
'table_id':
'mother_indicators',
'named_expressions': {
'on_a_date': {
'type': 'property_name',
'property_name': 'on_date',
}
},
'named_filters': {
'pregnant': {
'type': 'property_match',
'property_name': 'mother_state',
'property_value': 'pregnant',
},
'evil': {
'type': 'property_match',
'property_name': 'evil',
'property_value': 'yes',
},
'has_alibi': {
'type': 'boolean_expression',
'expression': {
'type': 'named',
'name': 'on_a_date'
},
'operator': 'eq',
'property_value': 'yes',
}
},
'configured_filter': {
'type':
'and',
'filters': [{
'property_name': 'type',
'property_value': 'ttc_mother',
'type': 'property_match',
}, {
'type': 'named',
'name': 'pregnant',
}]
},
'configured_indicators': [{
"type": "boolean",
"column_id": "is_evil",
"filter": {
"type": "named",
"name": "evil"
}
}, {
"type": "expression",
"column_id": "laugh_sound",
"datatype": "string",
"expression": {
'type': 'conditional',
'test': {
"type": "named",
"name": "evil"
},
'expression_if_true': {
'type': 'constant',
'constant': 'mwa-ha-ha',
},
'expression_if_false': {
'type': 'constant',
'constant': 'hehe',
},
}
}]
})
