def test_rebuild_indicators(self, datetime_mock):
datetime_mock.utcnow.return_value = self.fake_time_now
sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
CommCareCase.get_db().save_doc(sample_doc)
self.addCleanup(lambda id: CommCareCase.get_db().delete_doc(id), sample_doc['_id'])
rebuild_indicators(self.config._id)
self._check_sample_doc_state(expected_indicators)
def create_adherence_cases(self, data):
factory = CaseFactory(domain=self.domain)
cases = factory.create_or_update_cases([
get_adherence_case_structure(case_id,
self.episode_id,
adherence_date,
extra_update={
"name": adherence_date,
"adherence_value":
adherence_value,
"source": source,
"closure_reason": closure_reason
})
for (case_id, adherence_date, adherence_value, source, _,
closure_reason, _) in data
])
cases_by_id = {c.case_id: c for c in cases}
for (case_id, _, _, _, should_close, _, modified_on) in data:
if should_close:
factory.close_case(case_id)
if modified_on:
cases_by_id[case_id].modified_on = modified_on
cases_by_id[case_id].save()
rebuild_indicators(self.data_store.datasource._id)
self.data_store.adapter.refresh_table()
def test_rebuild_indicators(self, datetime_mock):
datetime_mock.utcnow.return_value = self.fake_time_now
sample_doc, expected_indicators = get_sample_doc_and_indicators(
self.fake_time_now)
_save_sql_case(sample_doc)
rebuild_indicators(self.config._id)
self._check_sample_doc_state(expected_indicators)
def test_rebuild_indicators(self, datetime_mock):
datetime_mock.utcnow.return_value = self.fake_time_now
self.config.save()
sample_doc, _ = get_sample_doc_and_indicators(self.fake_time_now)
CommCareCase.get_db().save_doc(sample_doc)
rebuild_indicators(self.config._id)
self._check_sample_doc_state()
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is load_data_source %s' % self.args)
filename = args[0]
with open(filename) as f:
body = json.loads(f.read())
# intentionally fails hard if bad or missing doc_type
cls = {
'DataSourceConfiguration': DataSourceConfiguration,
'ReportConfiguration': ReportConfiguration,
}[body['doc_type']]
if '_id' in body:
try:
rev = cls.get_db().get_rev(body['_id'])
body['_rev'] = rev
except ResourceNotFound:
pass
to_save = cls.wrap(body)
to_save.validate()
to_save.save()
print 'updated {}: "{}"'.format(to_save.doc_type, to_save)
if options['rebuild'] and isinstance(to_save, DataSourceConfiguration):
print 'rebuilding table...'
rebuild_indicators(to_save._id)
print '...done!'
def test_rebuild_indicators(self, datetime_mock):
datetime_mock.utcnow.return_value = self.fake_time_now
sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
CommCareCase.get_db().save_doc(sample_doc)
self.addCleanup(lambda id: CommCareCase.get_db().delete_doc(id), sample_doc['_id'])
rebuild_indicators(self.config._id)
self._check_sample_doc_state(expected_indicators)
def handle(self, filename, **options):
with open(filename, encoding='utf-8') as f:
body = json.loads(f.read())
# intentionally fails hard if bad or missing doc_type
cls = {
'DataSourceConfiguration': DataSourceConfiguration,
'ReportConfiguration': ReportConfiguration,
}[body['doc_type']]
if '_id' in body:
try:
rev = cls.get_db().get_rev(body['_id'])
body['_rev'] = rev
except ResourceNotFound:
pass
to_save = cls.wrap(body)
to_save.validate()
to_save.save()
print('updated {}: "{}"'.format(to_save.doc_type, to_save))
if options['rebuild'] and isinstance(to_save,
DataSourceConfiguration):
print('rebuilding table...')
rebuild_indicators(to_save._id)
print('...done!')
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is load_data_source %s' % self.args)
filename = args[0]
with open(filename) as f:
body = json.loads(f.read())
# intentionally fails hard if bad or missing doc_type
cls = {
'DataSourceConfiguration': DataSourceConfiguration,
'ReportConfiguration': ReportConfiguration,
}[body['doc_type']]
if '_id' in body:
try:
rev = cls.get_db().get_rev(body['_id'])
body['_rev'] = rev
except ResourceNotFound:
pass
to_save = cls.wrap(body)
to_save.validate()
to_save.save()
print 'updated {}: "{}"'.format(to_save.doc_type, to_save)
if options['rebuild'] and isinstance(to_save,
DataSourceConfiguration):
print 'rebuilding table...'
rebuild_indicators(to_save._id)
print '...done!'
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
# this is a hack to have both sql and es backends created in a class
# method. alternative would be to have these created on each test run
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_string',
"display_name": 'date_as_string',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_date',
"datatype": "date"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "my_datetime",
},
"column_id": "datetime_as_datetime",
"datatype": "datetime"
}
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is rebuild_indicator_table %s' %
self.args)
config_id = args[0]
tasks.rebuild_indicators(config_id)
def test_location_data_source(self):
self._make_loc("Westworld", self.region)
sweetwater = self._make_loc("Sweetwater", self.town)
las_mudas = self._make_loc("Las Mudas", self.town)
rebuild_indicators(self.data_source_config._id)
self.assertDataSourceAccurate(["Westworld", "Sweetwater", "Las Mudas"])
# Insert new location
since = self.pillow.get_change_feed().get_latest_offsets()
self._make_loc("Blood Arroyo", self.town)
# Change an existing location
sweetwater.name = "Pariah"
sweetwater.save()
# Process both changes together and verify that they went through
self.pillow.process_changes(since=since, forever=False)
self.assertDataSourceAccurate(
["Westworld", "Pariah", "Las Mudas", "Blood Arroyo"])
# Delete a location
since = self.pillow.get_change_feed().get_latest_offsets()
las_mudas.delete()
self.pillow.process_changes(since=since, forever=False)
self.assertDataSourceAccurate(["Westworld", "Pariah", "Blood Arroyo"])
def test_location_data_source(self):
self._make_loc("Westworld", self.region)
sweetwater = self._make_loc("Sweetwater", self.town)
las_mudas = self._make_loc("Las Mudas", self.town)
rebuild_indicators(self.data_source_config._id)
self.assertDataSourceAccurate(["Westworld", "Sweetwater", "Las Mudas"])
# Insert new location
since = self.pillow.get_change_feed().get_current_offsets()
self._make_loc("Blood Arroyo", self.town)
# Change an existing location
sweetwater.name = "Pariah"
sweetwater.save()
# Process both changes together and verify that they went through
self.pillow.process_changes(since=since, forever=False)
self.assertDataSourceAccurate(["Westworld", "Pariah", "Las Mudas", "Blood Arroyo"])
# Delete a location
since = self.pillow.get_change_feed().get_current_offsets()
las_mudas.delete()
self.pillow.process_changes(since=since, forever=False)
# No actual change - deletions are not yet processed
self.assertDataSourceAccurate(["Westworld", "Pariah", "Las Mudas", "Blood Arroyo"])
def test_rebuild_indicators(self, datetime_mock):
datetime_mock.utcnow.return_value = self.fake_time_now
self.config.save()
sample_doc, _ = get_sample_doc_and_indicators(self.fake_time_now)
CommCareCase.get_db().save_doc(sample_doc)
rebuild_indicators(self.config._id)
self._check_sample_doc_state()
def _create_data_source(cls):
cls.data_source_config = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type="CommCareCase",
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {"type": "property_name", "property_name": "type"},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {"type": "property_name", "property_name": "first_name"},
"column_id": "indicator_col_id_first_name",
"display_name": "indicator_display_name_first_name",
"datatype": "string",
},
{
"type": "expression",
"expression": {"type": "property_name", "property_name": "number"},
"column_id": "indicator_col_id_number",
"datatype": "integer",
},
],
)
cls.data_source_config.validate()
cls.data_source_config.save()
rebuild_indicators(cls.data_source_config._id)
def handle(self, domains, **options):
if not domains and not options['all']:
raise CommandError('Specify specific domains or --all')
all_domains = [domain.name for domain in get_call_center_domains() if domain.use_fixtures]
if domains:
for domain in domains:
assert domain in all_domains, "Domain '{}' is not a Call Center domain".format(domain)
else:
domains = all_domains
threshold = options['threshold']
domain_stats = get_call_center_data_source_stats(domains)
for domain in domains:
stats = domain_stats[domain]
print('Checking domain:', domain)
if stats.error:
print('Error getting stats:\n', stats.error)
continue
for stat in stats.iter_data_source_stats():
diff = math.fabs(stat.ucr_percent - stat.es_percent)
if diff > threshold:
print("rebuilding data source '{}' in domain '{}': diff = {}".format(
stat.name, domain, diff
))
try:
rebuild_indicators(
StaticDataSourceConfiguration.get_doc_id(domain, TABLE_IDS[stat.name])
)
except Exception as e:
sys.stderr.write("Error rebuilding data source '{}' in domain '{}':\n{}".format(
stat.name, domain, e
))
def handle(self, filename, **options):
with open(filename, encoding='utf-8') as f:
body = json.loads(f.read())
# intentionally fails hard if bad or missing doc_type
cls = {
'DataSourceConfiguration': DataSourceConfiguration,
'ReportConfiguration': ReportConfiguration,
}[body['doc_type']]
if '_id' in body:
try:
rev = cls.get_db().get_rev(body['_id'])
body['_rev'] = rev
except ResourceNotFound:
pass
to_save = cls.wrap(body)
to_save.validate()
to_save.save()
print('updated {}: "{}"'.format(to_save.doc_type, to_save))
if options['rebuild'] and isinstance(to_save, DataSourceConfiguration):
print('rebuilding table...')
rebuild_indicators(to_save._id)
print('...done!')
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'indicator_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'city'
},
"column_id": 'indicator_col_id_city',
"display_name": 'indicator_display_name_city',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'indicator_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'city'
},
"column_id": 'indicator_col_id_city',
"display_name": 'indicator_display_name_city',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def handle(self, domain, **options):
tables = StaticDataSourceConfiguration.by_domain(domain)
tables.extend(DataSourceConfiguration.by_domain(domain))
print("Rebuilding {} tables".format(len(tables)))
for table in tables:
tasks.rebuild_indicators(table._id)
def handle(self, domain, **options):
tables = StaticDataSourceConfiguration.by_domain(domain)
tables.extend(DataSourceConfiguration.by_domain(domain))
print("Rebuilding {} tables".format(len(tables)))
for table in tables:
tasks.rebuild_indicators(table._id)
def handle(self, indicator_config_id, **options):
if options['in_place']:
tasks.rebuild_indicators_in_place(indicator_config_id,
options['initiated'],
source='rebuild_indicator_table')
else:
tasks.rebuild_indicators(indicator_config_id,
initiated_by=options['initiated'],
source='rebuild_indicator_table')
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
config = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_string',
"display_name": 'date_as_string',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_date',
"datatype": "date"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "my_datetime",
},
"column_id": "datetime_as_datetime",
"datatype": "datetime"
}
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
cls.data_sources[UCR_SQL_BACKEND] = config
cls.adapters[UCR_SQL_BACKEND] = adapter
def _build_report(self, vals, field="my_field", build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
self._new_case({field: v}).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name="foo",
referenced_doc_type="CommCareCase",
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {"type": "property_name", "property_name": "type"},
"property_value": self.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {"type": "property_name", "property_name": field},
"column_id": field,
"display_name": field,
"datatype": "string",
}
],
)
data_source_config.validate()
data_source_config.save()
if build_data_source:
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=self.domain,
config_id=data_source_config._id,
title="foo",
aggregation_columns=["doc_id"],
columns=[{"type": "expanded", "field": field, "display": field, "format": "default"}],
filters=[],
configured_charts=[],
)
report_config.save()
data_source = ReportFactory.from_spec(report_config)
return data_source, data_source.column_configs[0]
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
config = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_string',
"display_name": 'date_as_string',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'my_date'
},
"column_id": 'date_as_date',
"datatype": "date"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "my_datetime",
},
"column_id": "datetime_as_datetime",
"datatype": "datetime"
}
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
cls.data_sources[UCR_SQL_BACKEND] = config
cls.adapters[UCR_SQL_BACKEND] = adapter
def handle(self, indicator_config_id, **options):
if options['in_place']:
tasks.rebuild_indicators_in_place(
indicator_config_id, options['initiated'], source='rebuild_indicator_table'
)
else:
tasks.rebuild_indicators(
indicator_config_id,
initiated_by=options['initiated'],
source='rebuild_indicator_table'
)
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is rebuild_tables_by_domain %s' %
self.args)
domain = args[0]
tables = StaticDataSourceConfiguration.by_domain(domain)
tables.extend(DataSourceConfiguration.by_domain(domain))
print("Rebuilding {} tables".format(len(tables)))
for table in tables:
tasks.rebuild_indicators(table._id)
def calculate_adherence_update(self, input):
self.case_updater.purge_date = input[0]
# setup episode and adherence cases
adherence_schedule_date_start, adherence_schedule_id = input[1]
adherence_cases = input[2]
episode = self._create_episode_case(adherence_schedule_date_start,
adherence_schedule_id)
self._create_adherence_cases(adherence_cases)
rebuild_indicators(self.data_store.datasource._id)
self.data_store.adapter.refresh_table()
return EpisodeUpdate(episode, self.case_updater)
def _create_data_source(cls):
cls.data_sources = {}
cls.adapters = {}
# this is a hack to have both sql and es backends created in a class
# method. alternative would be to have these created on each test run
for backend_id in UCR_BACKENDS:
config = DataSourceConfiguration(
backend_id=backend_id,
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'first_name'
},
"column_id": 'indicator_col_id_first_name',
"display_name": 'indicator_display_name_first_name',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
],
)
config.validate()
config.save()
rebuild_indicators(config._id)
adapter = get_indicator_adapter(config)
adapter.refresh_table()
cls.data_sources[backend_id] = config
cls.adapters[backend_id] = adapter
def _create_data_source(cls):
cls.data_source = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'first_name'
},
"column_id": 'indicator_col_id_first_name',
"display_name": 'indicator_display_name_first_name',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'rank'
},
"column_id": 'indicator_col_id_rank',
"datatype": "integer"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
cls.adapter = get_indicator_adapter(cls.data_source)
def _build_report(self, vals, field='my_field', build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
update_props = {field: v} if v is not None else {}
self._new_case(update_props).save()
if build_data_source:
tasks.rebuild_indicators(self.data_source_config._id)
adapter = get_indicator_adapter(self.data_source_config)
adapter.refresh_table()
report_config = ReportConfiguration(
domain=self.domain,
config_id=self.data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[{
"type": "expanded",
"field": field,
"display": field,
"format": "default",
}],
filters=[],
configured_charts=[]
)
report_config.save()
self.addCleanup(report_config.delete)
data_source = ConfigurableReportDataSource.from_spec(report_config)
return data_source, data_source.top_level_columns[0]
def _build_report(self, vals, field='my_field', build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
update_props = {field: v} if v is not None else {}
self._new_case(update_props).save()
if build_data_source:
tasks.rebuild_indicators(self.data_source_config._id)
report_config = ReportConfiguration(
domain=self.domain,
config_id=self.data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[{
"type": "expanded",
"field": field,
"display": field,
"format": "default",
}],
filters=[],
configured_charts=[]
)
report_config.save()
self.addCleanup(report_config.delete)
data_source = ConfigurableReportDataSource.from_spec(report_config)
return data_source, data_source.top_level_columns[0]
def setUpClass(cls):
super(TestEpisodeDatasource, cls).setUpClass()
cls._create_case_structure()
rebuild_indicators(cls.datasource._id)
adapter = get_indicator_adapter(cls.datasource)
adapter.refresh_table()
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is rebuild_indicator_table %s' % self.args)
config_id = args[0]
tasks.rebuild_indicators(config_id)
def _build_report(self, vals, field='my_field', build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
update_props = {field: v} if v is not None else {}
self._new_case(update_props).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": self.case_type,
},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": field
},
"column_id": field,
"display_name": field,
"datatype": "string"
}],
)
data_source_config.validate()
data_source_config.save()
self.addCleanup(data_source_config.delete)
if build_data_source:
tasks.rebuild_indicators(data_source_config._id)
adapter = get_indicator_adapter(data_source_config)
adapter.refresh_table()
report_config = ReportConfiguration(domain=self.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[{
"type": "expanded",
"field": field,
"display": field,
"format": "default",
}],
filters=[],
configured_charts=[])
report_config.save()
self.addCleanup(report_config.delete)
data_source = ReportFactory.from_spec(report_config)
adapter = get_indicator_adapter(data_source_config)
if build_data_source:
adapter.refresh_table()
return data_source, data_source.top_level_columns[0]
def _build_report(cls):
# Create Cases
cls._new_case({'fruit': 'apple', 'num1': 4, 'num2': 6}).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=cls.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id="woop_woop",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'fruit'
},
"column_id": 'indicator_col_id_fruit',
"display_name": 'indicator_display_name_fruit',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num1'
},
"column_id": 'indicator_col_id_num1',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num2'
},
"column_id": 'indicator_col_id_num2',
"datatype": "integer"
},
],
)
data_source_config.validate()
data_source_config.save()
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=cls.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[
{
"type": "field",
"display": "report_column_display_fruit",
"field": 'indicator_col_id_fruit',
'column_id': 'report_column_col_id_fruit',
'aggregation': 'simple'
},
{
"type": "percent",
"display": "report_column_display_percent",
'column_id': 'report_column_col_id_percent',
'format': 'percent',
"denominator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num1",
"column_id": "report_column_col_id_percent_num1"
},
"numerator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num2",
"column_id": "report_column_col_id_percent_num2"
}
},
],
)
report_config.save()
return report_config
def _rebuild_indicators(self):
# rebuild so that adherence UCR data gets updated
rebuild_indicators(self.data_store.datasource._id)
self.data_store.adapter.refresh_table()
def _build_report(cls):
# Create Cases
cls._new_case({'fruit': 'apple', 'num1': 4, 'num2': 6}).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=cls.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id="woop_woop",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'fruit'
},
"column_id": 'indicator_col_id_fruit',
"display_name": 'indicator_display_name_fruit',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num1'
},
"column_id": 'indicator_col_id_num1',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num2'
},
"column_id": 'indicator_col_id_num2',
"datatype": "integer"
},
],
)
data_source_config.validate()
data_source_config.save()
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=cls.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[
{
"type": "field",
"display": "report_column_display_fruit",
"field": 'indicator_col_id_fruit',
'column_id': 'report_column_col_id_fruit',
'aggregation': 'simple'
},
{
"type": "percent",
"display": "report_column_display_percent",
'column_id': 'report_column_col_id_percent',
'format': 'percent',
"denominator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num1",
"column_id": "report_column_col_id_percent_num1"
},
"numerator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num2",
"column_id": "report_column_col_id_percent_num2"
}
},
],
)
report_config.save()
return report_config
def _create_data_source(cls):
cls.data_source = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'indicator_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
# Except for column_id, this indicator is identical to the indicator above
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'report_column_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'city'
},
"column_id": 'indicator_col_id_city',
"display_name": 'indicator_display_name_city',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'age_at_registration'
},
"column_id": 'age_at_registration',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'date'
},
"column_id": 'date',
"datatype": "date"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
adapter = get_indicator_adapter(cls.data_source)
cls.adapter = adapter
def _rebuild_table_get_query_object(self):
rebuild_indicators(self.datasource._id)
adapter = get_indicator_adapter(self.datasource)
adapter.refresh_table()
return adapter.get_query_object()
def handle(self, indicator_config_id, **options):
tasks.rebuild_indicators(indicator_config_id)
def test_rebuild_indicators(self):
self.config.save()
sample_doc, _ = get_sample_doc_and_indicators()
CommCareCase.get_db().save_doc(sample_doc)
rebuild_indicators(self.config._id)
self._check_sample_doc_state()
def _build_report(self, vals, field='my_field', build_data_source=True):
"""
Build a new report, and populate it with cases.
Return a ConfigurableReportDataSource and a FieldColumn
:param vals: List of values to populate the given report field with.
:param field: The name of a field in the data source/report
:return: Tuple containing a ConfigurableReportDataSource and FieldColumn.
The column is a column mapped to the given field.
"""
# Create Cases
for v in vals:
update_props = {field: v} if v is not None else {}
self._new_case(update_props).save()
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": self.case_type,
},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": field
},
"column_id": field,
"display_name": field,
"datatype": "string"
}],
)
data_source_config.validate()
data_source_config.save()
self.addCleanup(data_source_config.delete)
if build_data_source:
tasks.rebuild_indicators(data_source_config._id)
adapter = get_indicator_adapter(data_source_config)
adapter.refresh_table()
report_config = ReportConfiguration(
domain=self.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[{
"type": "expanded",
"field": field,
"display": field,
"format": "default",
}],
filters=[],
configured_charts=[]
)
report_config.save()
self.addCleanup(report_config.delete)
data_source = ReportFactory.from_spec(report_config)
adapter = get_indicator_adapter(data_source_config)
if build_data_source:
adapter.refresh_table()
return data_source, data_source.top_level_columns[0]
def _build_report_and_view(self):
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id="woop_woop",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": self.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'fruit'
},
"column_id": 'indicator_col_id_fruit',
"display_name": 'indicator_display_name_fruit',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num1'
},
"column_id": 'indicator_col_id_num1',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num2'
},
"column_id": 'indicator_col_id_num2',
"datatype": "integer"
},
],
)
data_source_config.validate()
data_source_config.save()
self.addCleanup(data_source_config.delete)
tasks.rebuild_indicators(data_source_config._id)
report_config = ReportConfiguration(
domain=self.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[
{
"type": "field",
"display": "report_column_display_fruit",
"field": 'indicator_col_id_fruit',
'column_id': 'report_column_col_id_fruit',
'aggregation': 'simple'
},
{
"type": "percent",
"display": "report_column_display_percent",
'column_id': 'report_column_col_id_percent',
'format': 'percent',
"denominator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num1",
"column_id": "report_column_col_id_percent_num1"
},
"numerator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num2",
"column_id": "report_column_col_id_percent_num2"
}
},
],
)
report_config.save()
self.addCleanup(report_config.delete)
view = ConfigurableReportView(request=HttpRequest())
view._domain = self.domain
view._lang = "en"
view._report_config_id = report_config._id
return report_config, view
def setUpClass(cls):
super(TestConfigurableReportDataResource, cls).setUpClass()
case_type = "my_case_type"
cls.field_name = "my_field"
cls.case_property_values = ["foo", "foo", "bar", "baz"]
cls.cases = []
for val in cls.case_property_values:
id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=id,
case_type=case_type,
update={cls.field_name: val},
).as_xml()
post_case_blocks([case_block], {'domain': cls.domain.name})
cls.cases.append(CommCareCase.get(id))
cls.report_columns = [
{
"column_id": cls.field_name,
"type": "field",
"field": cls.field_name,
"aggregation": "simple",
}
]
cls.report_filters = [
{
'datatype': 'string',
'field': cls.field_name,
'type': 'dynamic_choice_list',
'slug': 'my_field_filter',
}
]
cls.data_source = DataSourceConfiguration(
domain=cls.domain.name,
referenced_doc_type="CommCareCase",
table_id=uuid.uuid4().hex,
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": cls.field_name
},
"column_id": cls.field_name,
"display_name": cls.field_name,
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "opened_by"
},
"column_id": "opened_by",
"display_name": "opened_by",
"datatype": "string"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
cls.report_configuration = ReportConfiguration(
domain=cls.domain.name,
config_id=cls.data_source._id,
aggregation_columns=["doc_id"],
columns=cls.report_columns,
filters=cls.report_filters,
)
cls.report_configuration.save()
def _rebuild_table_get_query_object(self):
rebuild_indicators(self.datasource._id)
adapter = get_indicator_adapter(self.datasource)
adapter.refresh_table()
return adapter.get_query_object()
def setUpClass(cls):
super(TestConfigurableReportDataResource, cls).setUpClass()
case_type = "my_case_type"
cls.field_name = "my_field"
cls.case_property_values = ["foo", "foo", "bar", "baz"]
cls.cases = []
for val in cls.case_property_values:
id = uuid.uuid4().hex
case_block = CaseBlock(
create=True,
case_id=id,
case_type=case_type,
update={
cls.field_name: val
},
).as_xml()
post_case_blocks([case_block], {'domain': cls.domain.name})
cls.cases.append(CommCareCase.get(id))
cls.report_columns = [{
"column_id": cls.field_name,
"type": "field",
"field": cls.field_name,
"aggregation": "simple",
}]
cls.report_filters = [{
'datatype': 'string',
'field': cls.field_name,
'type': 'dynamic_choice_list',
'slug': 'my_field_filter',
}]
cls.data_source = DataSourceConfiguration(
domain=cls.domain.name,
referenced_doc_type="CommCareCase",
table_id=uuid.uuid4().hex,
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": cls.field_name
},
"column_id": cls.field_name,
"display_name": cls.field_name,
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "opened_by"
},
"column_id": "opened_by",
"display_name": "opened_by",
"datatype": "string"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
cls.report_configuration = ReportConfiguration(
domain=cls.domain.name,
config_id=cls.data_source._id,
aggregation_columns=["doc_id"],
columns=cls.report_columns,
filters=cls.report_filters,
)
cls.report_configuration.save()
def _build_report_and_view(self):
# Create report
data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='foo',
referenced_doc_type='CommCareCase',
table_id="woop_woop",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": self.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'fruit'
},
"column_id": 'indicator_col_id_fruit',
"display_name": 'indicator_display_name_fruit',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num1'
},
"column_id": 'indicator_col_id_num1',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'num2'
},
"column_id": 'indicator_col_id_num2',
"datatype": "integer"
},
],
)
data_source_config.validate()
data_source_config.save()
self.addCleanup(data_source_config.delete)
tasks.rebuild_indicators(data_source_config._id)
adapter = get_indicator_adapter(data_source_config)
adapter.refresh_table()
report_config = ReportConfiguration(
domain=self.domain,
config_id=data_source_config._id,
title='foo',
aggregation_columns=['doc_id'],
columns=[
{
"type": "field",
"display": "report_column_display_fruit",
"field": 'indicator_col_id_fruit',
'column_id': 'report_column_col_id_fruit',
'aggregation': 'simple'
},
{
"type": "percent",
"display": "report_column_display_percent",
'column_id': 'report_column_col_id_percent',
'format': 'percent',
"denominator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num1",
"column_id": "report_column_col_id_percent_num1"
},
"numerator": {
"type": "field",
"aggregation": "sum",
"field": "indicator_col_id_num2",
"column_id": "report_column_col_id_percent_num2"
}
},
],
)
report_config.save()
self.addCleanup(report_config.delete)
view = ConfigurableReport(request=HttpRequest())
view._domain = self.domain
view._lang = "en"
view._report_config_id = report_config._id
return report_config, view
def _rebuild_table_get_query_object(self):
rebuild_indicators(self.datasource._id)
adapter = IndicatorSqlAdapter(self.datasource)
return adapter.get_query_object()
def _rebuild_datasource(domain):
config_id = StaticDataSourceConfiguration.get_doc_id(
domain, 'food_consumption_indicators')
rebuild_indicators(config_id, source='populate_inddex_test_domain')
def handle(self, indicator_config_id, **options):
if options['in_place']:
tasks.rebuild_indicators_in_place(indicator_config_id, options['initiated'])
else:
tasks.rebuild_indicators(indicator_config_id, options['initiated'])
def _create_data_source(cls):
cls.data_source = DataSourceConfiguration(
domain=cls.domain,
display_name=cls.domain,
referenced_doc_type='CommCareCase',
table_id="foo",
configured_filter={
"type": "boolean_expression",
"operator": "eq",
"expression": {
"type": "property_name",
"property_name": "type"
},
"property_value": cls.case_type,
},
configured_indicators=[
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'indicator_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
# Except for column_id, this indicator is identical to the indicator above
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'state'
},
"column_id": 'report_column_col_id_state',
"display_name": 'indicator_display_name_state',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'city'
},
"column_id": 'indicator_col_id_city',
"display_name": 'indicator_display_name_city',
"datatype": "string"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'number'
},
"column_id": 'indicator_col_id_number',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'age_at_registration'
},
"column_id": 'age_at_registration',
"datatype": "integer"
},
{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": 'date'
},
"column_id": 'date',
"datatype": "date"
},
],
)
cls.data_source.validate()
cls.data_source.save()
rebuild_indicators(cls.data_source._id)
adapter = get_indicator_adapter(cls.data_source)
cls.adapter = adapter
def handle(self, indicator_config_id, **options):
if options['in_place']:
tasks.rebuild_indicators_in_place(indicator_config_id,
options['initiated'])
else:
tasks.rebuild_indicators(indicator_config_id, options['initiated'])
