def validate(self, required=True): def _check_for_duplicates(supposedly_unique_list, error_msg): # http://stackoverflow.com/questions/9835762/find-and-list-duplicates-in-python-list duplicate_items = set( [item for item in supposedly_unique_list if supposedly_unique_list.count(item) > 1] ) if len(duplicate_items) > 0: raise BadSpecError( _(error_msg).format(', '.join(sorted(duplicate_items))) ) super(ReportConfiguration, self).validate(required) # check duplicates before passing to factory since it chokes on them _check_for_duplicates( [FilterSpec.wrap(f).slug for f in self.filters], 'Filters cannot contain duplicate slugs: {}', ) _check_for_duplicates( [column_id for c in self.report_columns for column_id in c.get_column_ids()], 'Columns cannot contain duplicate column_ids: {}', ) # these calls all implicitly do validation ReportFactory.from_spec(self) self.ui_filters self.charts self.sort_order
def test_report_data_source(self): # bootstrap report data sources against indicator data sources report_config_template = get_sample_report_config() report_config_1 = ReportConfiguration.wrap(report_config_template.to_json()) report_config_1.config_id = self.ds_1._id report_config_2 = ReportConfiguration.wrap(report_config_template.to_json()) report_config_2.config_id = self.ds_2._id # save a few docs to ds 1 sample_doc, _ = get_sample_doc_and_indicators() num_docs = 3 for i in range(num_docs): sample_doc['_id'] = uuid.uuid4().hex self.ds1_adapter.save(sample_doc) # ds 1 should have data, ds2 should not ds1_rows = ReportFactory.from_spec(report_config_1).get_data() self.assertEqual(1, len(ds1_rows)) self.assertEqual(num_docs, ds1_rows[0]['count']) ds2_rows = ReportFactory.from_spec(report_config_2).get_data() self.assertEqual(0, len(ds2_rows)) # save one doc to ds 2 sample_doc['_id'] = uuid.uuid4().hex self.ds2_adapter.save(sample_doc) # ds 1 should still have same data, ds2 should now have one row ds1_rows = ReportFactory.from_spec(report_config_1).get_data() self.assertEqual(1, len(ds1_rows)) self.assertEqual(num_docs, ds1_rows[0]['count']) ds2_rows = ReportFactory.from_spec(report_config_2).get_data() self.assertEqual(1, len(ds2_rows)) self.assertEqual(1, ds2_rows[0]['count'])
def validate(self, required=True): def _check_for_duplicate_slugs(filters): slugs = [FilterSpec.wrap(f).slug for f in filters] # http://stackoverflow.com/questions/9835762/find-and-list-duplicates-in-python-list duplicated_slugs = set([slug for slug in slugs if slugs.count(slug) > 1]) if len(duplicated_slugs) > 0: raise BadSpecError( _("Filters cannot contain duplicate slugs: %s") % ", ".join(sorted(duplicated_slugs)) ) super(ReportConfiguration, self).validate(required) # these calls implicitly do validation ReportFactory.from_spec(self) self.ui_filters self.charts _check_for_duplicate_slugs(self.filters)
def test_skip(self): count = 5 self._add_some_rows(count) report_data_source = ReportFactory.from_spec(self.report_config) original_data = report_data_source.get_data() self.assertEqual(count, len(original_data)) skipped = report_data_source.get_data(start=3) self.assertEqual(count - 3, len(skipped)) self.assertEqual(original_data[3:], skipped)
def test_limit(self): count = 5 self._add_some_rows(count) report_data_source = ReportFactory.from_spec(self.report_config) original_data = report_data_source.get_data() self.assertEqual(count, len(original_data)) limited_data = report_data_source.get_data(limit=3) self.assertEqual(3, len(limited_data)) self.assertEqual(original_data[:3], limited_data)
def __init__(self, domain, filters, report_id): report_config = ReportFactory.from_spec( get_document_or_not_found( ReportConfiguration, domain, report_id ) ) report_config.set_filter_values(filters) self.report_config = report_config
def _build_report(self, vals, field="my_field", build_data_source=True): """ Build a new report, and populate it with cases. Return a ConfigurableReportDataSource and a FieldColumn :param vals: List of values to populate the given report field with. :param field: The name of a field in the data source/report :return: Tuple containing a ConfigurableReportDataSource and FieldColumn. The column is a column mapped to the given field. """ # Create Cases for v in vals: self._new_case({field: v}).save() # Create report data_source_config = DataSourceConfiguration( domain=self.domain, display_name="foo", referenced_doc_type="CommCareCase", table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)), configured_filter={ "type": "boolean_expression", "operator": "eq", "expression": {"type": "property_name", "property_name": "type"}, "property_value": self.case_type, }, configured_indicators=[ { "type": "expression", "expression": {"type": "property_name", "property_name": field}, "column_id": field, "display_name": field, "datatype": "string", } ], ) data_source_config.validate() data_source_config.save() if build_data_source: tasks.rebuild_indicators(data_source_config._id) report_config = ReportConfiguration( domain=self.domain, config_id=data_source_config._id, title="foo", aggregation_columns=["doc_id"], columns=[{"type": "expanded", "field": field, "display": field, "format": "default"}], filters=[], configured_charts=[], ) report_config.save() data_source = ReportFactory.from_spec(report_config) return data_source, data_source.column_configs[0]
def test_group_by_missing_from_columns(self): report_config = ReportConfiguration( domain="somedomain", config_id="someconfig", aggregation_columns=["doc_id"], columns=[{"type": "field", "field": "somefield", "format": "default", "aggregation": "sum"}], filters=[], configured_charts=[], ) data_source = ReportFactory.from_spec(report_config) self.assertEqual(["doc_id"], data_source.group_by)
def test_basic_query(self): # add a few rows to the data source rows = self._add_some_rows(3) # check the returned data from the report looks right report_data_source = ReportFactory.from_spec(self.report_config) report_data = report_data_source.get_data() self.assertEqual(len(rows), len(report_data)) rows_by_name = {r.name: r for r in rows} for row in report_data: self.assertTrue(row['name'] in rows_by_name) self.assertEqual(rows_by_name[row['name']].number, row['number'])
def _get_report_data(self, report_config, domain, start, limit, get_params): report = ReportFactory.from_spec(report_config) filter_values = get_filter_values( report_config.ui_filters, query_dict_to_dict(get_params, domain) ) report.set_filter_values(filter_values) page = list(report.get_data(start=start, limit=limit)) total_records = report.get_total_records() return page, total_records
def _report_to_fixture(self, report): report_elem = ElementTree.Element('report', attrib={'id': report._id}) report_elem.append(self._element('name', report.title)) report_elem.append(self._element('description', report.description)) # todo: set filter values properly? data_source = ReportFactory.from_spec(report) rows_elem = ElementTree.Element('rows') for i, row in enumerate(data_source.get_data()): row_elem = ElementTree.Element('row', attrib={'index': str(i)}) for k in sorted(row.keys()): row_elem.append(self._element('column', self._serialize(row[k]), attrib={'id': k})) rows_elem.append(row_elem) report_elem.append(rows_elem) return report_elem
def test_group_by_missing_from_columns(self): report_config = ReportConfiguration( domain='somedomain', config_id='someconfig', aggregation_columns=['doc_id'], columns=[{ "type": "field", "field": "somefield", "format": "default", "aggregation": "sum" }], filters=[], configured_charts=[] ) data_source = ReportFactory.from_spec(report_config) with mock_sql_backend(): with mock_datasource_config(): self.assertEqual(['doc_id'], data_source.group_by)
def _report_to_fixture(self, report, filters, user): report_elem = ElementTree.Element('report', attrib={'id': report._id}) report_elem.append(self._element('name', report.title)) report_elem.append(self._element('description', report.description)) data_source = ReportFactory.from_spec(report) data_source.set_filter_values({ filter_slug: wrap_by_filter_type(filters[filter_slug]).get_filter_value(user) for filter_slug in filters }) rows_elem = ElementTree.Element('rows') for i, row in enumerate(data_source.get_data()): row_elem = ElementTree.Element('row', attrib={'index': str(i)}) for k in sorted(row.keys()): row_elem.append(self._element('column', self._serialize(row[k]), attrib={'id': k})) rows_elem.append(row_elem) report_elem.append(rows_elem) return report_elem
def _get_report_data(self, report_config, domain, start, limit, get_params): report = ReportFactory.from_spec(report_config) filter_values = get_filter_values( report_config.ui_filters, query_dict_to_dict(get_params, domain) ) report.set_filter_values(filter_values) page = list(report.get_data(start=start, limit=limit)) columns = [] for column in report.columns: simple_column = { "header": column.header, "slug": column.slug, } if isinstance(column, UCRExpandDatabaseSubcolumn): simple_column['expand_column_value'] = column.expand_value columns.append(simple_column) total_records = report.get_total_records() return page, columns, total_records
def _report_config_to_fixture(report_config, user): report_elem = E.report(id=report_config.uuid) try: report = ReportConfiguration.get(report_config.report_id) except ResourceNotFound as err: # ReportConfiguration not found raise BadReportConfigurationError('Error getting ReportConfiguration with ID "{}": {}'.format( report_config.report_id, err)) data_source = ReportFactory.from_spec(report) all_filter_values = { filter_slug: filter.get_filter_value(user, report.get_ui_filter(filter_slug)) for filter_slug, filter in report_config.filters.items() } filter_values = { filter_slug: filter_value for filter_slug, filter_value in all_filter_values.items() if filter_value is not None } defer_filters = { filter_slug: report.get_ui_filter(filter_slug) for filter_slug, filter_value in all_filter_values.items() if filter_value is None } data_source.set_filter_values(filter_values) data_source.defer_filters(defer_filters) rows_elem = E.rows() deferred_fields = {ui_filter.field for ui_filter in defer_filters.values()} filter_options_by_field = defaultdict(set) def _row_to_row_elem(row, index, is_total_row=False): row_elem = E.row(index=str(index), is_total_row=str(is_total_row)) for k in sorted(row.keys()): value = serialize(row[k]) row_elem.append(E.column(value, id=k)) if not is_total_row and k in deferred_fields: filter_options_by_field[k].add(value) return row_elem for i, row in enumerate(data_source.get_data()): rows_elem.append(_row_to_row_elem(row, i)) if data_source.has_total_row: total_row = data_source.get_total_row() rows_elem.append(_row_to_row_elem( dict( zip( map(lambda column_config: column_config.column_id, data_source.column_configs), map(str, total_row) ) ), data_source.get_total_records(), is_total_row=True, )) filters_elem = E.filters() for filter_slug, ui_filter in defer_filters.items(): # @field is maybe a bad name for this attribute, # since it's actually the filter slug filter_elem = E.filter(field=filter_slug) option_values = filter_options_by_field[ui_filter.field] choices = ui_filter.choice_provider.get_sorted_choices_for_values(option_values) for choice in choices: # add the correct text from ui_filter.choice_provider option_elem = E.option(choice.display, value=choice.value) filter_elem.append(option_elem) filters_elem.append(filter_elem) report_elem.append(filters_elem) report_elem.append(rows_elem) return report_elem
def charts(self): case_finding_sql_data = self.case_finding_sql_data[0] sputum_conversion_report = ReportFactory.from_spec( StaticReportConfiguration.by_id('static-%s-sputum_conversion' % self.domain), include_prefilters=True ) filter_values = {'date': QuarterFilter.get_value(self.request, self.domain)} locations_id = [ Choice(value=location_id, display='') for location_id in self.report_config.locations_id if location_id ] if locations_id: filter_values['village'] = locations_id sputum_conversion_report.set_filter_values(filter_values) sputum_conversion_data = sputum_conversion_report.get_data()[0] charts_sql_data = self.charts_sql_data[0] treatment_outcome_sql_data = self.treatment_outcome_sql_data[0] default_value = {'sort_key': 0} chart = PieChart(title=_('Cases by Gender'), key='gender', values=[]) chart.data = [ {'label': _('Male'), 'value': case_finding_sql_data.get('male_total', default_value)['sort_key']}, { 'label': _('Female'), 'value': case_finding_sql_data.get('female_total', default_value)['sort_key'] }, { 'label': _('Transgender'), 'value': case_finding_sql_data.get('transgender_total', default_value)['sort_key'] } ] chart2 = MultiBarChart(_('Cases By Type'), x_axis=Axis(''), y_axis=Axis('')) chart2.stacked = False chart2.showLegend = False positive_smear = case_finding_sql_data.get('new_positive_tb_pulmonary', default_value)['sort_key'] negative_smear = case_finding_sql_data.get('new_negative_tb_pulmonary', default_value)['sort_key'] positive_extra_pulmonary = case_finding_sql_data.get( 'new_positive_tb_extrapulmonary', default_value )['sort_key'] relapse_cases = case_finding_sql_data.get('recurrent_positive_tb', default_value)['sort_key'] failure_cases = case_finding_sql_data.get('failure_positive_tb', default_value)['sort_key'] lfu_cases = case_finding_sql_data.get('lfu_positive_tb', default_value)['sort_key'] others_cases = case_finding_sql_data.get('others_positive_tb', default_value)['sort_key'] chart2.add_dataset( _('New'), [ {'x': 'Smear +ve', 'y': positive_smear}, {'x': 'Smear -ve', 'y': negative_smear}, {'x': 'EP', 'y': positive_extra_pulmonary} ] ) chart2.add_dataset( _('Retreatment'), [ {'x': 'Relapse', 'y': relapse_cases}, {'x': 'Failure', 'y': failure_cases}, {'x': 'Treatment After Default', 'y': lfu_cases}, {'x': 'Others', 'y': others_cases} ] ) chart3 = MultiBarChart('Sputum Conversion By Patient Type', Axis(''), Axis('')) chart3.stacked = True chart3.add_dataset('Positive', [ { 'x': _('New Sputum +ve (2 month IP)'), 'y': sputum_conversion_data.get('new_sputum_positive_patient_2months_ip', 0) }, { 'x': _('New Sputum +ve (3 month IP)'), 'y': sputum_conversion_data.get('new_sputum_positive_patient_3months_ip', 0) }, { 'x': _('Cat II (3 month IP)'), 'y': sputum_conversion_data.get('positive_endofip_patients_cat2', 0) }, ]) chart3.add_dataset(_('Negative'), [ { 'x': _('New Sputum +ve (2 month IP)'), 'y': sputum_conversion_data.get('new_sputum_negative_patient_2months_ip', 0) }, { 'x': _('New Sputum +ve (3 month IP)'), 'y': sputum_conversion_data.get('new_sputum_negative_patient_3months_ip', 0) }, { 'x': _('Cat II (3 month IP)'), 'y': sputum_conversion_data.get('negative_endofip_patients_cat2', 0) }, ]) chart3.add_dataset('NA', [ { 'x': _('New Sputum +ve (2 month IP)'), 'y': sputum_conversion_data.get('new_sputum_na_patient_2months_ip', 0) }, { 'x': _('New Sputum +ve (3 month IP)'), 'y': sputum_conversion_data.get('new_sputum_na_patient_3months_ip', 0) }, { 'x': _('Cat II (3 month IP)'), 'y': sputum_conversion_data.get('na_endofip_patients_cat2', 0) }, ]) chart4 = PieChart( title=_('Total number of patients by category'), key='', values=[] ) chart4.data = [ { 'label': _('Cat1'), 'value': charts_sql_data.get('cat1_patients', default_value)['sort_key'] }, { 'label': _('Cat2'), 'value': charts_sql_data.get('cat2_patients', default_value)['sort_key'] } ] chart5 = MultiBarChart('Outcome By Type', Axis(''), Axis('')) chart5.stacked = True chart5.add_dataset(_('Cured'), [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_cured', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get('recurrent_patients_cured', default_value)['sort_key'] } ]) chart5.add_dataset('Treatment Complete', [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_treatment_complete', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get( 'recurrent_patients_treatment_complete', default_value)['sort_key'] } ]) chart5.add_dataset('Died', [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_died', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get('recurrent_patients_died', default_value)['sort_key'] } ]) chart5.add_dataset(_('Failure'), [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_treatment_failure', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get( 'recurrent_patients_treatment_failure', default_value )['sort_key'] } ]) chart5.add_dataset(_('Loss to Follow-up'), [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_loss_to_follow_up', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get( 'recurrent_patients_loss_to_follow_up', default_value )['sort_key'] } ]) chart5.add_dataset(_('Regimen Changed'), [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_regimen_changed', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get( 'recurrent_patients_regimen_changed', default_value )['sort_key'] } ]) chart5.add_dataset('Not Evaluated', [ { 'x': _('New'), 'y': treatment_outcome_sql_data.get('new_patients_not_evaluated', default_value)['sort_key'] }, { 'x': _('Retreatment'), 'y': treatment_outcome_sql_data.get('recurrent_patients_not_evaluated', default_value)['sort_key'] } ]) return [ chart, chart2, chart3, chart4, chart5 ]
def validate(self, required=True): super(ReportConfiguration, self).validate(required) # these calls implicitly do validation ReportFactory.from_spec(self) self.ui_filters self.charts
def _build_report(self, vals, field='my_field', build_data_source=True): """ Build a new report, and populate it with cases. Return a ConfigurableReportDataSource and a FieldColumn :param vals: List of values to populate the given report field with. :param field: The name of a field in the data source/report :return: Tuple containing a ConfigurableReportDataSource and FieldColumn. The column is a column mapped to the given field. """ # Create Cases for v in vals: update_props = {field: v} if v is not None else {} self._new_case(update_props).save() # Create report data_source_config = DataSourceConfiguration( domain=self.domain, display_name='foo', referenced_doc_type='CommCareCase', table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)), configured_filter={ "type": "boolean_expression", "operator": "eq", "expression": { "type": "property_name", "property_name": "type" }, "property_value": self.case_type, }, configured_indicators=[{ "type": "expression", "expression": { "type": "property_name", "property_name": field }, "column_id": field, "display_name": field, "datatype": "string" }], ) data_source_config.validate() data_source_config.save() self.addCleanup(data_source_config.delete) if build_data_source: tasks.rebuild_indicators(data_source_config._id) adapter = get_indicator_adapter(data_source_config) adapter.refresh_table() report_config = ReportConfiguration(domain=self.domain, config_id=data_source_config._id, title='foo', aggregation_columns=['doc_id'], columns=[{ "type": "expanded", "field": field, "display": field, "format": "default", }], filters=[], configured_charts=[]) report_config.save() self.addCleanup(report_config.delete) data_source = ReportFactory.from_spec(report_config) adapter = get_indicator_adapter(data_source_config) if build_data_source: adapter.refresh_table() return data_source, data_source.top_level_columns[0]
def _get_report_and_data_source(report_id, domain): report = get_report_config(report_id, domain)[0] data_source = ReportFactory.from_spec(report, include_prefilters=True) return report, data_source
def data_source(self): report = ReportFactory.from_spec(self.spec, include_prefilters=True) report.lang = self.lang return report
def __init__(self, domain, filters, report_id): report_config = ReportFactory.from_spec( StaticReportConfiguration.by_id(report_id.format(domain=domain)) ) report_config.set_filter_values(filters) self.report_config = report_config
def _get_report_and_data_source(report_id, domain): report = get_report_config(report_id, domain)[0] data_source = ReportFactory.from_spec(report) return report, data_source
def setUp(self): super(ReportTranslationTest, self).setUp() report = ReportConfiguration.by_domain(self.DOMAIN)[0] self.report_source = ReportFactory.from_spec(report)
def data_source(self): return ReportFactory.from_spec(self.spec)
def _build_report(self, vals, field='my_field', build_data_source=True): """ Build a new report, and populate it with cases. Return a ConfigurableReportDataSource and a FieldColumn :param vals: List of values to populate the given report field with. :param field: The name of a field in the data source/report :return: Tuple containing a ConfigurableReportDataSource and FieldColumn. The column is a column mapped to the given field. """ # Create Cases for v in vals: update_props = {field: v} if v is not None else {} self._new_case(update_props).save() # Create report data_source_config = DataSourceConfiguration( domain=self.domain, display_name='foo', referenced_doc_type='CommCareCase', table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)), configured_filter={ "type": "boolean_expression", "operator": "eq", "expression": { "type": "property_name", "property_name": "type" }, "property_value": self.case_type, }, configured_indicators=[{ "type": "expression", "expression": { "type": "property_name", "property_name": field }, "column_id": field, "display_name": field, "datatype": "string" }], ) data_source_config.validate() data_source_config.save() self.addCleanup(data_source_config.delete) if build_data_source: tasks.rebuild_indicators(data_source_config._id) adapter = get_indicator_adapter(data_source_config) adapter.refresh_table() report_config = ReportConfiguration( domain=self.domain, config_id=data_source_config._id, title='foo', aggregation_columns=['doc_id'], columns=[{ "type": "expanded", "field": field, "display": field, "format": "default", }], filters=[], configured_charts=[] ) report_config.save() self.addCleanup(report_config.delete) data_source = ReportFactory.from_spec(report_config) adapter = get_indicator_adapter(data_source_config) if build_data_source: adapter.refresh_table() return data_source, data_source.top_level_columns[0]
def data_source(self): report = ReportFactory.from_spec(self.spec) report.lang = self.request.couch_user.language return report
def data_source(self): report = ReportFactory.from_spec(self.spec) report.lang = self.lang return report
def setUp(self): report = ReportConfiguration.by_domain(self.DOMAIN)[0] self.report_source = ReportFactory.from_spec(report)