def update_report(self): from corehq.apps.userreports.views import delete_data_source_shared matching_data_source = self.ds_builder.get_existing_match() if matching_data_source: reactivated = False if matching_data_source._id != self.existing_report.config_id: # If no one else is using the current data source, delete it. data_source = DataSourceConfiguration.get(self.existing_report.config_id) if data_source.get_report_count() <= 1: data_source.deactivate() self.existing_report.config_id = matching_data_source._id elif matching_data_source.is_deactivated: existing_sources = DataSourceConfiguration.by_domain(self.domain) active_sources = filter(lambda config: not config.is_deactivated, existing_sources) if len(active_sources) >= 5: raise forms.ValidationError(_( "Editing this report would require a new data source. The limit is 5. " "To continue, first delete all of the reports using a particular " "data source (or the data source itself) and try again. " )) matching_data_source.is_deactivated = False reactivated = True changed = False indicators = self.ds_builder.indicators(self._number_columns) if matching_data_source.configured_indicators != indicators: matching_data_source.configured_indicators = indicators changed = True if changed or reactivated: matching_data_source.save() tasks.rebuild_indicators.delay(matching_data_source._id) else: # We need to create a new data source existing_sources = DataSourceConfiguration.by_domain(self.domain) active_sources = filter(lambda config: not config.is_deactivated, existing_sources) # Delete the old one if no other reports use it old_data_source = DataSourceConfiguration.get(self.existing_report.config_id) if old_data_source.get_report_count() <= 1: old_data_source.deactivate() # Make sure the user can create more data sources elif len(active_sources) >= 5: raise forms.ValidationError(_( "Editing this report would require a new data source. The limit is 5. " "To continue, first delete all of the reports using a particular " "data source (or the data source itself) and try again. " )) data_source_config_id = self._build_data_source() self.existing_report.config_id = data_source_config_id self.existing_report.aggregation_columns = self._report_aggregation_cols self.existing_report.columns = self._report_columns self.existing_report.filters = self._report_filters self.existing_report.configured_charts = self._report_charts self.existing_report.validate() self.existing_report.save() return self.existing_report
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith( CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) # Save the start time now in case anything goes wrong. This way we'll be # able to see if the rebuild started a long time ago without finishing. config.meta.build.initiated = datetime.datetime.utcnow() config.save() adapter = IndicatorSqlAdapter(config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format( doc['_id'], e)) if not is_static: config.meta.build.finished = True config.save()
def _iteratively_build_table(config, last_id=None, resume_helper=None): resume_helper = resume_helper or DataSourceResumeHelper(config) indicator_config_id = config._id relevant_ids = [] document_store = get_document_store(config.domain, config.referenced_doc_type) for relevant_id in document_store.iter_document_ids(last_id): relevant_ids.append(relevant_id) if len(relevant_ids) >= ID_CHUNK_SIZE: resume_helper.set_ids_to_resume_from(relevant_ids) _build_indicators(config, document_store, relevant_ids, resume_helper) relevant_ids = [] if relevant_ids: resume_helper.set_ids_to_resume_from(relevant_ids) _build_indicators(config, document_store, relevant_ids, resume_helper) if not id_is_static(indicator_config_id): resume_helper.clear_ids() config.meta.build.finished = True try: config.save() except ResourceConflict: current_config = DataSourceConfiguration.get(config._id) # check that a new build has not yet started if config.meta.build.initiated == current_config.meta.build.initiated: current_config.meta.build.finished = True current_config.save() adapter = get_indicator_adapter(config, raise_errors=True, can_handle_laboratory=True) adapter.after_table_build()
def _iteratively_build_table(config, last_id=None, resume_helper=None): resume_helper = resume_helper or DataSourceResumeHelper(config) indicator_config_id = config._id relevant_ids = [] document_store = get_document_store(config.domain, config.referenced_doc_type) for relevant_id in document_store.iter_document_ids(last_id): relevant_ids.append(relevant_id) if len(relevant_ids) >= ID_CHUNK_SIZE: resume_helper.set_ids_to_resume_from(relevant_ids) _build_indicators(config, document_store, relevant_ids, resume_helper) relevant_ids = [] if relevant_ids: resume_helper.set_ids_to_resume_from(relevant_ids) _build_indicators(config, document_store, relevant_ids, resume_helper) if not id_is_static(indicator_config_id): resume_helper.clear_ids() config.meta.build.finished = True try: config.save() except ResourceConflict: current_config = DataSourceConfiguration.get(config._id) # check that a new build has not yet started if config.meta.build.initiated == current_config.meta.build.initiated: current_config.meta.build.finished = True current_config.save()
def _build_indicators(indicator_config_id, relevant_ids): config = _get_config_by_id(indicator_config_id) adapter = IndicatorSqlAdapter(config) couchdb = _get_db(config.referenced_doc_type) redis_client = get_redis_client().client.get_client() redis_key = _get_redis_key_for_config(config) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) redis_client.srem(redis_key, doc.get('_id')) except Exception as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e)) if not is_static(indicator_config_id): redis_client.delete(redis_key) config.meta.build.finished = True try: config.save() except ResourceConflict: current_config = DataSourceConfiguration.get(config._id) # check that a new build has not yet started if config.meta.build.initiated == current_config.meta.build.initiated: current_config.meta.build.finished = True current_config.save()
def test_stale_rebuild(self): # rebuild indicators in another test will save this later_config = DataSourceConfiguration.get(self.config._id) later_config.save() self.assertNotEqual(self.config._rev, later_config._rev) with self.assertRaises(StaleRebuildError): self.pillow.processors[0].rebuild_table(get_indicator_adapter(self.config))
def test_updating_report_data_source(self): """ Test that changing the app or number column for a report results in an update to the data source next time the report is saved. """ # Make report builder_form = ConfigureTableReportForm( "Test Report", self.app._id, "case", "some_case_type", existing_report=None, data={ 'group_by': ['closed'], 'chart': 'bar', 'user_filters': '[]', 'default_filters': '[]', 'columns': '[{"property": "closed", "display_text": "closed", "calculation": "Count per Choice"}]', }) self.assertTrue(builder_form.is_valid()) report = builder_form.create_report() self.assertEqual(report.config.configured_indicators[0]['datatype'], "string") # Make an edit to the first report builder report builder_form = ConfigureTableReportForm( "Test Report", self.app._id, "case", "some_case_type", existing_report=report, data={ 'group_by': ['user_id'], 'chart': 'bar', 'user_filters': '[]', 'default_filters': '[]', # Note that a "Sum" calculation on the closed case property isn't very sensical, but doing it so # that I can have a numeric calculation without having to create real case properties for this case # type. 'columns': '[{"property": "closed", "display_text": "closed", "calculation": "Sum"}]', }) self.assertTrue(builder_form.is_valid()) builder_form.update_report() # reload report data source, because report.config is memoized data_source = DataSourceConfiguration.get(report.config._id) # The closed property indicator should now be decimal type because the user indicated that it was numeric # by giving the column the "Sum" aggregation. self.assertEqual(data_source.configured_indicators[0]['datatype'], "decimal")
def test_stale_rebuild(self): # rebuild indicators in another test will save this later_config = DataSourceConfiguration.get(self.config._id) later_config.save() self.assertNotEqual(self.config._rev, later_config._rev) with self.assertRaises(StaleRebuildError): self.pillow.rebuild_table(get_indicator_adapter(self.config))
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) # Save the start time now in case anything goes wrong. This way we'll be # able to see if the rebuild started a long time ago without finishing. config.meta.build.initiated = datetime.datetime.utcnow() config.save() adapter = IndicatorSqlAdapter(config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e)) if not is_static: config.meta.build.finished = True config.save()
def test_invalid_data_serial_processor(self, process_changes_chunk): # re-fetch from DB to bust object caches self.config = DataSourceConfiguration.get(self.config.data_source_id) process_changes_chunk.side_effect = Exception self.config.validations = [ Validation.wrap({ "name": "impossible_condition", "error_message": "This condition is impossible to satisfy", "expression": { "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "doesnt_exist" }, "operator": "in", "property_value": ["nonsense"] } }) ] self.config.save() cases = self._create_and_process_changes() num_rows = self.adapter.get_query_object().count() self.assertEqual(num_rows, 0) invalid_data = InvalidUCRData.objects.all().values_list('doc_id', flat=True) self.assertEqual(set([case.case_id for case in cases]), set(invalid_data))
def tearDown(self): self.adapter.clear_table() delete_all_cases() delete_all_xforms() InvalidUCRData.objects.all().delete() self.config = DataSourceConfiguration.get(self.config.data_source_id) self.config.validations = [] self.config.save()
def tearDown(self): delete_all_cases() delete_all_xforms() AsyncIndicator.objects.all().delete() InvalidUCRData.objects.all().delete() self.config = DataSourceConfiguration.get(self.config.data_source_id) self.config.validations = [] self.config.save()
def test_link_creates_datasource_and_report(self): link_info = create_linked_ucr(self.domain_link, self.report.get_id) new_datasource = DataSourceConfiguration.get(link_info.datasource.get_id) self.assertEqual(new_datasource.domain, self.domain_link.linked_domain) new_report = ReportConfiguration.get(link_info.report.get_id) self.assertEqual(new_report.domain, self.domain_link.linked_domain) self.assertEqual(self.report.get_id, new_report.report_meta.master_id)
def ucr_config(request, domain, config_id): report_config = ReportConfiguration.get(config_id) datasource_id = report_config.config_id datasource_config = DataSourceConfiguration.get(datasource_id) return JsonResponse({ "report": report_config.to_json(), "datasource": datasource_config.to_json(), })
def _iteratively_build_table(config, resume_helper=None, in_place=False, limit=-1): resume_helper = resume_helper or DataSourceResumeHelper(config) indicator_config_id = config._id case_type_or_xmlns_list = config.get_case_type_or_xmlns_filter() completed_ct_xmlns = resume_helper.get_completed_case_type_or_xmlns() if completed_ct_xmlns: case_type_or_xmlns_list = [ case_type_or_xmlns for case_type_or_xmlns in case_type_or_xmlns_list if case_type_or_xmlns not in completed_ct_xmlns ] for case_type_or_xmlns in case_type_or_xmlns_list: relevant_ids = [] document_store = get_document_store( config.domain, config.referenced_doc_type, case_type_or_xmlns=case_type_or_xmlns) for i, relevant_id in enumerate(document_store.iter_document_ids()): if i >= limit > -1: break relevant_ids.append(relevant_id) if len(relevant_ids) >= ID_CHUNK_SIZE: _build_indicators(config, document_store, relevant_ids) relevant_ids = [] if relevant_ids: _build_indicators(config, document_store, relevant_ids) resume_helper.add_completed_case_type_or_xmlns(case_type_or_xmlns) resume_helper.clear_resume_info() if not id_is_static(indicator_config_id): if in_place: config.meta.build.finished_in_place = True else: config.meta.build.finished = True try: config.save() except ResourceConflict: current_config = DataSourceConfiguration.get(config._id) # check that a new build has not yet started if in_place: if config.meta.build.initiated_in_place == current_config.meta.build.initiated_in_place: current_config.meta.build.finished_in_place = True else: if config.meta.build.initiated == current_config.meta.build.initiated: current_config.meta.build.finished = True current_config.save() adapter = get_indicator_adapter(config, raise_errors=True, can_handle_laboratory=True) adapter.after_table_build()
def update_report(self): from corehq.apps.userreports.views import delete_data_source_shared matching_data_source = self.ds_builder.get_existing_match() if matching_data_source: if matching_data_source['id'] != self.existing_report.config_id: # If no one else is using the current data source, delete it. data_source = DataSourceConfiguration.get( self.existing_report.config_id) if data_source.get_report_count() <= 1: delete_data_source_shared(self.domain, data_source._id) self.existing_report.config_id = matching_data_source['id'] else: # We need to create a new data source existing_sources = DataSourceConfiguration.by_domain(self.domain) # Delete the old one if no other reports use it old_data_source = DataSourceConfiguration.get( self.existing_report.config_id) if old_data_source.get_report_count() <= 1: delete_data_source_shared(self.domain, old_data_source._id) # Make sure the user can create more data sources elif len(existing_sources) >= 5: raise forms.ValidationError( _("Editing this report would require a new data source. The limit is 5. " "To continue, first delete all of the reports using a particular " "data source (or the data source itself) and try again. " )) data_source_config_id = self._build_data_source() self.existing_report.config_id = data_source_config_id self.existing_report.aggregation_columns = self._report_aggregation_cols self.existing_report.columns = self._report_columns self.existing_report.filters = self._report_filters self.existing_report.configured_charts = self._report_charts self.existing_report.validate() self.existing_report.save() return self.existing_report
def create_linked_ucr(domain_link, report_config_id): if domain_link.is_remote: remote_configs = remote_get_ucr_config(domain_link, report_config_id) datasource = remote_configs["datasource"] report_config = remote_configs["report"] else: report_config = ReportConfiguration.get(report_config_id) datasource = DataSourceConfiguration.get(report_config.config_id) new_datasource = _get_or_create_datasource_link(domain_link, datasource) new_report = _get_or_create_report_link(domain_link, report_config, new_datasource) return LinkedUCRInfo(datasource=new_datasource, report=new_report)
def update_report(self): from corehq.apps.userreports.views import delete_data_source_shared matching_data_source = self.ds_builder.get_existing_match() if matching_data_source: reactivated = False if matching_data_source._id != self.existing_report.config_id: # If no one else is using the current data source, delete it. data_source = DataSourceConfiguration.get(self.existing_report.config_id) if data_source.get_report_count() <= 1: data_source.deactivate() self.existing_report.config_id = matching_data_source._id elif matching_data_source.is_deactivated: matching_data_source.is_deactivated = False reactivated = True changed = False indicators = self.ds_builder.indicators(self._number_columns) if matching_data_source.configured_indicators != indicators: matching_data_source.configured_indicators = indicators changed = True if changed or reactivated: matching_data_source.save() tasks.rebuild_indicators.delay(matching_data_source._id) else: # Delete the old one if no other reports use it old_data_source = DataSourceConfiguration.get(self.existing_report.config_id) if old_data_source.get_report_count() <= 1: old_data_source.deactivate() data_source_config_id = self._build_data_source() self.existing_report.config_id = data_source_config_id self.existing_report.aggregation_columns = self._report_aggregation_cols self.existing_report.columns = self._report_columns self.existing_report.filters = self._report_filters self.existing_report.configured_charts = self._report_charts self.existing_report.validate() self.existing_report.save() return self.existing_report
def update_report(self): from corehq.apps.userreports.views import delete_data_source_shared matching_data_source = self.ds_builder.get_existing_match() if matching_data_source: if matching_data_source['id'] != self.existing_report.config_id: # If no one else is using the current data source, delete it. data_source = DataSourceConfiguration.get(self.existing_report.config_id) if data_source.get_report_count() <= 1: delete_data_source_shared(self.domain, data_source._id) self.existing_report.config_id = matching_data_source['id'] else: # We need to create a new data source existing_sources = DataSourceConfiguration.by_domain(self.domain) # Delete the old one if no other reports use it old_data_source = DataSourceConfiguration.get(self.existing_report.config_id) if old_data_source.get_report_count() <= 1: delete_data_source_shared(self.domain, old_data_source._id) # Make sure the user can create more data sources elif len(existing_sources) >= 5: raise forms.ValidationError(_( "Editing this report would require a new data source. The limit is 5. " "To continue, first delete all of the reports using a particular " "data source (or the data source itself) and try again. " )) data_source_config_id = self._build_data_source() self.existing_report.config_id = data_source_config_id self.existing_report.aggregation_columns = self._report_aggregation_cols self.existing_report.columns = self._report_columns self.existing_report.filters = self._report_filters self.existing_report.configured_charts = self._report_charts self.existing_report.validate() self.existing_report.save() return self.existing_report
def test_async_invalid_data(self): # re-fetch from DB to bust object caches self.config = DataSourceConfiguration.get(self.config.data_source_id) self.config.validations = [ Validation.wrap({ "name": "impossible_condition", "error_message": "This condition is impossible to satisfy", "expression": { "type": "boolean_expression", "expression": { "type": "property_name", "property_name": "doesnt_exist" }, "operator": "in", "property_value": ["nonsense"] } }) ] self.config.save() parent_id, child_id = uuid.uuid4().hex, uuid.uuid4().hex since = self.pillow.get_change_feed().get_latest_offsets() for i in range(3): form, cases = post_case_blocks([ CaseBlock.deprecated_init( create=i == 0, case_id=parent_id, case_name='parent-name', case_type='bug', update={ 'update-prop-parent': i }, ).as_xml(), CaseBlock.deprecated_init(create=i == 0, case_id=child_id, case_name='child-name', case_type='bug-child', index={ 'parent': ('bug', parent_id) }, update={ 'update-prop-child': i }).as_xml() ], domain=self.domain) self.pillow.process_changes(since=since, forever=False) # run async queue queue_async_indicators() self.assertEqual(InvalidUCRData.objects.count(), 1)
def _iteratively_build_table(config, resume_helper=None, in_place=False, limit=-1): resume_helper = resume_helper or DataSourceResumeHelper(config) indicator_config_id = config._id case_type_or_xmlns_list = config.get_case_type_or_xmlns_filter() completed_ct_xmlns = resume_helper.get_completed_case_type_or_xmlns() if completed_ct_xmlns: case_type_or_xmlns_list = [ case_type_or_xmlns for case_type_or_xmlns in case_type_or_xmlns_list if case_type_or_xmlns not in completed_ct_xmlns ] for case_type_or_xmlns in case_type_or_xmlns_list: relevant_ids = [] document_store = get_document_store_for_doc_type( config.domain, config.referenced_doc_type, case_type_or_xmlns=case_type_or_xmlns, load_source="build_indicators", ) for i, relevant_id in enumerate(document_store.iter_document_ids()): if i >= limit > -1: break relevant_ids.append(relevant_id) if len(relevant_ids) >= ID_CHUNK_SIZE: _build_indicators(config, document_store, relevant_ids) relevant_ids = [] if relevant_ids: _build_indicators(config, document_store, relevant_ids) resume_helper.add_completed_case_type_or_xmlns(case_type_or_xmlns) resume_helper.clear_resume_info() if not id_is_static(indicator_config_id): if in_place: config.meta.build.finished_in_place = True else: config.meta.build.finished = True try: config.save() except ResourceConflict: current_config = DataSourceConfiguration.get(config._id) # check that a new build has not yet started if in_place: if config.meta.build.initiated_in_place == current_config.meta.build.initiated_in_place: current_config.meta.build.finished_in_place = True else: if config.meta.build.initiated == current_config.meta.build.initiated: current_config.meta.build.finished = True current_config.save()
def test_updating_report_data_source(self): """ Test that changing the app or number column for a report results in an update to the data source next time the report is saved. """ # Make report builder_form = ConfigureTableReportForm( "Test Report", self.app._id, "case", "some_case_type", existing_report=None, data={ 'group_by': 'closed', 'user_filters': '[]', 'default_filters': '[]', 'columns': '[{"property": "closed", "display_text": "closed", "calculation": "Count per Choice"}]', } ) self.assertTrue(builder_form.is_valid()) report = builder_form.create_report() self.assertEqual(report.config.configured_indicators[0]['datatype'], "string") # Make an edit to the first report builder report builder_form = ConfigureTableReportForm( "Test Report", self.app._id, "case", "some_case_type", existing_report=report, data={ 'group_by': 'user_id', 'user_filters': '[]', 'default_filters': '[]', # Note that a "Sum" calculation on the closed case property isn't very sensical, but doing it so # that I can have a numeric calculation without having to create real case properties for this case # type. 'columns': '[{"property": "closed", "display_text": "closed", "calculation": "Sum"}]', } ) self.assertTrue(builder_form.is_valid()) builder_form.update_report() # reload report data source, because report.config is memoized data_source = DataSourceConfiguration.get(report.config._id) # The closed property indicator should now be decimal type because the user indicated that it was numeric # by giving the column the "Sum" aggregation. self.assertEqual(data_source.configured_indicators[0]['datatype'], "decimal")
def rebuild_indicators(indicator_config_id): config = DataSourceConfiguration.get(indicator_config_id) adapter = IndicatorSqlAdapter(get_engine(), config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): if config.filter.filter(doc): try: adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
def test_data_source_columns(self): """ Report Builder should create a data source that includes columns for all possible aggregations, so that if the user switches between a list report and a summary report the data source has all the required columns (FB 268655) """ builder_form = ConfigureListReportForm(self.domain, "My Report", self.app._id, "form", self.form.unique_id, data={ 'user_filters': '[]', 'default_filters': '[]', 'columns': """[ {"property": "/data/first_name", "display_text": "first name"}, {"property": "/data/last_name", "display_text": "last name"}, {"property": "/data/children", "display_text": "children"} ]""", }) self.assertTrue(builder_form.is_valid()) with patch('corehq.apps.userreports.tasks.delete_data_source_task'): data_source_config_id = builder_form.create_temp_data_source_if_necessary( '*****@*****.**') data_source = DataSourceConfiguration.get(data_source_config_id) indicators = sorted([(ind['column_id'], ind['type']) for ind in data_source.configured_indicators]) expected_indicators = [ ('count', 'boolean'), ('data_children_25bd0e0d', 'expression'), # "children" should have 2 columns because it is ('data_children_25bd0e0d_decimal', 'expression'), # numeric ('data_dob_b6293169', 'expression'), ('data_first_name_ac8c51a7', 'expression'), ('data_last_name_ce36e9e1', 'expression'), ('data_state_6e36b993', 'choice_list'), ('data_state_6e36b993', 'expression'), ('deviceID_a7307e7d', 'expression'), ('timeEnd_09f40526', 'expression'), ('timeStart_c5a1ba73', 'expression'), ('userID_41e1d44e', 'expression'), ('username_ea02198f', 'expression'), ] self.assertEqual(indicators, expected_indicators)
def migrate_linked_reports(upstream_domain=None): logger.setLevel(logging.INFO) if upstream_domain: domain_links = DomainLink.all_objects.filter( master_domain=upstream_domain) else: domain_links = DomainLink.all_objects.all() num_of_failed_attempts = 0 for domain_link in domain_links: reports = get_report_configs_for_domain(domain_link.linked_domain) for report in reports: if report.report_meta.master_id and not report.config.meta.build.app_id: upstream_report = ReportConfiguration.get( report.report_meta.master_id) upstream_datasource = DataSourceConfiguration.get( upstream_report.config_id) downstream_app_id = get_downstream_app_id( domain_link.linked_domain, upstream_datasource.meta.build.app_id, ) if not downstream_app_id: # just as a backup in case upstream_app_id is not set but family_id is downstream_app_id = get_downstream_app_id( domain_link.linked_domain, upstream_datasource.meta.build.app_id, use_upstream_app_id=False) if downstream_app_id: logger.info( f"Needed to use family_id to find downstream app {downstream_app_id}" ) if not downstream_app_id: logger.warning( f"Could not find downstream_app_id for upstream app" f" {upstream_datasource.meta.build.app_id} " f"in downstream domain {domain_link.linked_domain}") num_of_failed_attempts += 1 report.config.meta.build.app_id = downstream_app_id report.config.save() logger.info( f"Completed linked report migration with {num_of_failed_attempts} failed attempts" ) return num_of_failed_attempts
def create_linked_ucr(domain_link, report_config_id): if domain_link.is_remote: remote_configs = remote_get_ucr_config(domain_link, report_config_id) datasource = remote_configs["datasource"] report_config = remote_configs["report"] else: report_config = ReportConfiguration.get(report_config_id) datasource = DataSourceConfiguration.get(report_config.config_id) # grab the linked app this linked report references try: downstream_app_id = get_downstream_app_id(domain_link.linked_domain, datasource.meta.build.app_id) except MultipleDownstreamAppsError: raise DomainLinkError(_("This report cannot be linked because it references an app that has multiple " "downstream apps.")) new_datasource = _get_or_create_datasource_link(domain_link, datasource, downstream_app_id) new_report = _get_or_create_report_link(domain_link, report_config, new_datasource) return LinkedUCRInfo(datasource=new_datasource, report=new_report)
def _iteratively_build_table(config, last_id=None): couchdb = _get_db(config.referenced_doc_type) redis_client = get_redis_client().client.get_client() redis_key = _get_redis_key_for_config(config) indicator_config_id = config._id start_key = None if last_id: last_doc = _DOC_TYPE_MAPPING[config.referenced_doc_type].get(last_id) start_key = [config.domain, config.referenced_doc_type] if config.referenced_doc_type in _DATE_MAP.keys(): date = json_format_datetime(last_doc[_DATE_MAP[config.referenced_doc_type]]) start_key.append(date) relevant_ids = [] for relevant_id in iterate_doc_ids_in_domain_by_type( config.domain, config.referenced_doc_type, chunk_size=CHUNK_SIZE, database=couchdb, startkey=start_key, startkey_docid=last_id): relevant_ids.append(relevant_id) if len(relevant_ids) >= CHUNK_SIZE: redis_client.rpush(redis_key, *relevant_ids) _build_indicators(indicator_config_id, relevant_ids) relevant_ids = [] if relevant_ids: redis_client.rpush(redis_key, *relevant_ids) _build_indicators(indicator_config_id, relevant_ids) if not is_static(indicator_config_id): redis_client.delete(redis_key) config.meta.build.finished = True try: config.save() except ResourceConflict: current_config = DataSourceConfiguration.get(config._id) # check that a new build has not yet started if config.meta.build.initiated == current_config.meta.build.initiated: current_config.meta.build.finished = True current_config.save()
def test_skip_destructive_rebuild(self): self.config = self._get_config('add_non_nullable_col') self.config.disable_destructive_rebuild = True self.config.save() get_case_pillow(ucr_configs=[self.config]) self.adapter = get_indicator_adapter(self.config) self.engine = self.adapter.engine # assert new date isn't in the config insp = reflection.Inspector.from_engine(self.engine) table_name = get_table_name(self.config.domain, self.config.table_id) self.assertEqual( len([ c for c in insp.get_columns(table_name) if c['name'] == 'new_date' ]), 0) # add the column to the config self.config.configured_indicators.append({ "column_id": "new_date", "type": "raw", "display_name": "new_date opened", "datatype": "datetime", "property_name": "other_opened_on", "is_nullable": False }) self.config.save() # re-fetch from DB to bust object caches self.config = DataSourceConfiguration.get(self.config.data_source_id) # bootstrap to trigger rebuild get_case_pillow(ucr_configs=[self.config]) logs = DataSourceActionLog.objects.filter( indicator_config_id=self.config.data_source_id, skip_destructive=True) self.assertEqual(1, len(logs)) self.assertEqual(logs[0].migration_diffs, [{ 'type': 'add_column', 'item_name': 'new_date' }])
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) adapter = IndicatorSqlAdapter(get_engine(), config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e)) adapter.engine.dispose()
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith(StaticDataSourceConfiguration._datasource_id_prefix) if is_static: config = StaticDataSourceConfiguration.by_id(indicator_config_id) rev = 'static' else: config = DataSourceConfiguration.get(indicator_config_id) rev = config._rev # Save the start time now in case anything goes wrong. This way we'll be # able to see if the rebuild started a long time ago without finishing. config.meta.build.initiated = datetime.datetime.utcnow() config.save() adapter = IndicatorSqlAdapter(config) couchdb = _get_db(config.referenced_doc_type) client = get_redis_client().client.get_client() redis_key = 'ucr_queue-{}:{}'.format(indicator_config_id, rev) if len(client.smembers(redis_key)) > 0: relevant_ids = client.smembers(redis_key) else: adapter.rebuild_table() relevant_ids = get_doc_ids_in_domain_by_type(config.domain, config.referenced_doc_type, database=couchdb) if relevant_ids: client.sadd(redis_key, *relevant_ids) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) client.srem(redis_key, doc.get('_id')) except DataError as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e)) if not is_static: client.delete(redis_key) config.meta.build.finished = True config.save()
def _get_config_by_id(indicator_config_id): if is_static(indicator_config_id): return StaticDataSourceConfiguration.by_id(indicator_config_id) else: return DataSourceConfiguration.get(indicator_config_id)
def config(self): if self._config is None: self._config = DataSourceConfiguration.get(self._config_id) return self._config
def _get_config_by_id(indicator_config_id): if id_is_static(indicator_config_id): return StaticDataSourceConfiguration.by_id(indicator_config_id) else: return DataSourceConfiguration.get(indicator_config_id)
def tearDown(self): self.config = DataSourceConfiguration.get(self.config.data_source_id) self.config.delete() self.adapter.drop_table() delete_all_cases() delete_all_xforms()