def _shared_context(domain): custom_data_sources = list(CustomDataSourceConfiguration.by_domain(domain)) return { 'domain': domain, 'reports': ReportConfiguration.by_domain(domain), 'data_sources': DataSourceConfiguration.by_domain(domain) + custom_data_sources, }
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) # Save the start time now in case anything goes wrong. This way we'll be # able to see if the rebuild started a long time ago without finishing. config.meta.build.initiated = datetime.datetime.utcnow() config.save() adapter = IndicatorSqlAdapter(config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e)) if not is_static: config.meta.build.finished = True config.save()
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith( CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) # Save the start time now in case anything goes wrong. This way we'll be # able to see if the rebuild started a long time ago without finishing. config.meta.build.initiated = datetime.datetime.utcnow() config.save() adapter = IndicatorSqlAdapter(config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format( doc['_id'], e)) if not is_static: config.meta.build.finished = True config.save()
def get_datasource_config_or_404(config_id, domain): is_static = config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(config_id) if not config or config.domain != domain: raise Http404() else: config = get_document_or_404(DataSourceConfiguration, domain, config_id) return config, is_static
def test_get_all(self): with override_settings(CUSTOM_DATA_SOURCES=[self.get_path('sample_custom_data_source', 'json')]): all = list(CustomDataSourceConfiguration.all()) self.assertEqual(2, len(all)) example, dimagi = all self.assertEqual('example', example.domain) self.assertEqual('dimagi', dimagi.domain) for config in all: self.assertEqual('all_candidates', config.table_id)
def __init__(self, domain, *args, **kwargs): self.domain = domain standard_sources = DataSourceConfiguration.by_domain(self.domain) custom_sources = list(CustomDataSourceConfiguration.by_domain(domain)) available_data_sources = standard_sources + custom_sources super(ReportDataSourceField, self).__init__( choices=[(src._id, src.display_name) for src in available_data_sources], *args, **kwargs )
def test_get_all(self): with override_settings(CUSTOM_DATA_SOURCES=[self.get_path("sample_custom_data_source", "json")]): all = list(CustomDataSourceConfiguration.all()) self.assertEqual(2, len(all)) example, dimagi = all self.assertEqual("example", example.domain) self.assertEqual("dimagi", dimagi.domain) for config in all: self.assertEqual("all_candidates", config.table_id)
def __init__(self, domain, *args, **kwargs): self.domain = domain standard_sources = DataSourceConfiguration.by_domain(self.domain) custom_sources = list(CustomDataSourceConfiguration.by_domain(domain)) available_data_sources = standard_sources + custom_sources super(ReportDataSourceField, self).__init__(choices=[(src._id, src.display_name) for src in available_data_sources], *args, **kwargs)
def get_datasource_config_or_404(config_id, domain): is_static = config_id.startswith( CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(config_id) if not config or config.domain != domain: raise Http404() else: config = get_document_or_404(DataSourceConfiguration, domain, config_id) return config, is_static
def rebuild_indicators(indicator_config_id): is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix) if is_static: config = CustomDataSourceConfiguration.by_id(indicator_config_id) else: config = DataSourceConfiguration.get(indicator_config_id) adapter = IndicatorSqlAdapter(get_engine(), config) adapter.rebuild_table() couchdb = _get_db(config.referenced_doc_type) relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type, database=couchdb) for doc in iter_docs(couchdb, relevant_ids, chunksize=500): try: # save is a noop if the filter doesn't match adapter.save(doc) except DataError as e: logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e)) adapter.engine.dispose()
def get_asha_table(self, metadata): config = CustomDataSourceConfiguration.by_id(CustomDataSourceConfiguration.get_doc_id(TABLE_ID)) return get_indicator_table(config, custom_metadata=metadata)
def test_production_config(self): for data_source in CustomDataSourceConfiguration.all(): data_source.validate()
def test_wrap(self): wrapped = CustomDataSourceConfiguration.wrap(self.get_json("sample_custom_data_source")) self.assertEqual(["example", "dimagi"], wrapped.domains)
def get_asha_table(self, metadata): config = CustomDataSourceConfiguration.by_id( CustomDataSourceConfiguration.get_doc_id(DOMAIN, TABLE_ID)) return get_indicator_table(config, custom_metadata=metadata)
def get_all_configs(self): return CustomDataSourceConfiguration.all()
def test_wrap(self): wrapped = CustomDataSourceConfiguration.wrap(self.get_json('sample_custom_data_source')) self.assertEqual(["example", "dimagi"], wrapped.domains)