def _shared_context(domain):
custom_data_sources = list(CustomDataSourceConfiguration.by_domain(domain))
return {
'domain': domain,
'reports': ReportConfiguration.by_domain(domain),
'data_sources': DataSourceConfiguration.by_domain(domain) + custom_data_sources,
}
def rebuild_indicators(indicator_config_id):
is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(indicator_config_id)
else:
config = DataSourceConfiguration.get(indicator_config_id)
# Save the start time now in case anything goes wrong. This way we'll be
# able to see if the rebuild started a long time ago without finishing.
config.meta.build.initiated = datetime.datetime.utcnow()
config.save()
adapter = IndicatorSqlAdapter(config)
adapter.rebuild_table()
couchdb = _get_db(config.referenced_doc_type)
relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type,
database=couchdb)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
except DataError as e:
logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
if not is_static:
config.meta.build.finished = True
config.save()
def rebuild_indicators(indicator_config_id):
is_static = indicator_config_id.startswith(
CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(indicator_config_id)
else:
config = DataSourceConfiguration.get(indicator_config_id)
# Save the start time now in case anything goes wrong. This way we'll be
# able to see if the rebuild started a long time ago without finishing.
config.meta.build.initiated = datetime.datetime.utcnow()
config.save()
adapter = IndicatorSqlAdapter(config)
adapter.rebuild_table()
couchdb = _get_db(config.referenced_doc_type)
relevant_ids = get_doc_ids(config.domain,
config.referenced_doc_type,
database=couchdb)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
except DataError as e:
logging.exception('problem saving document {} to table. {}'.format(
doc['_id'], e))
if not is_static:
config.meta.build.finished = True
config.save()
def get_datasource_config_or_404(config_id, domain):
is_static = config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(config_id)
if not config or config.domain != domain:
raise Http404()
else:
config = get_document_or_404(DataSourceConfiguration, domain, config_id)
return config, is_static
def test_get_all(self):
with override_settings(CUSTOM_DATA_SOURCES=[self.get_path('sample_custom_data_source', 'json')]):
all = list(CustomDataSourceConfiguration.all())
self.assertEqual(2, len(all))
example, dimagi = all
self.assertEqual('example', example.domain)
self.assertEqual('dimagi', dimagi.domain)
for config in all:
self.assertEqual('all_candidates', config.table_id)
def __init__(self, domain, *args, **kwargs):
self.domain = domain
standard_sources = DataSourceConfiguration.by_domain(self.domain)
custom_sources = list(CustomDataSourceConfiguration.by_domain(domain))
available_data_sources = standard_sources + custom_sources
super(ReportDataSourceField, self).__init__(
choices=[(src._id, src.display_name) for src in available_data_sources],
*args, **kwargs
)
def test_get_all(self):
with override_settings(CUSTOM_DATA_SOURCES=[self.get_path("sample_custom_data_source", "json")]):
all = list(CustomDataSourceConfiguration.all())
self.assertEqual(2, len(all))
example, dimagi = all
self.assertEqual("example", example.domain)
self.assertEqual("dimagi", dimagi.domain)
for config in all:
self.assertEqual("all_candidates", config.table_id)
def __init__(self, domain, *args, **kwargs):
self.domain = domain
standard_sources = DataSourceConfiguration.by_domain(self.domain)
custom_sources = list(CustomDataSourceConfiguration.by_domain(domain))
available_data_sources = standard_sources + custom_sources
super(ReportDataSourceField,
self).__init__(choices=[(src._id, src.display_name)
for src in available_data_sources],
*args,
**kwargs)
def _shared_context(domain):
custom_data_sources = list(CustomDataSourceConfiguration.by_domain(domain))
return {
'domain':
domain,
'reports':
ReportConfiguration.by_domain(domain),
'data_sources':
DataSourceConfiguration.by_domain(domain) + custom_data_sources,
}
def get_datasource_config_or_404(config_id, domain):
is_static = config_id.startswith(
CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(config_id)
if not config or config.domain != domain:
raise Http404()
else:
config = get_document_or_404(DataSourceConfiguration, domain,
config_id)
return config, is_static
def rebuild_indicators(indicator_config_id):
is_static = indicator_config_id.startswith(CustomDataSourceConfiguration._datasource_id_prefix)
if is_static:
config = CustomDataSourceConfiguration.by_id(indicator_config_id)
else:
config = DataSourceConfiguration.get(indicator_config_id)
adapter = IndicatorSqlAdapter(get_engine(), config)
adapter.rebuild_table()
couchdb = _get_db(config.referenced_doc_type)
relevant_ids = get_doc_ids(config.domain, config.referenced_doc_type,
database=couchdb)
for doc in iter_docs(couchdb, relevant_ids, chunksize=500):
try:
# save is a noop if the filter doesn't match
adapter.save(doc)
except DataError as e:
logging.exception('problem saving document {} to table. {}'.format(doc['_id'], e))
adapter.engine.dispose()
def get_asha_table(self, metadata):
config = CustomDataSourceConfiguration.by_id(CustomDataSourceConfiguration.get_doc_id(TABLE_ID))
return get_indicator_table(config, custom_metadata=metadata)
def test_production_config(self):
for data_source in CustomDataSourceConfiguration.all():
data_source.validate()
def test_wrap(self):
wrapped = CustomDataSourceConfiguration.wrap(self.get_json("sample_custom_data_source"))
self.assertEqual(["example", "dimagi"], wrapped.domains)
def test_production_config(self):
for data_source in CustomDataSourceConfiguration.all():
data_source.validate()
def get_asha_table(self, metadata):
config = CustomDataSourceConfiguration.by_id(
CustomDataSourceConfiguration.get_doc_id(DOMAIN, TABLE_ID))
return get_indicator_table(config, custom_metadata=metadata)
def get_all_configs(self):
return CustomDataSourceConfiguration.all()
def get_all_configs(self):
return CustomDataSourceConfiguration.all()
def test_wrap(self):
wrapped = CustomDataSourceConfiguration.wrap(self.get_json('sample_custom_data_source'))
self.assertEqual(["example", "dimagi"], wrapped.domains)
