def resume_building_indicators(indicator_config_id, initiated_by=None):
config = _get_config_by_id(indicator_config_id)
success = _('Your UCR table {} has finished rebuilding').format(config.table_id)
failure = _('There was an error rebuilding Your UCR table {}.').format(config.table_id)
send = toggles.SEND_UCR_REBUILD_INFO.enabled(initiated_by)
with notify_someone(initiated_by, success_message=success, error_message=failure, send=send):
resume_helper = DataSourceResumeHelper(config)
relevant_ids = resume_helper.get_ids_to_resume_from()
if len(relevant_ids) > 0:
_build_indicators(config, get_document_store(config.domain, config.referenced_doc_type), relevant_ids,
resume_helper)
last_id = relevant_ids[-1]
iteratively_build_table(config, last_id, resume_helper)
def resume_building_indicators(indicator_config_id, initiated_by=None):
config = _get_config_by_id(indicator_config_id)
success = _('Your UCR table {} has finished rebuilding').format(config.table_id)
failure = _('There was an error rebuilding Your UCR table {}.').format(config.table_id)
send = toggles.SEND_UCR_REBUILD_INFO.enabled(initiated_by)
with notify_someone(initiated_by, success_message=success, error_message=failure, send=send):
resume_helper = DataSourceResumeHelper(config)
relevant_ids = resume_helper.get_ids_to_resume_from()
if len(relevant_ids) > 0:
_build_indicators(config, get_document_store(config.domain, config.referenced_doc_type), relevant_ids,
resume_helper)
last_id = relevant_ids[-1]
_iteratively_build_table(config, last_id, resume_helper)
def _iteratively_build_table(config, last_id=None, resume_helper=None):
resume_helper = resume_helper or DataSourceResumeHelper(config)
indicator_config_id = config._id
relevant_ids = []
document_store = get_document_store(config.domain,
config.referenced_doc_type)
for relevant_id in document_store.iter_document_ids(last_id):
relevant_ids.append(relevant_id)
if len(relevant_ids) >= ID_CHUNK_SIZE:
resume_helper.set_ids_to_resume_from(relevant_ids)
_build_indicators(config, document_store, relevant_ids,
resume_helper)
relevant_ids = []
if relevant_ids:
resume_helper.set_ids_to_resume_from(relevant_ids)
_build_indicators(config, document_store, relevant_ids, resume_helper)
if not id_is_static(indicator_config_id):
resume_helper.clear_ids()
config.meta.build.finished = True
try:
config.save()
except ResourceConflict:
current_config = DataSourceConfiguration.get(config._id)
# check that a new build has not yet started
if config.meta.build.initiated == current_config.meta.build.initiated:
current_config.meta.build.finished = True
current_config.save()
adapter = get_indicator_adapter(config,
raise_errors=True,
can_handle_laboratory=True)
adapter.after_table_build()
def _iteratively_build_table(config,
resume_helper=None,
in_place=False,
limit=-1):
resume_helper = resume_helper or DataSourceResumeHelper(config)
indicator_config_id = config._id
case_type_or_xmlns_list = config.get_case_type_or_xmlns_filter()
completed_ct_xmlns = resume_helper.get_completed_case_type_or_xmlns()
if completed_ct_xmlns:
case_type_or_xmlns_list = [
case_type_or_xmlns
for case_type_or_xmlns in case_type_or_xmlns_list
if case_type_or_xmlns not in completed_ct_xmlns
]
for case_type_or_xmlns in case_type_or_xmlns_list:
relevant_ids = []
document_store = get_document_store(
config.domain,
config.referenced_doc_type,
case_type_or_xmlns=case_type_or_xmlns)
for i, relevant_id in enumerate(document_store.iter_document_ids()):
if i >= limit > -1:
break
relevant_ids.append(relevant_id)
if len(relevant_ids) >= ID_CHUNK_SIZE:
_build_indicators(config, document_store, relevant_ids)
relevant_ids = []
if relevant_ids:
_build_indicators(config, document_store, relevant_ids)
resume_helper.add_completed_case_type_or_xmlns(case_type_or_xmlns)
resume_helper.clear_resume_info()
if not id_is_static(indicator_config_id):
if in_place:
config.meta.build.finished_in_place = True
else:
config.meta.build.finished = True
try:
config.save()
except ResourceConflict:
current_config = DataSourceConfiguration.get(config._id)
# check that a new build has not yet started
if in_place:
if config.meta.build.initiated_in_place == current_config.meta.build.initiated_in_place:
current_config.meta.build.finished_in_place = True
else:
if config.meta.build.initiated == current_config.meta.build.initiated:
current_config.meta.build.finished = True
current_config.save()
adapter = get_indicator_adapter(config,
raise_errors=True,
can_handle_laboratory=True)
adapter.after_table_build()
def resume_building_indicators(indicator_config_id, initiated_by=None):
config = _get_config_by_id(indicator_config_id)
success = _('Your UCR table {} has finished rebuilding').format(config.table_id)
failure = _('There was an error rebuilding Your UCR table {}.').format(config.table_id)
send = toggles.SEND_UCR_REBUILD_INFO.enabled(initiated_by)
with notify_someone(initiated_by, success_message=success, error_message=failure, send=send):
resume_helper = DataSourceResumeHelper(config)
_iteratively_build_table(config, resume_helper)
def _iteratively_build_table(config,
resume_helper=None,
in_place=False,
limit=-1):
resume_helper = resume_helper or DataSourceResumeHelper(config)
indicator_config_id = config._id
case_type_or_xmlns_list = config.get_case_type_or_xmlns_filter()
domains = config.data_domains
loop_iterations = list(itertools.product(domains, case_type_or_xmlns_list))
completed_iterations = resume_helper.get_completed_iterations()
if completed_iterations:
loop_iterations = list(
set(loop_iterations) - set(completed_iterations))
for domain, case_type_or_xmlns in loop_iterations:
relevant_ids = []
document_store = get_document_store_for_doc_type(
domain,
config.referenced_doc_type,
case_type_or_xmlns=case_type_or_xmlns,
load_source="build_indicators",
)
for i, relevant_id in enumerate(document_store.iter_document_ids()):
if i >= limit > -1:
break
relevant_ids.append(relevant_id)
if len(relevant_ids) >= ID_CHUNK_SIZE:
_build_indicators(config, document_store, relevant_ids)
relevant_ids = []
if relevant_ids:
_build_indicators(config, document_store, relevant_ids)
resume_helper.add_completed_iteration(domain, case_type_or_xmlns)
resume_helper.clear_resume_info()
if not id_is_static(indicator_config_id):
if in_place:
config.meta.build.finished_in_place = True
else:
config.meta.build.finished = True
try:
config.save()
except ResourceConflict:
current_config = get_ucr_datasource_config_by_id(config._id)
# check that a new build has not yet started
if in_place:
if config.meta.build.initiated_in_place == current_config.meta.build.initiated_in_place:
current_config.meta.build.finished_in_place = True
else:
if config.meta.build.initiated == current_config.meta.build.initiated:
current_config.meta.build.finished = True
current_config.save()
def resume_building_data_source(request, domain, config_id):
config, is_static = get_datasource_config_or_404(config_id, domain)
if not is_static and config.meta.build.finished:
messages.warning(
request,
_(u'Table "{}" has already finished building. Rebuild table to start over.'
).format(config.display_name))
elif not DataSourceResumeHelper(config).has_resume_info():
messages.warning(
request,
_(u'Table "{}" did not finish building but resume information is not available. '
u'Unfortunately, this means you need to rebuild the table.').
format(config.display_name))
else:
messages.success(
request,
_(u'Resuming rebuilding table "{}".').format(config.display_name))
resume_building_indicators.delay(config_id, request.user.username)
return HttpResponseRedirect(
reverse(EditDataSourceView.urlname, args=[domain, config._id]))
def setUpClass(cls):
super(DataSourceResumeBuildTest, cls).setUpClass()
cls._data_source = get_sample_data_source()
cls._resume_helper = DataSourceResumeHelper(cls._data_source)
def setUpClass(cls):
cls._data_source = get_sample_data_source()
cls._resume_helper = DataSourceResumeHelper(cls._data_source)