def _save_document_helper(indicator, doc):
eval_context = EvaluationContext(doc)
something_failed = False
for config_id in indicator.indicator_config_ids:
adapter = None
try:
config = _get_config(config_id)
except (ResourceNotFound, StaticDataSourceConfigurationNotFoundError):
celery_task_logger.info(
"{} no longer exists, skipping".format(config_id))
continue
except ESError:
celery_task_logger.info(
"ES errored when trying to retrieve config")
something_failed = True
return
try:
adapter = get_indicator_adapter(config, can_handle_laboratory=True)
adapter.save(doc, eval_context)
eval_context.reset_iteration()
except (DatabaseError, ESError, InternalError, RequestError,
ConnectionTimeout, ProtocolError, ReadTimeout):
# a database had an issue so log it and go on to the next document
celery_task_logger.info(
"DB error when saving config: {}".format(config_id))
something_failed = True
return
except Exception as e:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, e)
something_failed = True
return
return not something_failed
def process_change(self, pillow_instance, change):
self.bootstrap_if_needed()
if change.deleted:
# we don't currently support hard-deletions at all.
# we may want to change this at some later date but seem ok for now.
# see https://github.com/dimagi/commcare-hq/pull/6944 for rationale
return
domain = change.metadata.domain
if not domain:
# if no domain we won't save to any UCR table
return
async_tables = []
doc = change.get_document()
ensure_document_exists(change)
ensure_matched_revisions(change)
if doc is None:
return
eval_context = EvaluationContext(doc)
for table in self.table_adapters_by_domain[domain]:
if table.config.filter(doc):
if table.run_asynchronous:
async_tables.append(table.config._id)
else:
self._save_doc_to_table(table, doc, eval_context)
eval_context.reset_iteration()
elif table.config.deleted_filter(doc):
table.delete(doc)
if async_tables:
AsyncIndicator.update_indicators(change, async_tables)
def _process_chunk_for_domain(self, domain, changes_chunk):
adapters = list(self.table_adapters_by_domain[domain])
changes_by_id = {change.id: change for change in changes_chunk}
to_delete_by_adapter = defaultdict(list)
rows_to_save_by_adapter = defaultdict(list)
async_configs_by_doc_id = defaultdict(list)
to_update = {change for change in changes_chunk if not change.deleted}
retry_changes, docs = self.get_docs_for_changes(to_update, domain)
change_exceptions = []
for doc in docs:
eval_context = EvaluationContext(doc)
for adapter in adapters:
if adapter.config.filter(doc):
if adapter.run_asynchronous:
async_configs_by_doc_id[doc['_id']].append(
adapter.config._id)
else:
try:
rows_to_save_by_adapter[adapter].extend(
adapter.get_all_values(doc, eval_context))
except Exception as e:
change_exceptions.append(
(changes_by_id[doc["_id"]], e))
eval_context.reset_iteration()
elif adapter.config.deleted_filter(doc) or adapter.doc_exists(
doc):
to_delete_by_adapter[adapter].append(doc['_id'])
# bulk delete by adapter
to_delete = [c.id for c in changes_chunk if c.deleted]
for adapter in adapters:
delete_ids = to_delete_by_adapter[adapter] + to_delete
try:
adapter.bulk_delete(delete_ids)
except Exception as ex:
notify_exception(
None, "Error in deleting changes chunk {ids}: {ex}".format(
ids=delete_ids, ex=ex))
retry_changes.update(
[c for c in changes_chunk if c.id in delete_ids])
# bulk update by adapter
for adapter, rows in six.iteritems(rows_to_save_by_adapter):
try:
adapter.save_rows(rows)
except Exception as ex:
notify_exception(
None, "Error in saving changes chunk {ids}: {ex}".format(
ids=[c.id for c in to_update], ex=repr(ex)))
retry_changes.update(to_update)
if async_configs_by_doc_id:
doc_type_by_id = {
_id: changes_by_id[_id].metadata.document_type
for _id in async_configs_by_doc_id.keys()
}
AsyncIndicator.bulk_update_records(async_configs_by_doc_id, domain,
doc_type_by_id)
return retry_changes, change_exceptions
def _process_chunk_for_domain(self, domain, changes_chunk):
adapters = list(self.table_adapters_by_domain[domain])
changes_by_id = {change.id: change for change in changes_chunk}
to_delete_by_adapter = defaultdict(list)
rows_to_save_by_adapter = defaultdict(list)
async_configs_by_doc_id = defaultdict(list)
to_update = {change for change in changes_chunk if not change.deleted}
retry_changes, docs = self.get_docs_for_changes(to_update, domain)
change_exceptions = []
for doc in docs:
eval_context = EvaluationContext(doc)
for adapter in adapters:
if adapter.config.filter(doc):
if adapter.run_asynchronous:
async_configs_by_doc_id[doc['_id']].append(adapter.config._id)
else:
try:
rows_to_save_by_adapter[adapter].extend(adapter.get_all_values(doc, eval_context))
except Exception as e:
change_exceptions.append((changes_by_id[doc["_id"]], e))
eval_context.reset_iteration()
elif adapter.config.deleted_filter(doc) or adapter.doc_exists(doc):
to_delete_by_adapter[adapter].append(doc['_id'])
# bulk delete by adapter
to_delete = [c.id for c in changes_chunk if c.deleted]
for adapter in adapters:
delete_ids = to_delete_by_adapter[adapter] + to_delete
try:
adapter.bulk_delete(delete_ids)
except Exception as ex:
notify_exception(
None,
"Error in deleting changes chunk {ids}: {ex}".format(
ids=delete_ids, ex=ex))
retry_changes.update([c for c in changes_chunk if c.id in delete_ids])
# bulk update by adapter
for adapter, rows in six.iteritems(rows_to_save_by_adapter):
try:
adapter.save_rows(rows)
except Exception as ex:
notify_exception(
None,
"Error in saving changes chunk {ids}: {ex}".format(
ids=[c.id for c in to_update], ex=repr(ex)))
retry_changes.update(to_update)
if async_configs_by_doc_id:
doc_type_by_id = {
_id: changes_by_id[_id].metadata.document_type
for _id in async_configs_by_doc_id.keys()
}
AsyncIndicator.bulk_update_records(async_configs_by_doc_id, domain, doc_type_by_id)
return retry_changes, change_exceptions
def _save_document_helper(indicator, doc):
eval_context = EvaluationContext(doc)
something_failed = False
configs_to_remove = []
configs = dict()
for config_id in indicator.indicator_config_ids:
try:
configs[config_id] = _get_config(config_id)
except (ResourceNotFound, StaticDataSourceConfigurationNotFoundError):
celery_task_logger.info(
"{} no longer exists, skipping".format(config_id))
configs_to_remove.append(config_id)
continue
except ESError:
celery_task_logger.info(
"ES errored when trying to retrieve config")
something_failed = True
continue
for config_id, config in six.iteritems(configs):
adapter = None
try:
adapter = get_indicator_adapter(config, can_handle_laboratory=True)
adapter.save(doc, eval_context)
eval_context.reset_iteration()
except (ProtocolError, ReadTimeout):
celery_task_logger.info(
"Riak error when saving config: {}".format(config_id))
something_failed = True
except RequestError:
celery_task_logger.info(
"Couch error when saving config: {}".format(config_id))
something_failed = True
except (ESError, ConnectionTimeout):
# a database had an issue so log it and go on to the next document
celery_task_logger.info(
"ES error when saving config: {}".format(config_id))
something_failed = True
except (DatabaseError, InternalError):
# a database had an issue so log it and go on to the next document
celery_task_logger.info(
"psql error when saving config: {}".format(config_id))
something_failed = True
except Exception as e:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, e)
something_failed = True
else:
configs_to_remove.append(config_id)
rebuild_related_docs = any(config.icds_rebuild_related_docs
for config in six.itervalues(configs) if config)
return (not something_failed, configs_to_remove, rebuild_related_docs)
def save_document(doc_ids):
lock_keys = []
for doc_id in doc_ids:
lock_keys.append(get_async_indicator_modify_lock_key(doc_id))
with CriticalSection(lock_keys):
indicators = AsyncIndicator.objects.filter(doc_id__in=doc_ids)
if not indicators:
return
first_indicator = indicators[0]
processed_indicators = []
failed_indicators = []
for i in indicators:
assert i.domain == first_indicator.domain
assert i.doc_type == first_indicator.doc_type
indicator_by_doc_id = {i.doc_id: i for i in indicators}
doc_store = get_document_store(first_indicator.domain,
first_indicator.doc_type)
for doc in doc_store.iter_documents(doc_ids):
indicator = indicator_by_doc_id[doc['_id']]
eval_context = EvaluationContext(doc)
for config_id in indicator.indicator_config_ids:
adapter = None
try:
config = _get_config(config_id)
adapter = get_indicator_adapter(config,
can_handle_laboratory=True)
adapter.save(doc, eval_context)
eval_context.reset_iteration()
except (ESError, RequestError, ConnectionTimeout):
# couch or es had an issue so don't log it and go on to the next doc
failed_indicators.append(indicator.pk)
break
except Exception as e:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, e)
failed_indicators.append(indicator.pk)
break
else:
processed_indicators.append(indicator.pk)
AsyncIndicator.objects.filter(pk__in=processed_indicators).delete()
AsyncIndicator.objects.filter(pk__in=failed_indicators).update(
date_queued=None,
unsuccessful_attempts=F('unsuccessful_attempts') + 1)
def process_change(self, change):
self.bootstrap_if_needed()
domain = change.metadata.domain
if not domain or domain not in self.table_adapters_by_domain:
# if no domain we won't save to any UCR table
return
if change.deleted:
adapters = list(self.table_adapters_by_domain[domain])
for table in adapters:
table.delete({'_id': change.metadata.document_id})
async_tables = []
doc = change.get_document()
ensure_document_exists(change)
ensure_matched_revisions(change, doc)
if doc is None:
return
with TimingContext() as timer:
eval_context = EvaluationContext(doc)
# make copy to avoid modifying list during iteration
adapters = list(self.table_adapters_by_domain[domain])
doc_subtype = change.metadata.document_subtype
for table in adapters:
if table.config.filter(doc, eval_context):
if table.run_asynchronous:
async_tables.append(table.config._id)
else:
self._save_doc_to_table(domain, table, doc, eval_context)
eval_context.reset_iteration()
elif (doc_subtype is None
or doc_subtype in table.config.get_case_type_or_xmlns_filter()):
table.delete(doc)
if async_tables:
AsyncIndicator.update_from_kafka_change(change, async_tables)
self.domain_timing_context.update(**{
domain: timer.duration
})
def process_change(self, change):
self.bootstrap_if_needed()
domain = change.metadata.domain
if not domain or domain not in self.table_adapters_by_domain:
# if no domain we won't save to any UCR table
return
if change.deleted:
adapters = list(self.table_adapters_by_domain[domain])
for table in adapters:
table.delete({'_id': change.metadata.document_id})
async_tables = []
doc = change.get_document()
ensure_document_exists(change)
ensure_matched_revisions(change, doc)
if doc is None:
return
with TimingContext() as timer:
eval_context = EvaluationContext(doc)
# make copy to avoid modifying list during iteration
adapters = list(self.table_adapters_by_domain[domain])
for table in adapters:
if table.config.filter(doc):
if table.run_asynchronous:
async_tables.append(table.config._id)
else:
self._save_doc_to_table(domain, table, doc, eval_context)
eval_context.reset_iteration()
elif table.config.deleted_filter(doc) or table.doc_exists(doc):
table.delete(doc)
if async_tables:
AsyncIndicator.update_from_kafka_change(change, async_tables)
self.domain_timing_context.update(**{
domain: timer.duration
})
def build_async_indicators(indicator_doc_ids):
# written to be used with _queue_indicators, indicator_doc_ids must
# be a chunk of 100
memoizers = {'configs': {}, 'adapters': {}}
assert(len(indicator_doc_ids)) <= ASYNC_INDICATOR_CHUNK_SIZE
def handle_exception(exception, config_id, doc, adapter):
metric = None
if isinstance(exception, (ProtocolError, ReadTimeout)):
metric = 'commcare.async_indicator.riak_error'
elif isinstance(exception, (ESError, ConnectionTimeout)):
# a database had an issue so log it and go on to the next document
metric = 'commcare.async_indicator.es_error'
elif isinstance(exception, (DatabaseError, InternalError)):
# a database had an issue so log it and go on to the next document
metric = 'commcare.async_indicator.psql_error'
else:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, exception)
if metric:
metrics_counter(metric, tags={'config_id': config_id})
def doc_ids_from_rows(rows):
formatted_rows = [
{column.column.database_column_name.decode('utf-8'): column.value for column in row}
for row in rows
]
return set(row['doc_id'] for row in formatted_rows)
def _get_config(config_id):
config_by_id = memoizers['configs']
if config_id in config_by_id:
return config_by_id[config_id]
else:
config = _get_config_by_id(config_id)
config_by_id[config_id] = config
return config
def _get_adapter(config):
adapter_by_config = memoizers['adapters']
if config._id in adapter_by_config:
return adapter_by_config[config._id]
else:
adapter = get_indicator_adapter(config, load_source='build_async_indicators')
adapter_by_config[config._id] = adapter
return adapter
def _metrics_timer(step, config_id=None):
tags = {
'action': step,
}
if config_id and settings.ENTERPRISE_MODE:
tags['config_id'] = config_id
else:
# Prometheus requires consistent tags even if not available
tags['config_id'] = None
return metrics_histogram_timer(
'commcare.async_indicator.timing',
timing_buckets=(.03, .1, .3, 1, 3, 10), tags=tags
)
# tracks processed/deleted configs to be removed from each indicator
configs_to_remove_by_indicator_id = defaultdict(list)
def _mark_config_to_remove(config_id, indicator_ids):
for _id in indicator_ids:
configs_to_remove_by_indicator_id[_id].append(config_id)
timer = TimingContext()
lock_keys = [
get_async_indicator_modify_lock_key(indicator_doc_id)
for indicator_doc_id in indicator_doc_ids
]
with CriticalSection(lock_keys):
all_indicators = AsyncIndicator.objects.filter(
doc_id__in=indicator_doc_ids
)
if not all_indicators:
return
doc_store = get_document_store_for_doc_type(
all_indicators[0].domain, all_indicators[0].doc_type,
load_source="build_async_indicators",
)
failed_indicators = set()
rows_to_save_by_adapter = defaultdict(list)
docs_to_delete_by_adapter = defaultdict(list)
# there will always be one AsyncIndicator per doc id
indicator_by_doc_id = {i.doc_id: i for i in all_indicators}
config_ids = set()
with timer:
for doc in doc_store.iter_documents(list(indicator_by_doc_id.keys())):
indicator = indicator_by_doc_id[doc['_id']]
eval_context = EvaluationContext(doc)
for config_id in indicator.indicator_config_ids:
with _metrics_timer('transform', config_id):
config_ids.add(config_id)
try:
config = _get_config(config_id)
except (ResourceNotFound, StaticDataSourceConfigurationNotFoundError):
celery_task_logger.info("{} no longer exists, skipping".format(config_id))
# remove because the config no longer exists
_mark_config_to_remove(config_id, [indicator.pk])
continue
except ESError:
celery_task_logger.info("ES errored when trying to retrieve config")
failed_indicators.add(indicator)
continue
adapter = None
try:
adapter = _get_adapter(config)
rows_to_save = adapter.get_all_values(doc, eval_context)
if rows_to_save:
rows_to_save_by_adapter[adapter].extend(rows_to_save)
else:
docs_to_delete_by_adapter[adapter].append(doc)
eval_context.reset_iteration()
except Exception as e:
failed_indicators.add(indicator)
handle_exception(e, config_id, doc, adapter)
with _metrics_timer('single_batch_update'):
for adapter, rows in rows_to_save_by_adapter.items():
doc_ids = doc_ids_from_rows(rows)
indicators = [indicator_by_doc_id[doc_id] for doc_id in doc_ids]
try:
with _metrics_timer('update', adapter.config._id):
adapter.save_rows(rows, use_shard_col=True)
except Exception as e:
failed_indicators.union(indicators)
message = str(e)
notify_exception(None, "Exception bulk saving async indicators:{}".format(message))
else:
# remove because it's successfully processed
_mark_config_to_remove(
config_id,
[i.pk for i in indicators]
)
with _metrics_timer('single_batch_delete'):
for adapter, docs in docs_to_delete_by_adapter.items():
with _metrics_timer('delete', adapter.config._id):
adapter.bulk_delete(docs)
# delete fully processed indicators
processed_indicators = set(all_indicators) - failed_indicators
AsyncIndicator.objects.filter(pk__in=[i.pk for i in processed_indicators]).delete()
# update failure for failed indicators
with transaction.atomic():
for indicator in failed_indicators:
indicator.update_failure(
configs_to_remove_by_indicator_id.get(indicator.pk, [])
)
indicator.save()
metrics_counter('commcare.async_indicator.processed_success', len(processed_indicators))
metrics_counter('commcare.async_indicator.processed_fail', len(failed_indicators))
metrics_counter(
'commcare.async_indicator.processing_time', timer.duration,
tags={'config_ids': config_ids}
)
metrics_counter(
'commcare.async_indicator.processed_total', len(indicator_doc_ids),
tags={'config_ids': config_ids}
)
def _build_async_indicators(indicator_doc_ids):
def handle_exception(exception, config_id, doc, adapter):
metric = None
if isinstance(exception, (ProtocolError, ReadTimeout)):
metric = 'commcare.async_indicator.riak_error'
elif isinstance(exception, (ESError, ConnectionTimeout)):
# a database had an issue so log it and go on to the next document
metric = 'commcare.async_indicator.es_error'
elif isinstance(exception, (DatabaseError, InternalError)):
# a database had an issue so log it and go on to the next document
metric = 'commcare.async_indicator.psql_error'
else:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, exception)
if metric:
datadog_counter(metric,
1,
tags={
'config_id': config_id,
'doc_id': doc['_id']
})
def doc_ids_from_rows(rows):
formatted_rows = [{
column.column.database_column_name.decode('utf-8'): column.value
for column in row
} for row in rows]
return set(row['doc_id'] for row in formatted_rows)
# tracks processed/deleted configs to be removed from each indicator
configs_to_remove_by_indicator_id = defaultdict(list)
def _mark_config_to_remove(config_id, indicator_ids):
for _id in indicator_ids:
configs_to_remove_by_indicator_id[_id].append(config_id)
timer = TimingContext()
lock_keys = [
get_async_indicator_modify_lock_key(indicator_id)
for indicator_id in indicator_doc_ids
]
with CriticalSection(lock_keys):
all_indicators = AsyncIndicator.objects.filter(
doc_id__in=indicator_doc_ids)
if not all_indicators:
return
doc_store = get_document_store_for_doc_type(all_indicators[0].domain,
all_indicators[0].doc_type)
failed_indicators = set()
rows_to_save_by_adapter = defaultdict(list)
indicator_by_doc_id = {i.doc_id: i for i in all_indicators}
config_ids = set()
with timer:
for doc in doc_store.iter_documents(
list(indicator_by_doc_id.keys())):
indicator = indicator_by_doc_id[doc['_id']]
eval_context = EvaluationContext(doc)
for config_id in indicator.indicator_config_ids:
config_ids.add(config_id)
try:
config = _get_config_by_id(config_id)
except (ResourceNotFound,
StaticDataSourceConfigurationNotFoundError):
celery_task_logger.info(
"{} no longer exists, skipping".format(config_id))
# remove because the config no longer exists
_mark_config_to_remove(config_id, [indicator.pk])
continue
except ESError:
celery_task_logger.info(
"ES errored when trying to retrieve config")
failed_indicators.add(indicator)
continue
adapter = None
try:
adapter = get_indicator_adapter(config)
rows_to_save_by_adapter[adapter].extend(
adapter.get_all_values(doc, eval_context))
eval_context.reset_iteration()
except Exception as e:
failed_indicators.add(indicator)
handle_exception(e, config_id, doc, adapter)
for adapter, rows in six.iteritems(rows_to_save_by_adapter):
doc_ids = doc_ids_from_rows(rows)
indicators = [
indicator_by_doc_id[doc_id] for doc_id in doc_ids
]
try:
adapter.save_rows(rows)
except Exception as e:
failed_indicators.union(indicators)
message = six.text_type(e)
notify_exception(
None,
"Exception bulk saving async indicators:{}".format(
message))
else:
# remove because it's sucessfully processed
_mark_config_to_remove(config_id,
[i.pk for i in indicators])
# delete fully processed indicators
processed_indicators = set(all_indicators) - failed_indicators
AsyncIndicator.objects.filter(
pk__in=[i.pk for i in processed_indicators]).delete()
# update failure for failed indicators
with transaction.atomic():
for indicator in failed_indicators:
indicator.update_failure(
configs_to_remove_by_indicator_id.get(indicator.pk, []))
indicator.save()
datadog_counter('commcare.async_indicator.processed_success',
len(processed_indicators))
datadog_counter('commcare.async_indicator.processed_fail',
len(failed_indicators))
datadog_histogram('commcare.async_indicator.processing_time',
timer.duration / len(indicator_doc_ids),
tags=[
'config_ids:{}'.format(config_ids),
])
def _process_chunk_for_domain(self, domain, changes_chunk):
adapters = list(self.table_adapters_by_domain[domain])
changes_by_id = {change.id: change for change in changes_chunk}
to_delete_by_adapter = defaultdict(list)
rows_to_save_by_adapter = defaultdict(list)
async_configs_by_doc_id = defaultdict(list)
to_update = {change for change in changes_chunk if not change.deleted}
with self._datadog_timing('extract'):
retry_changes, docs = self.get_docs_for_changes(to_update, domain)
change_exceptions = []
with self._datadog_timing('single_batch_transform'):
for doc in docs:
change = changes_by_id[doc['_id']]
doc_subtype = change.metadata.document_subtype
eval_context = EvaluationContext(doc)
with self._datadog_timing('single_doc_transform'):
for adapter in adapters:
with self._datadog_timing('transform',
adapter.config._id):
if adapter.config.filter(doc, eval_context):
if adapter.run_asynchronous:
async_configs_by_doc_id[doc['_id']].append(
adapter.config._id)
else:
try:
rows_to_save_by_adapter[
adapter].extend(
adapter.get_all_values(
doc, eval_context))
except Exception as e:
change_exceptions.append((change, e))
eval_context.reset_iteration()
elif (doc_subtype is None
or doc_subtype in adapter.config.
get_case_type_or_xmlns_filter()):
# Delete if the subtype is unknown or
# if the subtype matches our filters, but the full filter no longer applies
to_delete_by_adapter[adapter].append(doc)
with self._datadog_timing('single_batch_delete'):
# bulk delete by adapter
to_delete = [{'_id': c.id} for c in changes_chunk if c.deleted]
for adapter in adapters:
delete_docs = to_delete_by_adapter[adapter] + to_delete
if not delete_docs:
continue
with self._datadog_timing('delete', adapter.config._id):
try:
adapter.bulk_delete(delete_docs)
except Exception:
delete_ids = [doc['_id'] for doc in delete_docs]
retry_changes.update(
[c for c in changes_chunk if c.id in delete_ids])
with self._datadog_timing('single_batch_load'):
# bulk update by adapter
for adapter, rows in rows_to_save_by_adapter.items():
with self._datadog_timing('load', adapter.config._id):
try:
adapter.save_rows(rows)
except Exception:
retry_changes.update(to_update)
if async_configs_by_doc_id:
with self._datadog_timing('async_config_load'):
doc_type_by_id = {
_id: changes_by_id[_id].metadata.document_type
for _id in async_configs_by_doc_id.keys()
}
AsyncIndicator.bulk_update_records(async_configs_by_doc_id,
domain, doc_type_by_id)
return retry_changes, change_exceptions
def _build_async_indicators(indicator_doc_ids):
def handle_exception(exception, config_id, doc, adapter):
metric = None
if isinstance(exception, (ProtocolError, ReadTimeout)):
metric = 'commcare.async_indicator.riak_error'
elif isinstance(exception, (ESError, ConnectionTimeout)):
# a database had an issue so log it and go on to the next document
metric = 'commcare.async_indicator.es_error'
elif isinstance(exception, (DatabaseError, InternalError)):
# a database had an issue so log it and go on to the next document
metric = 'commcare.async_indicator.psql_error'
else:
# getting the config could fail before the adapter is set
if adapter:
adapter.handle_exception(doc, exception)
if metric:
datadog_counter(metric, 1,
tags={'config_id': config_id, 'doc_id': doc['_id']})
def doc_ids_from_rows(rows):
formatted_rows = [
{column.column.database_column_name.decode('utf-8'): column.value for column in row}
for row in rows
]
return set(row['doc_id'] for row in formatted_rows)
# tracks processed/deleted configs to be removed from each indicator
configs_to_remove_by_indicator_id = defaultdict(list)
def _mark_config_to_remove(config_id, indicator_ids):
for _id in indicator_ids:
configs_to_remove_by_indicator_id[_id].append(config_id)
timer = TimingContext()
lock_keys = [
get_async_indicator_modify_lock_key(indicator_id)
for indicator_id in indicator_doc_ids
]
with CriticalSection(lock_keys):
all_indicators = AsyncIndicator.objects.filter(
doc_id__in=indicator_doc_ids
)
if not all_indicators:
return
doc_store = get_document_store_for_doc_type(
all_indicators[0].domain, all_indicators[0].doc_type,
load_source="build_async_indicators",
)
failed_indicators = set()
rows_to_save_by_adapter = defaultdict(list)
indicator_by_doc_id = {i.doc_id: i for i in all_indicators}
config_ids = set()
with timer:
for doc in doc_store.iter_documents(list(indicator_by_doc_id.keys())):
indicator = indicator_by_doc_id[doc['_id']]
eval_context = EvaluationContext(doc)
for config_id in indicator.indicator_config_ids:
config_ids.add(config_id)
try:
config = _get_config_by_id(config_id)
except (ResourceNotFound, StaticDataSourceConfigurationNotFoundError):
celery_task_logger.info("{} no longer exists, skipping".format(config_id))
# remove because the config no longer exists
_mark_config_to_remove(config_id, [indicator.pk])
continue
except ESError:
celery_task_logger.info("ES errored when trying to retrieve config")
failed_indicators.add(indicator)
continue
adapter = None
try:
adapter = get_indicator_adapter(config, load_source='build_async_indicators')
rows_to_save_by_adapter[adapter].extend(adapter.get_all_values(doc, eval_context))
eval_context.reset_iteration()
except Exception as e:
failed_indicators.add(indicator)
handle_exception(e, config_id, doc, adapter)
for adapter, rows in six.iteritems(rows_to_save_by_adapter):
doc_ids = doc_ids_from_rows(rows)
indicators = [indicator_by_doc_id[doc_id] for doc_id in doc_ids]
try:
adapter.save_rows(rows)
except Exception as e:
failed_indicators.union(indicators)
message = six.text_type(e)
notify_exception(None,
"Exception bulk saving async indicators:{}".format(message))
else:
# remove because it's sucessfully processed
_mark_config_to_remove(
config_id,
[i.pk for i in indicators]
)
# delete fully processed indicators
processed_indicators = set(all_indicators) - failed_indicators
AsyncIndicator.objects.filter(pk__in=[i.pk for i in processed_indicators]).delete()
# update failure for failed indicators
with transaction.atomic():
for indicator in failed_indicators:
indicator.update_failure(
configs_to_remove_by_indicator_id.get(indicator.pk, [])
)
indicator.save()
datadog_counter('commcare.async_indicator.processed_success', len(processed_indicators))
datadog_counter('commcare.async_indicator.processed_fail', len(failed_indicators))
datadog_histogram(
'commcare.async_indicator.processing_time', timer.duration / len(indicator_doc_ids),
tags=[
'config_ids:{}'.format(config_ids),
]
)
