def process_changes_chunk(self, changes_chunk):
with self._datadog_timing('bulk_extract'):
bad_changes, docs = bulk_fetch_changes_docs(changes_chunk)
with self._datadog_timing('bulk_transform'):
changes_to_process = {
change.id: change
for change in changes_chunk
if change.document and not self.doc_filter_fn(change.document)
}
retry_changes = list(bad_changes)
error_collector = ErrorCollector()
es_actions = build_bulk_payload(self.index_info,
list(changes_to_process.values()),
self.doc_transform_fn,
error_collector)
error_changes = error_collector.errors
try:
with self._datadog_timing('bulk_load'):
_, errors = self.es_interface.bulk_ops(
es_actions, raise_on_error=False, raise_on_exception=False)
except Exception as e:
pillow_logging.exception("[%s] ES bulk load error")
error_changes.extend([(change, e)
for change in changes_to_process.values()])
else:
for change_id, error_msg in get_errors_with_ids(errors):
error_changes.append((changes_to_process[change_id],
BulkDocException(error_msg)))
return retry_changes, error_changes
def test_get_errors_with_ids(self):
errors = get_errors_with_ids([{
'index': {
'_id': 1,
'status': 500,
'error': 'e1'
}
}, {
'index': {
'_id': 2,
'status': 500,
'error': 'e2'
}
}])
self.assertEqual([(1, 'e1'), (2, 'e2')], errors)