def process_bulk_docs(self, docs, progress_logger): if len(docs) == 0: return True pillow_logging.info("Processing batch of %s docs", len(docs)) changes = [ self._doc_to_change(doc) for doc in docs if self.process_deletes or not is_deletion(doc.get('doc_type')) ] error_collector = ErrorCollector() bulk_changes = build_bulk_payload(self.index_info, changes, self.doc_transform, error_collector) for change, exception in error_collector.errors: pillow_logging.error("Error procesing doc %s: %s (%s)", change.id, type(exception), exception) es_interface = ElasticsearchInterface(self.es) try: es_interface.bulk_ops(bulk_changes) except (ESBulkIndexError, ES2BulkIndexError, ES7BulkIndexError) as e: pillow_logging.error("Bulk index errors\n%s", e.errors) except Exception: pillow_logging.exception("\tException sending payload to ES") return False return True
def _doc_to_change(doc): return Change( id=doc['_id'], sequence_id=None, document=doc, deleted=is_deletion(doc.get('doc_type')) )