def process_change(self, pillow_instance, change, do_set_checkpoint): # todo: if deletion - delete # prepare doc for es doc_ready_to_save = self.doc_transform_fn(change.get_document()) # send it across send_to_elasticsearch( index=self.index_meta.index, doc_type=self.index_meta.type, doc_id=change.id, es_getter=self.es_getter, name=pillow_instance.get_name(), data=doc_ready_to_save, update=self.elasticsearch.exists(self.index_meta.index, change.id, self.index_meta.type), )
def process_change(self, pillow_instance, change, do_set_checkpoint): # todo: if deletion - delete # prepare doc for es doc = change.get_document() if doc is None: pillow_logging.warning("Unable to get document from change: {}".format(change)) return doc_ready_to_save = self.doc_transform_fn(doc) # send it across send_to_elasticsearch( index=self.index_meta.index, doc_type=self.index_meta.type, doc_id=change.id, es_getter=self.es_getter, name=pillow_instance.get_name(), data=doc_ready_to_save, update=self.elasticsearch.exists(self.index_meta.index, self.index_meta.type, change.id), )
def _send_to_es_and_check(self, doc, update=False, delete=False, esgetter=None): send_to_elasticsearch( index=self.index, doc_type=self.pillow.es_type, doc_id=doc['_id'], es_getter=esgetter or self.pillow.get_es_new, name='test', data=doc, update=update, delete=delete, except_on_failure=True, retries=1 ) if not delete: self.assertEqual(1, get_doc_count(self.es, self.index)) es_doc = self.es.get_source(self.index, doc['_id']) for prop in doc: self.assertEqual(doc[prop], es_doc[prop]) else: self.assertEqual(0, get_doc_count(self.es, self.index))
def process_change(self, pillow_instance, change): if change.deleted and change.id: self._delete_doc_if_exists(change.id) return # prepare doc for es doc = change.get_document() if doc is None: pillow_logging.warning("Unable to get document from change: {}".format(change)) return doc_ready_to_save = self.doc_transform_fn(doc) # send it across send_to_elasticsearch( index=self.index_info.index, doc_type=self.index_info.type, doc_id=change.id, es_getter=self.es_getter, name=pillow_instance.get_name(), data=doc_ready_to_save, update=self._doc_exists(change.id), )