def process_change(self, pillow_instance, change, do_set_checkpoint):
# todo: if deletion - delete
# prepare doc for es
doc_ready_to_save = self.doc_transform_fn(change.get_document())
# send it across
send_to_elasticsearch(
index=self.index_meta.index,
doc_type=self.index_meta.type,
doc_id=change.id,
es_getter=self.es_getter,
name=pillow_instance.get_name(),
data=doc_ready_to_save,
update=self.elasticsearch.exists(self.index_meta.index, change.id, self.index_meta.type),
)
def process_change(self, pillow_instance, change, do_set_checkpoint):
# todo: if deletion - delete
# prepare doc for es
doc = change.get_document()
if doc is None:
pillow_logging.warning("Unable to get document from change: {}".format(change))
return
doc_ready_to_save = self.doc_transform_fn(doc)
# send it across
send_to_elasticsearch(
index=self.index_meta.index,
doc_type=self.index_meta.type,
doc_id=change.id,
es_getter=self.es_getter,
name=pillow_instance.get_name(),
data=doc_ready_to_save,
update=self.elasticsearch.exists(self.index_meta.index, self.index_meta.type, change.id),
)
def _send_to_es_and_check(self, doc, update=False, delete=False, esgetter=None):
send_to_elasticsearch(
index=self.index,
doc_type=self.pillow.es_type,
doc_id=doc['_id'],
es_getter=esgetter or self.pillow.get_es_new,
name='test',
data=doc,
update=update,
delete=delete,
except_on_failure=True,
retries=1
)
if not delete:
self.assertEqual(1, get_doc_count(self.es, self.index))
es_doc = self.es.get_source(self.index, doc['_id'])
for prop in doc:
self.assertEqual(doc[prop], es_doc[prop])
else:
self.assertEqual(0, get_doc_count(self.es, self.index))
def process_change(self, pillow_instance, change):
if change.deleted and change.id:
self._delete_doc_if_exists(change.id)
return
# prepare doc for es
doc = change.get_document()
if doc is None:
pillow_logging.warning("Unable to get document from change: {}".format(change))
return
doc_ready_to_save = self.doc_transform_fn(doc)
# send it across
send_to_elasticsearch(
index=self.index_info.index,
doc_type=self.index_info.type,
doc_id=change.id,
es_getter=self.es_getter,
name=pillow_instance.get_name(),
data=doc_ready_to_save,
update=self._doc_exists(change.id),
)
