def test_app_pillow_kafka(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app_name = 'app-{}'.format(uuid.uuid4().hex)
app = self._create_app(app_name)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
app_doc = results.hits[0]
self.assertEqual(self.domain, app_doc['domain'])
self.assertEqual(app['_id'], app_doc['_id'])
self.assertEqual(app_name, app_doc['name'])
def test_app_pillow_kafka(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app_name = 'app-{}'.format(uuid.uuid4().hex)
app = self._create_app(app_name)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
app_doc = results.hits[0]
self.assertEqual(self.domain, app_doc['domain'])
self.assertEqual(app['_id'], app_doc['_id'])
self.assertEqual(app_name, app_doc['name'])
def test_hard_delete_app(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app = self._create_app('test_hard_deleted_app', cleanup=False)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
couch_seq = get_current_seq(Application.get_db())
kafka_seq = get_topic_offset(topics.APP)
app.delete()
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka. Would raise StopIteration otherwise
next(consumer)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm deletion made it to elasticserach
results = AppES().run()
self.assertEqual(0, results.total)
def refresh_elasticsearch(self, kafka_seq, couch_seq):
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
def refresh_elasticsearch(self, kafka_seq, couch_seq):
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
