def test_app_pillow_kafka(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app_name = 'app-{}'.format(uuid.uuid4().hex)
app = self._create_app(app_name)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
app_doc = results.hits[0]
self.assertEqual(self.domain, app_doc['domain'])
self.assertEqual(app['_id'], app_doc['_id'])
self.assertEqual(app_name, app_doc['name'])
def test_form_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
form_id = self._save_doc_to_db('indicator_form.json', XFormInstance)
form_instance = XFormInstance.get(form_id)
# Form Label Indicator
form_label = FormLabelIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='create_form',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
form_label.save()
# Form Alias
form_alias = FormDataAliasIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
form_alias.save()
self.form_pillow.process_changes(since=since, forever=False)
indicator_form = IndicatorXForm.get(form_id)
self.assertNotEqual(indicator_form.get_db().dbname,
form_instance.get_db().dbname)
self.assertNotEqual(indicator_form.computed_, {})
def test_case_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
self._save_doc_to_db('indicator_form.json', XFormInstance)
case_id = self._save_doc_to_db('indicator_case.json', CommCareCase)
case_instance = CommCareCase.get(case_id)
# FormDataInCaseIndicatorDef (For those forgotten properties)
forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
case_type='song_tag',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
forgotten_property.save()
self.case_pillow.process_changes(since=since, forever=False)
indicator_case = IndicatorCase.get(case_id)
self.assertEqual(indicator_case.get_id, case_instance.get_id)
self.assertNotEqual(indicator_case.get_db().dbname,
case_instance.get_db().dbname)
self.assertNotEqual(indicator_case.computed_, {})
def test_case_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
self._save_doc_to_db('indicator_form.json', XFormInstance)
case_id = self._save_doc_to_db('indicator_case.json', CommCareCase)
case_instance = CommCareCase.get(case_id)
# FormDataInCaseIndicatorDef (For those forgotten properties)
forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
case_type='song_tag',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
forgotten_property.save()
self.case_pillow.process_changes(since=since, forever=False)
indicator_case = IndicatorCase.get(case_id)
self.assertEqual(indicator_case.get_id, case_instance.get_id)
self.assertNotEqual(
indicator_case.get_db().dbname, case_instance.get_db().dbname
)
self.assertNotEqual(indicator_case.computed_, {})
def test_prune_autogenerated_builds(self):
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
# Build #1, manually generated
app = self._create_app('test-prune-app')
build1 = app.make_build()
build1.save()
self.assertFalse(build1.is_auto_generated)
# Build #2, auto-generated
app.save()
autogenerate_build(app, 'username')
# Build #3, manually generated
app.save()
build3 = app.make_build()
build3.save()
# All 3 builds should show up in ES
self.refresh_elasticsearch(kafka_seq, couch_seq)
build_ids_in_es = AppES().domain(self.domain).is_build().values_list(
'_id', flat=True)
self.assertEqual(len(build_ids_in_es), 3)
# prune, which should delete the autogenerated build
prune_auto_generated_builds(self.domain, app.id)
# Build2 should no longer be in ES
self.refresh_elasticsearch(kafka_seq, couch_seq)
build_ids_in_es = AppES().domain(self.domain).is_build().values_list(
'_id', flat=True)
self.assertItemsEqual(build_ids_in_es, [build1._id, build3._id])
def test_form_pillow_indicators(self):
since = get_current_seq(XFormInstance.get_db())
form_id = self._save_doc_to_db('indicator_form.json', XFormInstance)
form_instance = XFormInstance.get(form_id)
# Form Label Indicator
form_label = FormLabelIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='create_form',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
form_label.save()
# Form Alias
form_alias = FormDataAliasIndicatorDefinition.increment_or_create_unique(
INDICATOR_TEST_NAMESPACE,
INDICATOR_TEST_DOMAIN,
slug='club_name',
question_id='location.club',
xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns',
)
form_alias.save()
self.form_pillow.process_changes(since=since, forever=False)
indicator_form = IndicatorXForm.get(form_id)
self.assertNotEqual(
indicator_form.get_db().dbname, form_instance.get_db().dbname
)
self.assertNotEqual(indicator_form.computed_, {})
def test_prune_autogenerated_builds(self):
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
# Build #1, manually generated
app = self._create_app('test-prune-app')
build1 = app.make_build()
build1.save()
self.assertFalse(build1.is_auto_generated)
# Build #2, auto-generated
app.save()
build2 = make_async_build(app, 'someone')
# Build #3, manually generated
app.save()
build3 = app.make_build()
build3.save()
# All 3 builds should show up in ES
self.refresh_elasticsearch(kafka_seq, couch_seq)
build_ids_in_es = AppES().domain(self.domain).is_build().values_list('_id', flat=True)
self.assertItemsEqual(build_ids_in_es, [build1._id, build2._id, build3._id])
# prune, which should delete the autogenerated build
prune_auto_generated_builds(self.domain, app.id)
# Build2 should no longer be in ES
self.refresh_elasticsearch(kafka_seq, couch_seq)
build_ids_in_es = AppES().domain(self.domain).is_build().values_list('_id', flat=True)
self.assertItemsEqual(build_ids_in_es, [build1._id, build3._id])
def test_app_pillow_kafka(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app_name = 'app-{}'.format(uuid.uuid4().hex)
app = self._create_app(app_name)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
app_doc = results.hits[0]
self.assertEqual(self.domain, app_doc['domain'])
self.assertEqual(app['_id'], app_doc['_id'])
self.assertEqual(app_name, app_doc['name'])
def test_hard_delete_app(self):
consumer = get_test_kafka_consumer(topics.APP)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.APP)
couch_seq = get_current_seq(Application.get_db())
app = self._create_app('test_hard_deleted_app', cleanup=False)
app_db_pillow = get_application_db_kafka_pillow('test_app_db_pillow')
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka
message = next(consumer)
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(app._id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm change made it to elasticserach
results = AppES().run()
self.assertEqual(1, results.total)
couch_seq = get_current_seq(Application.get_db())
kafka_seq = get_topic_offset(topics.APP)
app.delete()
app_db_pillow.process_changes(couch_seq, forever=False)
# confirm change made it to kafka. Would raise StopIteration otherwise
next(consumer)
# send to elasticsearch
app_pillow = get_app_to_elasticsearch_pillow()
app_pillow.process_changes(since=kafka_seq, forever=False)
self.es.indices.refresh(APP_INDEX_INFO.index)
# confirm deletion made it to elasticserach
results = AppES().run()
self.assertEqual(0, results.total)
def setUp(self):
super(ChangeFeedDbTest, self).setUp()
self.couch_db = XFormInstance.get_db()
self.update_seq = get_current_seq(self.couch_db)
def setUp(self):
super(ChangeFeedDbTest, self).setUp()
self.couch_db = XFormInstance.get_db()
self.update_seq = get_current_seq(self.couch_db)
def setUp(self):
super(ChangeFeedDbTest, self).setUp()
self.couch_db = couch_config.get_db(None)
self.update_seq = get_current_seq(self.couch_db)
