def test_xform_pillow_sql(self): consumer = get_test_kafka_consumer(topics.FORM_SQL) # have to get the seq id before the change is processed kafka_seq = consumer.offsets()['fetch'][(topics.FORM_SQL, 0)] metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(form.form_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch sql_pillow = get_sql_xform_to_elasticsearch_pillow() sql_pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(self.pillow.es_index) # confirm change made it to elasticserach results = FormES().run() self.assertEqual(1, results.total) form_doc = results.hits[0] self.assertEqual(self.domain, form_doc['domain']) self.assertEqual(metadata.xmlns, form_doc['xmlns']) self.assertEqual('XFormInstance', form_doc['doc_type'])
def test_case_pillow_sql(self): consumer = get_test_kafka_consumer(topics.CASE_SQL) # have to get the seq id before the change is processed kafka_seq = consumer.offsets()['fetch'][(topics.CASE_SQL, 0)] # make a case case_id = uuid.uuid4().hex case_name = 'case-name-{}'.format(uuid.uuid4().hex) case = self._make_a_case(case_id, case_name) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch sql_pillow = get_sql_case_to_elasticsearch_pillow() sql_pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(self.pillow.es_index) # confirm change made it to elasticserach results = CaseES().run() self.assertEqual(1, results.total) case_doc = results.hits[0] self.assertEqual(self.domain, case_doc['domain']) self.assertEqual(case_id, case_doc['_id']) self.assertEqual(case_name, case_doc['name'])
def test_sql_sms_pillow(self, mock_do_publish): mock_do_publish.return_value = True consumer = get_test_kafka_consumer(topics.SMS) # get the seq id before the change is published kafka_seq = consumer.offsets()['fetch'][(topics.SMS, 0)] # create an sms self._create_sms() sms_json = self._to_json(self.sms_dict, self.sms) # test serialization self.assertEqual(self.sms.to_json(), sms_json) # publish the change and confirm it gets to kafka self.sms.publish_change() message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(self.sms.couch_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch sms_pillow = get_sql_sms_pillow('SqlSMSPillow') sms_pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(ES_SMS_INDEX) # confirm change made it to elasticserach results = SMSES().run() self.assertEqual(1, results.total) sms_doc = results.hits[0] self.assertEqual(sms_doc, sms_json)