Example #1
0
    def test_unknown_user_pillow(self):
        FormProcessorTestUtils.delete_all_xforms()
        user_id = 'test-unknown-user'
        metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user')
        form = get_form_ready_to_save(metadata)
        FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form])

        # send to kafka
        topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
        since = self._get_kafka_seq()
        producer.send_change(topic, _form_to_change_meta(form))

        # send to elasticsearch
        pillow = get_unknown_users_pillow()
        pillow.process_changes(since=since, forever=False)
        self.elasticsearch.indices.refresh(self.index_info.index)

        # the default query doesn't include unknown users so should have no results
        self.assertEqual(0, UserES().run().total)
        # clear the default filters which hide unknown users
        user_es = UserES().remove_default_filters()
        results = user_es.run()
        self.assertEqual(1, results.total)
        user_doc = results.hits[0]
        self.assertEqual(TEST_DOMAIN, user_doc['domain'])
        self.assertEqual(user_id, user_doc['_id'])
        self.assertEqual('UnknownUser', user_doc['doc_type'])
    def test_xform_pillow_sql(self):
        consumer = get_test_kafka_consumer(topics.FORM_SQL)

        # have to get the seq id before the change is processed
        kafka_seq = consumer.offsets()['fetch'][(topics.FORM_SQL, 0)]

        metadata = TestFormMetadata(domain=self.domain)
        form = get_form_ready_to_save(metadata, is_db_test=True)
        form_processor = FormProcessorInterface(domain=self.domain)
        form_processor.save_processed_models([form])

        # confirm change made it to kafka
        message = consumer.next()
        change_meta = change_meta_from_kafka_message(message.value)
        self.assertEqual(form.form_id, change_meta.document_id)
        self.assertEqual(self.domain, change_meta.domain)

        # send to elasticsearch
        sql_pillow = get_sql_xform_to_elasticsearch_pillow()
        sql_pillow.process_changes(since=kafka_seq, forever=False)
        self.elasticsearch.indices.refresh(self.pillow.es_index)

        # confirm change made it to elasticserach
        results = FormES().run()
        self.assertEqual(1, results.total)
        form_doc = results.hits[0]
        self.assertEqual(self.domain, form_doc['domain'])
        self.assertEqual(metadata.xmlns, form_doc['xmlns'])
        self.assertEqual('XFormInstance', form_doc['doc_type'])
Example #3
0
 def _create_form_and_sync_to_es(self):
     with self.process_form_changes:
         form = get_form_ready_to_save(self.metadata, is_db_test=True)
         form_processor = FormProcessorInterface(domain=self.domain)
         form_processor.save_processed_models([form])
     self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index)
     return form, self.metadata
    def test_unknown_user_pillow(self):
        FormProcessorTestUtils.delete_all_xforms()
        user_id = 'test-unknown-user'
        metadata = TestFormMetadata(domain=TEST_DOMAIN,
                                    user_id='test-unknown-user')
        form = get_form_ready_to_save(metadata)
        FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models(
            [form])

        # send to kafka
        topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
        since = self._get_kafka_seq()
        producer.send_change(topic, _form_to_change_meta(form))

        # send to elasticsearch
        pillow = get_xform_pillow()
        pillow.process_changes(since=since, forever=False)
        self.elasticsearch.indices.refresh(self.index_info.index)

        # the default query doesn't include unknown users so should have no results
        self.assertEqual(0, UserES().run().total)
        # clear the default filters which hide unknown users
        user_es = UserES().remove_default_filters()
        results = user_es.run()
        self.assertEqual(1, results.total)
        user_doc = results.hits[0]
        self.assertEqual(TEST_DOMAIN, user_doc['domain'])
        self.assertEqual(user_id, user_doc['_id'])
        self.assertEqual('UnknownUser', user_doc['doc_type'])
Example #5
0
 def _create_form_and_sync_to_es(self, domain):
     with process_kafka_changes('ReportXFormToElasticsearchPillow'):
         with process_couch_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=domain)
             form = get_form_ready_to_save(metadata)
             FormProcessorInterface(domain=domain).save_processed_models([form])
     self.elasticsearch.indices.refresh(REPORT_XFORM_INDEX_INFO.index)
     return form, metadata
Example #6
0
 def _create_form_and_sync_to_es(self, domain):
     with process_pillow_changes('ReportXFormToElasticsearchPillow'):
         with process_pillow_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=domain)
             form = get_form_ready_to_save(metadata)
             FormProcessorInterface(domain=domain).save_processed_models([form])
     self.elasticsearch.indices.refresh(REPORT_XFORM_INDEX_INFO.index)
     return form, metadata
Example #7
0
 def create_form_and_sync_to_es(received_on):
     with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
         with process_pillow_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id,
                                         xmlns=cls.xmlns, received_on=received_on)
             form = get_form_ready_to_save(metadata, is_db_test=True)
             form_processor = FormProcessorInterface(domain=cls.domain)
             form_processor.save_processed_models([form])
     return form
Example #8
0
 def _create_form_and_sync_to_es(self):
     with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
         with process_pillow_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=self.domain)
             form = get_form_ready_to_save(metadata, is_db_test=True)
             form_processor = FormProcessorInterface(domain=self.domain)
             form_processor.save_processed_models([form])
     self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index)
     return form, metadata
Example #9
0
 def create_form_and_sync_to_es(received_on):
     with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
         with process_pillow_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id,
                                         xmlns=cls.xmlns, received_on=received_on)
             form = get_form_ready_to_save(metadata, is_db_test=True)
             form_processor = FormProcessorInterface(domain=cls.domain)
             form_processor.save_processed_models([form])
     return form
Example #10
0
 def create_form_and_sync_to_es(received_on):
     with process_kafka_changes('XFormToElasticsearchPillow'):
         with process_couch_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id,
                                         xmlns=cls.xmlns, received_on=received_on)
             form = get_form_ready_to_save(metadata, is_db_test=True)
             form_processor = FormProcessorInterface(domain=cls.domain)
             form_processor.save_processed_models([form])
     return form
Example #11
0
 def create_form_and_sync_to_es(received_on):
     with process_kafka_changes('XFormToElasticsearchPillow'):
         with process_couch_changes('DefaultChangeFeedPillow'):
             metadata = TestFormMetadata(domain=cls.domain,
                                         app_id=cls.app_id,
                                         xmlns=cls.xmlns,
                                         received_on=received_on)
             form = get_form_ready_to_save(metadata, is_db_test=True)
             form_processor = FormProcessorInterface(domain=cls.domain)
             form_processor.save_processed_models([form])
     return form
Example #12
0
 def test_xform_reindexers(self):
     FormProcessorTestUtils.delete_all_xforms()
     metadata = TestFormMetadata(domain=self.domain)
     form = get_form_ready_to_save(metadata)
     FormProcessorInterface(domain=self.domain).save_processed_models([form])
     call_command('ptop_fast_reindex_xforms', noinput=True, bulk=True)
     results = FormES().run()
     self.assertEqual(1, results.total)
     form_doc = results.hits[0]
     self.assertEqual(self.domain, form_doc['domain'])
     self.assertEqual(metadata.xmlns, form_doc['xmlns'])
     self.assertEqual('XFormInstance', form_doc['doc_type'])
     form.delete()
 def test_xform_pillow_couch(self):
     metadata = TestFormMetadata(domain=self.domain)
     form = get_form_ready_to_save(metadata)
     FormProcessorInterface(domain=self.domain).save_processed_models([form])
     self.pillow.process_changes(since=0, forever=False)
     self.elasticsearch.indices.refresh(self.pillow.es_index)
     results = FormES().run()
     self.assertEqual(1, results.total)
     form_doc = results.hits[0]
     self.assertEqual(self.domain, form_doc['domain'])
     self.assertEqual(metadata.xmlns, form_doc['xmlns'])
     self.assertEqual('XFormInstance', form_doc['doc_type'])
     form.delete()
Example #14
0
    def _create_form(self, user_id=None, received_on=None):
        """
        Create the models directly so that these tests aren't dependent on any
        other apps.
        :return: XFormInstanceSQL
        """
        user_id = user_id or 'mr_wednesday'
        received_on = received_on or datetime.utcnow()

        metadata = TestFormMetadata(domain=self.DOMAIN,
                                    received_on=received_on,
                                    user_id=user_id)
        form = get_form_ready_to_save(metadata)

        return form
    def _create_form(self, user_id=None, received_on=None):
        """
        Create the models directly so that these tests aren't dependent on any
        other apps.
        :return: XFormInstanceSQL
        """
        user_id = user_id or 'mr_wednesday'
        received_on = received_on or datetime.utcnow()

        metadata = TestFormMetadata(
            domain=self.DOMAIN,
            received_on=received_on,
            user_id=user_id
        )
        form = get_form_ready_to_save(metadata)

        return form
Example #16
0
 def test_unknown_user_reindexer(self):
     FormProcessorTestUtils.delete_all_xforms()
     user_id = 'test-unknown-user'
     metadata = TestFormMetadata(domain=self.domain, user_id='test-unknown-user')
     form = get_form_ready_to_save(metadata)
     FormProcessorInterface(domain=self.domain).save_processed_models([form])
     ensure_index_deleted(USER_INDEX)
     call_command('ptop_fast_reindex_unknownusers', noinput=True, bulk=True)
     # the default query doesn't include unknown users so should have no results
     self.assertEqual(0, UserES().run().total)
     user_es = UserES()
     # hack: clear the default filters which hide unknown users
     # todo: find a better way to do this.
     user_es._default_filters = ESQuery.default_filters
     results = user_es.run()
     self.assertEqual(1, results.total)
     user_doc = results.hits[0]
     self.assertEqual(self.domain, user_doc['domain'])
     self.assertEqual(user_id, user_doc['_id'])
     self.assertEqual('UnknownUser', user_doc['doc_type'])
     form.delete()
     delete_es_index(USER_INDEX)
Example #17
0
def get_form(form_id):
    metadata = TestFormMetadata(domain="test", user_id=USER_ID)
    return get_form_ready_to_save(metadata, form_id=form_id)
Example #18
0
 def _create_form(self, domain):
     metadata = TestFormMetadata(domain=domain)
     form = get_form_ready_to_save(metadata)
     FormProcessorInterface(domain=domain).save_processed_models([form])
     return form
Example #19
0
 def _create_form(self, domain):
     metadata = TestFormMetadata(domain=domain)
     form = get_form_ready_to_save(metadata)
     FormProcessorInterface(domain=domain).save_processed_models([form])
     return form
Example #20
0
def _create_and_save_a_form():
    metadata = TestFormMetadata(domain=DOMAIN)
    form = get_form_ready_to_save(metadata)
    FormProcessorInterface(domain=DOMAIN).save_processed_models([form])
    return form
Example #21
0
 def _create_form(self):
     form = get_form_ready_to_save(self.metadata, is_db_test=True)
     form_processor = FormProcessorInterface(domain=self.domain)
     form_processor.save_processed_models([form])
     return form