Example #1
0
    def test_pillow_save_to_one_database_at_a_time(self):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.ds_1])

        sample_doc, _ = get_sample_doc_and_indicators()
        pillow.process_change(doc_to_change(sample_doc))

        self.assertEqual(1, self.ds1_adapter.get_query_object().count())
        self.assertEqual(0, self.ds2_adapter.get_query_object().count())

        # save to the other
        pillow.bootstrap(configs=[self.ds_2])
        orig_id = sample_doc['_id']
        sample_doc['_id'] = uuid.uuid4().hex
        pillow.process_change(doc_to_change(sample_doc))
        self.assertEqual(1, self.ds1_adapter.get_query_object().count())
        self.assertEqual(1, self.ds2_adapter.get_query_object().count())
        self.assertEqual(
            1,
            self.ds1_adapter.get_query_object().filter_by(
                doc_id=orig_id).count())
        self.assertEqual(
            1,
            self.ds2_adapter.get_query_object().filter_by(
                doc_id=sample_doc['_id']).count())
Example #2
0
    def test_process_filter_no_longer_pass(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)

        self.pillow.process_change(doc_to_change(sample_doc))
        self._check_sample_doc_state(expected_indicators)

        sample_doc['type'] = 'wrong_type'

        self.pillow.process_change(doc_to_change(sample_doc))

        self.assertEqual(0, self.adapter.get_query_object().count())
Example #3
0
    def test_process_filter_no_longer_pass(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)

        self.pillow.process_change(doc_to_change(sample_doc))
        self._check_sample_doc_state(expected_indicators)

        sample_doc['type'] = 'wrong_type'

        self.pillow.process_change(doc_to_change(sample_doc))

        self.assertEqual(0, self.adapter.get_query_object().count())
Example #4
0
 def test_not_relevant_to_domain(self, datetime_mock):
     datetime_mock.utcnow.return_value = self.fake_time_now
     sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
     sample_doc['domain'] = 'not-this-domain'
     self.pillow.process_change(doc_to_change(sample_doc))
     self.adapter.refresh_table()
     self.assertEqual(0, self.adapter.get_query_object().count())
Example #5
0
    def test_xform_pillow_couch(self):
        form = self._make_form()
        kafka_seq = self._get_kafka_seq()
        producer.send_change(topics.FORM, doc_to_change(form.to_json()).metadata)
        self.assertFalse(self.app.has_submissions)

        self.pillow.process_changes(since=kafka_seq, forever=False)
        self.assertTrue(Application.get(self.app._id).has_submissions)
Example #6
0
    def test_xform_pillow_couch(self):
        form = self._make_form()
        kafka_seq = self._get_kafka_seq()
        producer.send_change(topics.FORM, doc_to_change(form.to_json()).metadata)
        self.assertFalse(self.app.has_submissions)

        self.pillow.process_changes(since=kafka_seq, forever=False)
        self.assertTrue(Application.get(self.app._id).has_submissions)
Example #7
0
    def test_pillow_save_to_one_database_at_a_time(self):
        pillow = get_case_pillow(ucr_configs=[self.ds_1])

        sample_doc, _ = get_sample_doc_and_indicators()
        pillow.process_change(doc_to_change(sample_doc))

        self.assertEqual(1, self.ds1_adapter.get_query_object().count())
        self.assertEqual(0, self.ds2_adapter.get_query_object().count())

        # save to the other
        pillow = get_case_pillow(ucr_configs=[self.ds_2])
        orig_id = sample_doc['_id']
        sample_doc['_id'] = uuid.uuid4().hex
        pillow.process_change(doc_to_change(sample_doc))
        self.assertEqual(1, self.ds1_adapter.get_query_object().count())
        self.assertEqual(1, self.ds2_adapter.get_query_object().count())
        self.assertEqual(1, self.ds1_adapter.get_query_object().filter_by(doc_id=orig_id).count())
        self.assertEqual(1, self.ds2_adapter.get_query_object().filter_by(doc_id=sample_doc['_id']).count())
Example #8
0
 def test_pillow_save_to_multiple_databases(self):
     self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
     pillow = get_case_pillow(ucr_configs=[self.ds_1, self.ds_2])
     self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
     sample_doc, _ = get_sample_doc_and_indicators()
     pillow.process_change(doc_to_change(sample_doc))
     self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
     self.assertEqual(1, self.ds1_adapter.get_query_object().count())
     self.assertEqual(1, self.ds2_adapter.get_query_object().count())
Example #9
0
    def test_check_if_doc_exist(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)

        self.assertFalse(self.adapter.doc_exists(sample_doc))

        self.pillow.process_change(doc_to_change(sample_doc))

        self.assertIs(self.adapter.doc_exists(sample_doc), True)
Example #10
0
 def test_pillow_save_to_multiple_databases(self):
     self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
     pillow = get_case_pillow(ucr_configs=[self.ds_1, self.ds_2])
     self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
     sample_doc, _ = get_sample_doc_and_indicators()
     pillow.process_change(doc_to_change(sample_doc))
     self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
     self.assertEqual(1, self.ds1_adapter.get_query_object().count())
     self.assertEqual(1, self.ds2_adapter.get_query_object().count())
Example #11
0
    def test_check_if_doc_exist(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)

        self.assertFalse(self.adapter.doc_exists(sample_doc))

        self.pillow.process_change(doc_to_change(sample_doc))

        self.assertIs(self.adapter.doc_exists(sample_doc), True)
Example #12
0
 def test_bad_integer_datatype(self):
     bad_ints = ['a', '', None]
     for bad_value in bad_ints:
         self.pillow.process_change(doc_to_change({
             '_id': uuid.uuid4().hex,
             'doc_type': 'CommCareCase',
             'domain': 'user-reports',
             'type': 'ticket',
             'priority': bad_value
         }))
     # make sure we saved rows to the table for everything
     self.assertEqual(len(bad_ints), self.adapter.get_query_object().count())
Example #13
0
 def test_bad_integer_datatype(self):
     bad_ints = ['a', '', None]
     for bad_value in bad_ints:
         self.pillow.process_change(doc_to_change({
             '_id': uuid.uuid4().hex,
             'doc_type': 'CommCareCase',
             'domain': 'user-reports',
             'type': 'ticket',
             'priority': bad_value
         }))
     # make sure we saved rows to the table for everything
     self.assertEqual(len(bad_ints), self.adapter.get_query_object().count())
    def _add_rows(self, rows):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.data_source])

        def _get_case(row):
            return {
                '_id': uuid.uuid4().hex,
                'domain': self.domain,
                'doc_type': 'CommCareCase',
                'type': 'city',
                'name': row.name,
                'number': row.number,
            }
        for row in rows:
            pillow.process_change(doc_to_change(_get_case(row)))
Example #15
0
    def _add_rows(self, rows):
        pillow = get_case_pillow(ucr_configs=[self.data_source])

        def _get_case(row):
            return {
                '_id': uuid.uuid4().hex,
                'domain': self.domain,
                'doc_type': 'CommCareCase',
                'type': 'city',
                'name': row.name,
                'number': row.number,
                'just_for_sorting': row.sort_key,
            }
        for row in rows:
            pillow.process_change(doc_to_change(_get_case(row)))
    def _add_rows(self, rows):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.data_source])

        def _get_case(row):
            return {
                '_id': uuid.uuid4().hex,
                'domain': self.domain,
                'doc_type': 'CommCareCase',
                'type': 'city',
                'name': row.name,
                'number': row.number,
            }

        for row in rows:
            pillow.process_change(doc_to_change(_get_case(row)))
Example #17
0
    def _add_rows(self, rows):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.data_source])

        def _get_case(row):
            return {
                "_id": uuid.uuid4().hex,
                "domain": self.domain,
                "doc_type": "CommCareCase",
                "type": "city",
                "name": row.name,
                "number": row.number,
            }

        for row in rows:
            pillow.process_change(doc_to_change(_get_case(row)))
Example #18
0
    def _test_process_doc_from_couch(self, datetime_mock, pillow):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)

        # make sure case is in DB
        case = CommCareCase.wrap(sample_doc)
        with drop_connected_signals(case_post_save):
            case.save()

        # send to kafka
        since = self.pillow.get_change_feed().get_latest_offsets()
        producer.send_change(topics.CASE, doc_to_change(sample_doc).metadata)

        # run pillow and check changes
        pillow.process_changes(since=since, forever=False)
        self._check_sample_doc_state(expected_indicators)
        case.delete()
Example #19
0
    def test_mirroring(self):
        ds3 = DataSourceConfiguration.wrap(get_sample_data_source().to_json())
        ds3.engine_id = DEFAULT_ENGINE_ID
        ds3.mirrored_engine_ids = ['engine-2']
        adapter = get_indicator_adapter(ds3)
        self.assertEqual(type(adapter.adapter), MultiDBSqlAdapter)
        self.assertEqual(len(adapter.all_adapters), 2)
        for db_adapter in adapter.all_adapters:
            with db_adapter.session_context() as session:
                self.assertEqual(0, session.query(db_adapter.get_table()).count())

        with patch('pillowtop.models.KafkaCheckpoint.get_or_create_for_checkpoint_id'):
            pillow = get_case_pillow(ucr_configs=[ds3])
        sample_doc, _ = get_sample_doc_and_indicators()
        pillow.process_change(doc_to_change(sample_doc))

        for db_adapter in adapter.all_adapters:
            with db_adapter.session_context() as session:
                self.assertEqual(1, session.query(db_adapter.get_table()).count())
Example #20
0
    def test_mirroring(self):
        ds3 = DataSourceConfiguration.wrap(get_sample_data_source().to_json())
        ds3.engine_id = "default"
        ds3.mirrored_engine_ids = ['engine-2']
        adapter = get_indicator_adapter(ds3)
        self.assertEqual(type(adapter.adapter), MultiDBSqlAdapter)
        self.assertEqual(len(adapter.all_adapters), 2)
        for db_adapter in adapter.all_adapters:
            with db_adapter.session_context() as session:
                self.assertEqual(0, session.query(db_adapter.get_table()).count())

        with patch('pillowtop.models.KafkaCheckpoint.get_or_create_for_checkpoint_id'):
            pillow = get_case_pillow(ucr_configs=[ds3])
        sample_doc, _ = get_sample_doc_and_indicators()
        pillow.process_change(doc_to_change(sample_doc))

        for db_adapter in adapter.all_adapters:
            with db_adapter.session_context() as session:
                self.assertEqual(1, session.query(db_adapter.get_table()).count())
    def test_case_search_pillow(self):
        consumer = get_test_kafka_consumer(topics.CASE)
        kafka_seq = self._get_kafka_seq()

        case = self._make_case(case_properties={'foo': 'bar'})
        producer.send_change(topics.CASE, doc_to_change(case.to_json()).metadata)
        # confirm change made it to kafka
        message = next(consumer)
        change_meta = change_meta_from_kafka_message(message.value)
        self.assertEqual(case.case_id, change_meta.document_id)
        self.assertEqual(self.domain, change_meta.domain)

        # enable case search for domain
        with patch('corehq.pillows.case_search.domain_needs_search_index',
                   new=MagicMock(return_value=True)) as fake_case_search_enabled_for_domain:
            # send to elasticsearch
            self.pillow.process_changes(since=kafka_seq, forever=False)
            fake_case_search_enabled_for_domain.assert_called_with(self.domain)

        self._assert_case_in_es(self.domain, case)
Example #22
0
class IndicatorPillowTest(TestCase):

    @classmethod
    def setUpClass(cls):
        super(IndicatorPillowTest, cls).setUpClass()
        cls.config = get_sample_data_source()
        cls.config.save()
        cls.adapter = get_indicator_adapter(cls.config)
        cls.adapter.build_table()
        cls.fake_time_now = datetime(2015, 4, 24, 12, 30, 8, 24886)
        cls.pillow = get_kafka_ucr_pillow()

    @classmethod
    def tearDownClass(cls):
        cls.config.delete()
        cls.adapter.drop_table()
        super(IndicatorPillowTest, cls).tearDownClass()

    def tearDown(self):
        self.adapter.clear_table()

    @patch('corehq.apps.userreports.specs.datetime')
    def _check_sample_doc_state(self, expected_indicators, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        self.adapter.refresh_table()
        self.assertEqual(1, self.adapter.get_query_object().count())
        row = self.adapter.get_query_object()[0]
        for k in row.keys():
            v = getattr(row, k)
            if isinstance(expected_indicators[k], decimal.Decimal):
                self.assertAlmostEqual(expected_indicators[k], v)
            else:
                self.assertEqual(
                    expected_indicators[k], v,
                    'mismatched property: {} (expected {}, was {})'.format(
                        k, expected_indicators[k], v
                    )
                )

    def test_stale_rebuild(self):
        # rebuild indicators in another test will save this
        later_config = DataSourceConfiguration.get(self.config._id)
        later_config.save()
        self.assertNotEqual(self.config._rev, later_config._rev)
        with self.assertRaises(StaleRebuildError):
            self.pillow.rebuild_table(get_indicator_adapter(self.config))

    @patch('corehq.apps.userreports.specs.datetime')
    def test_change_transport(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
        self.pillow.process_change(doc_to_change(sample_doc))
        self._check_sample_doc_state(expected_indicators)

    @patch('corehq.apps.userreports.specs.datetime')
    def test_rebuild_indicators(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
        CommCareCase.get_db().save_doc(sample_doc)
        self.addCleanup(lambda id: CommCareCase.get_db().delete_doc(id), sample_doc['_id'])
        rebuild_indicators(self.config._id)
        self._check_sample_doc_state(expected_indicators)

    def test_bad_integer_datatype(self):
        bad_ints = ['a', '', None]
        for bad_value in bad_ints:
            self.pillow.process_change(doc_to_change({
                '_id': uuid.uuid4().hex,
                'doc_type': 'CommCareCase',
                'domain': 'user-reports',
                'type': 'ticket',
                'priority': bad_value
            }))
        self.adapter.refresh_table()
        # make sure we saved rows to the table for everything
        self.assertEqual(len(bad_ints), self.adapter.get_query_object().count())

    @patch('corehq.apps.userreports.specs.datetime')
    def test_basic_doc_processing(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
        self.pillow.process_change(doc_to_change(sample_doc))
        self._check_sample_doc_state(expected_indicators)

    @patch('corehq.apps.userreports.specs.datetime')
    def test_not_relevant_to_domain(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
        sample_doc['domain'] = 'not-this-domain'
        self.pillow.process_change(doc_to_change(sample_doc))
        self.adapter.refresh_table()
        self.assertEqual(0, self.adapter.get_query_object().count())

    @patch('corehq.apps.userreports.specs.datetime')
    def test_process_doc_from_couch(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)

        # make sure case is in DB
        case = CommCareCase.wrap(sample_doc)
        with drop_connected_signals(case_post_save):
            case.save()

        # send to kafka
        since = self.pillow.get_change_feed().get_latest_offsets()
        producer.send_change(topics.CASE, doc_to_change(sample_doc).metadata)

        # run pillow and check changes
        self.pillow.process_changes(since=since, forever=False)
        self._check_sample_doc_state(expected_indicators)
        case.delete()
Example #23
0
class IndicatorPillowTest(IndicatorPillowTestBase):
    @softer_assert()
    def setUp(self):
        super(IndicatorPillowTest, self).setUp()
        self.pillow = get_kafka_ucr_pillow()
        self.pillow.bootstrap(configs=[self.config])
        with trap_extra_setup(KafkaUnavailableError):
            self.pillow.get_change_feed().get_current_offsets()

    @run_with_all_ucr_backends
    def test_stale_rebuild(self):
        later_config = copy(self.config)
        later_config.save()
        self.assertNotEqual(self.config._rev, later_config._rev)
        with self.assertRaises(StaleRebuildError):
            self.pillow.rebuild_table(get_indicator_adapter(self.config))

    @patch('corehq.apps.userreports.specs.datetime')
    @run_with_all_ucr_backends
    def test_change_transport(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(
            self.fake_time_now)
        self.pillow.process_change(doc_to_change(sample_doc))
        self._check_sample_doc_state(expected_indicators)

    @patch('corehq.apps.userreports.specs.datetime')
    @run_with_all_ucr_backends
    def test_rebuild_indicators(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        self.config.save()
        sample_doc, expected_indicators = get_sample_doc_and_indicators(
            self.fake_time_now)
        CommCareCase.get_db().save_doc(sample_doc)
        self.addCleanup(lambda id: CommCareCase.get_db().delete_doc(id),
                        sample_doc['_id'])
        rebuild_indicators(self.config._id)
        self._check_sample_doc_state(expected_indicators)

    @run_with_all_ucr_backends
    def test_bad_integer_datatype(self):
        self.config.save()
        bad_ints = ['a', '', None]
        for bad_value in bad_ints:
            self.pillow.process_change(
                doc_to_change({
                    '_id': uuid.uuid4().hex,
                    'doc_type': 'CommCareCase',
                    'domain': 'user-reports',
                    'type': 'ticket',
                    'priority': bad_value
                }))
        self.adapter.refresh_table()
        # make sure we saved rows to the table for everything
        self.assertEqual(len(bad_ints),
                         self.adapter.get_query_object().count())

    @patch('corehq.apps.userreports.specs.datetime')
    @run_with_all_ucr_backends
    def test_basic_doc_processing(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(
            self.fake_time_now)
        self.pillow.process_change(doc_to_change(sample_doc))
        self._check_sample_doc_state(expected_indicators)

    @patch('corehq.apps.userreports.specs.datetime')
    @run_with_all_ucr_backends
    def test_process_doc_from_couch(self, datetime_mock):
        datetime_mock.utcnow.return_value = self.fake_time_now
        sample_doc, expected_indicators = get_sample_doc_and_indicators(
            self.fake_time_now)

        # make sure case is in DB
        case = CommCareCase.wrap(sample_doc)
        with drop_connected_signals(case_post_save):
            case.save()

        # send to kafka
        since = self.pillow.get_change_feed().get_current_offsets()
        producer.send_change(topics.CASE, doc_to_change(sample_doc).metadata)

        # run pillow and check changes
        self.pillow.process_changes(since=since, forever=False)
        self._check_sample_doc_state(expected_indicators)
        case.delete()
Example #24
0
 def test_basic_doc_processing(self, datetime_mock):
     datetime_mock.utcnow.return_value = self.fake_time_now
     sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
     self.pillow.process_change(doc_to_change(sample_doc))
     self._check_sample_doc_state(expected_indicators)
Example #25
0
    def _process_docs(self, docs):
        pillow = get_case_pillow(ucr_configs=[self.data_source])

        for doc in docs:
            pillow.process_change(doc_to_change(doc))
Example #26
0
 def test_not_relevant_to_domain(self, datetime_mock):
     datetime_mock.utcnow.return_value = self.fake_time_now
     sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
     sample_doc['domain'] = 'not-this-domain'
     self.pillow.process_change(doc_to_change(sample_doc))
     self.assertEqual(0, self.adapter.get_query_object().count())
Example #27
0
 def test_basic_doc_processing(self, datetime_mock):
     datetime_mock.utcnow.return_value = self.fake_time_now
     sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now)
     self.pillow.process_change(doc_to_change(sample_doc))
     self._check_sample_doc_state(expected_indicators)
    def _process_docs(self, docs):
        pillow = get_case_pillow(ucr_configs=[self.data_source])

        for doc in docs:
            pillow.process_change(doc_to_change(doc))
    def _process_docs(self, docs):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.data_source])

        for doc in docs:
            pillow.process_change(doc_to_change(doc))