def test_add_index(self):
# build the table without an index
config = get_sample_data_source()
config.save()
self.addCleanup(config.delete)
pillow = get_kafka_ucr_pillow()
pillow.bootstrap([config])
adapter = get_indicator_adapter(config)
engine = adapter.engine
insp = reflection.Inspector.from_engine(engine)
table_name = get_table_name(config.domain, config.table_id)
self.assertEqual(len(insp.get_indexes(table_name)), 0)
# add the index to the config
config = get_sample_data_source()
self.addCleanup(config.delete)
config.configured_indicators[0]['create_index'] = True
config.save()
adapter = get_indicator_adapter(config)
# mock rebuild table to ensure the table isn't rebuilt when adding index
pillow = get_kafka_ucr_pillow()
pillow.processors[0].rebuild_table = MagicMock()
pillow.bootstrap([config])
self.assertFalse(pillow.processors[0].rebuild_table.called)
engine = adapter.engine
insp = reflection.Inspector.from_engine(engine)
self.assertEqual(len(insp.get_indexes(table_name)), 1)
def test_add_non_nullable_column(self):
self._setup_data_source('add_non_nullable_col')
# assert new date isn't in the config
insp = reflection.Inspector.from_engine(self.engine)
table_name = get_table_name(self.config.domain, self.config.table_id)
self.assertEqual(
len([
c for c in insp.get_columns(table_name)
if c['name'] == 'new_date'
]), 0)
# add the column to the config
config = self._get_config('add_non_nullable_col')
self.addCleanup(config.delete)
config.configured_indicators.append({
"column_id": "new_date",
"type": "raw",
"display_name": "new_date opened",
"datatype": "datetime",
"property_name": "other_opened_on",
"is_nullable": False
})
config.save()
adapter = get_indicator_adapter(config)
engine = adapter.engine
# mock rebuild table to ensure the table is rebuilt
pillow = get_kafka_ucr_pillow()
pillow.processors[0].rebuild_table = MagicMock()
pillow.bootstrap([config])
self.assertTrue(pillow.processors[0].rebuild_table.called)
# column doesn't exist because rebuild table was mocked
insp = reflection.Inspector.from_engine(engine)
self.assertEqual(
len([
c for c in insp.get_columns(table_name)
if c['name'] == 'new_date'
]), 0)
# Another time without the mock to ensure the column is there
pillow = get_kafka_ucr_pillow()
pillow.bootstrap([config])
insp = reflection.Inspector.from_engine(engine)
self.assertEqual(
len([
c for c in insp.get_columns(table_name)
if c['name'] == 'new_date'
]), 1)
def setUp(self):
config1 = get_data_source_with_related_doc_type()
config1.save()
config2 = get_data_source_with_related_doc_type()
config2.table_id = 'other-config'
config2.save()
self.configs = [config1, config2]
self.adapters = [get_indicator_adapter(c) for c in self.configs]
# one pillow that has one config, the other has both configs
self.pillow1 = get_kafka_ucr_pillow(topics=['case-sql'])
self.pillow2 = get_kafka_ucr_pillow(topics=['case-sql'])
self.pillow1.bootstrap(configs=[config1])
self.pillow2.bootstrap(configs=self.configs)
with trap_extra_setup(KafkaUnavailableError):
self.pillow1.get_change_feed().get_latest_offsets()
def test_pillow_save_to_one_database_at_a_time(self):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.ds_1])
sample_doc, _ = get_sample_doc_and_indicators()
pillow.process_change(doc_to_change(sample_doc))
self.assertEqual(1, self.ds1_adapter.get_query_object().count())
self.assertEqual(0, self.ds2_adapter.get_query_object().count())
# save to the other
pillow.bootstrap(configs=[self.ds_2])
orig_id = sample_doc['_id']
sample_doc['_id'] = uuid.uuid4().hex
pillow.process_change(doc_to_change(sample_doc))
self.assertEqual(1, self.ds1_adapter.get_query_object().count())
self.assertEqual(1, self.ds2_adapter.get_query_object().count())
self.assertEqual(
1,
self.ds1_adapter.get_query_object().filter_by(
doc_id=orig_id).count())
self.assertEqual(
1,
self.ds2_adapter.get_query_object().filter_by(
doc_id=sample_doc['_id']).count())
def setUp(self):
delete_all_locations()
self.domain_obj = create_domain(self.domain)
self.region = LocationType.objects.create(domain=self.domain, name="region")
self.town = LocationType.objects.create(domain=self.domain, name="town", parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=_clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.data_source_config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_current_offsets()
def _setup_data_source(self, extra_id):
self.config = self._get_config(extra_id)
self.config.save()
pillow = get_kafka_ucr_pillow()
pillow.bootstrap([self.config])
self.adapter = get_indicator_adapter(self.config)
self.engine = self.adapter.engine
def setUp(self):
self.domain_obj = create_domain(self.domain)
self.region = LocationType.objects.create(domain=self.domain,
name="region")
self.town = LocationType.objects.create(domain=self.domain,
name="town",
parent_type=self.region)
self.data_source_config = DataSourceConfiguration(
domain=self.domain,
display_name='Locations in Westworld',
referenced_doc_type='Location',
table_id=clean_table_name(self.domain, str(uuid.uuid4().hex)),
configured_filter={},
configured_indicators=[{
"type": "expression",
"expression": {
"type": "property_name",
"property_name": "name"
},
"column_id": "location_name",
"display_name": "location_name",
"datatype": "string"
}],
)
self.data_source_config.validate()
self.data_source_config.save()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.data_source_config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_latest_offsets()
def setUpClass(cls):
super(IndicatorPillowTest, cls).setUpClass()
cls.config = get_sample_data_source()
cls.config.save()
cls.adapter = get_indicator_adapter(cls.config)
cls.adapter.build_table()
cls.fake_time_now = datetime(2015, 4, 24, 12, 30, 8, 24886)
cls.pillow = get_kafka_ucr_pillow()
def setUp(self):
self.pillow = get_kafka_ucr_pillow(topics=['case-sql'])
self.config = get_data_source_with_related_doc_type()
self.config.save()
self.adapter = get_indicator_adapter(self.config)
self.pillow.bootstrap(configs=[self.config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_latest_offsets()
def test_pillow_save_to_multiple_databases(self):
self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.ds_1, self.ds_2])
self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
sample_doc, _ = get_sample_doc_and_indicators()
pillow.processor(doc_to_change(sample_doc))
self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
self.assertEqual(1, self.ds1_adapter.get_query_object().count())
self.assertEqual(1, self.ds2_adapter.get_query_object().count())
def test_pillow_save_to_multiple_databases(self):
self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.ds_1, self.ds_2])
self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
sample_doc, _ = get_sample_doc_and_indicators()
pillow.process_change(doc_to_change(sample_doc))
self.assertNotEqual(self.ds1_adapter.engine.url, self.ds2_adapter.engine.url)
self.assertEqual(1, self.ds1_adapter.get_query_object().count())
self.assertEqual(1, self.ds2_adapter.get_query_object().count())
def setUpClass(cls):
super(AsyncIndicatorTest, cls).setUpClass()
cls.pillow = get_kafka_ucr_pillow()
cls.config = get_data_source_with_related_doc_type()
cls.config.asynchronous = True
cls.config.save()
cls.adapter = get_indicator_adapter(cls.config)
cls.pillow.bootstrap(configs=[cls.config])
with trap_extra_setup(KafkaUnavailableError):
cls.pillow.get_change_feed().get_latest_offsets()
def _add_rows(self, rows):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.data_source])
def _get_case(row):
return {
'_id': uuid.uuid4().hex,
'domain': self.domain,
'doc_type': 'CommCareCase',
'type': 'city',
'name': row.name,
'number': row.number,
}
for row in rows:
pillow.process_change(doc_to_change(_get_case(row)))
def _add_rows(self, rows):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.data_source])
def _get_case(row):
return {
"_id": uuid.uuid4().hex,
"domain": self.domain,
"doc_type": "CommCareCase",
"type": "city",
"name": row.name,
"number": row.number,
}
for row in rows:
pillow.process_change(doc_to_change(_get_case(row)))
def _add_rows(self, rows):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.data_source])
def _get_case(row):
return {
'_id': uuid.uuid4().hex,
'domain': self.domain,
'doc_type': 'CommCareCase',
'type': 'city',
'name': row.name,
'number': row.number,
}
for row in rows:
pillow.process_change(doc_to_change(_get_case(row)))
def test_pillow_save_to_one_database_at_a_time(self):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.ds_1])
sample_doc, _ = get_sample_doc_and_indicators()
pillow.processor(doc_to_change(sample_doc))
self.assertEqual(1, self.ds1_adapter.get_query_object().count())
self.assertEqual(0, self.ds2_adapter.get_query_object().count())
# save to the other
pillow.bootstrap(configs=[self.ds_2])
orig_id = sample_doc['_id']
sample_doc['_id'] = uuid.uuid4().hex
pillow.processor(doc_to_change(sample_doc))
self.assertEqual(1, self.ds1_adapter.get_query_object().count())
self.assertEqual(1, self.ds2_adapter.get_query_object().count())
self.assertEqual(1, self.ds1_adapter.get_query_object().filter_by(doc_id=orig_id).count())
self.assertEqual(1, self.ds2_adapter.get_query_object().filter_by(doc_id=sample_doc['_id']).count())
def setUp(self):
super(IndicatorPillowTest, self).setUp()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_current_offsets()
def setUp(self):
super(IndicatorPillowTest, self).setUp()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.config])
with trap_extra_setup(KafkaUnavailableError):
self.pillow.get_change_feed().get_current_offsets()
def _process_docs(self, docs):
pillow = get_kafka_ucr_pillow()
pillow.bootstrap(configs=[self.data_source])
for doc in docs:
pillow.process_change(doc_to_change(doc))
def setUp(self):
super(IndicatorPillowTest, self).setUp()
self.pillow = get_kafka_ucr_pillow()
self.pillow.bootstrap(configs=[self.config])
