def __init__(self, *args, **kwargs):
if (
self.requires_examples and
not os.environ.get('SOLO_TEST') and
not os.environ.get('examples_loaded')
):
cli.load_examples(load_test_data=True)
utils.init(caravel)
os.environ['examples_loaded'] = '1'
super(CaravelTestCase, self).__init__(*args, **kwargs)
self.client = app.test_client()
self.maxDiff = None
utils.init(caravel)
admin = appbuilder.sm.find_user('admin')
if not admin:
appbuilder.sm.add_user(
'admin', 'admin', ' user', '*****@*****.**',
appbuilder.sm.find_role('Admin'),
password='******')
gamma = appbuilder.sm.find_user('gamma')
if not gamma:
appbuilder.sm.add_user(
'gamma', 'gamma', 'user', '*****@*****.**',
appbuilder.sm.find_role('Gamma'),
password='******')
alpha = appbuilder.sm.find_user('alpha')
if not alpha:
appbuilder.sm.add_user(
'alpha', 'alpha', 'user', '*****@*****.**',
appbuilder.sm.find_role('Alpha'),
password='******')
# create druid cluster and druid datasources
session = db.session
cluster = session.query(models.DruidCluster).filter_by(
cluster_name="druid_test").first()
if not cluster:
cluster = models.DruidCluster(cluster_name="druid_test")
session.add(cluster)
session.commit()
druid_datasource1 = models.DruidDatasource(
datasource_name='druid_ds_1',
cluster_name='druid_test'
)
session.add(druid_datasource1)
druid_datasource2 = models.DruidDatasource(
datasource_name='druid_ds_2',
cluster_name='druid_test'
)
session.add(druid_datasource2)
session.commit()
utils.init(caravel)
def test_druid_sync_from_config(self):
cluster = models.DruidCluster(cluster_name="new_druid")
db.session.add(cluster)
db.session.commit()
cfg = {
"user": "******",
"cluster": "new_druid",
"config": {
"name":
"test_click",
"dimensions": ["affiliate_id", "campaign", "first_seen"],
"metrics_spec": [{
"type": "count",
"name": "count"
}, {
"type": "sum",
"name": "sum"
}],
"batch_ingestion": {
"sql": "SELECT * FROM clicks WHERE d='{{ ds }}'",
"ts_column": "d",
"sources": [{
"table": "clicks",
"partition": "d='{{ ds }}'"
}]
}
}
}
resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
druid_ds = db.session.query(DruidDatasource).filter_by(
datasource_name="test_click").first()
assert set([c.column_name for c in druid_ds.columns
]) == set(["affiliate_id", "campaign", "first_seen"])
assert set([m.metric_name
for m in druid_ds.metrics]) == set(["count", "sum"])
assert resp.status_code == 201
# datasource exists, not changes required
resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
druid_ds = db.session.query(DruidDatasource).filter_by(
datasource_name="test_click").first()
assert set([c.column_name for c in druid_ds.columns
]) == set(["affiliate_id", "campaign", "first_seen"])
assert set([m.metric_name
for m in druid_ds.metrics]) == set(["count", "sum"])
assert resp.status_code == 201
# datasource exists, add new metrics and dimentions
cfg = {
"user": "******",
"cluster": "new_druid",
"config": {
"name":
"test_click",
"dimensions": ["affiliate_id", "second_seen"],
"metrics_spec": [{
"type": "bla",
"name": "sum"
}, {
"type": "unique",
"name": "unique"
}],
}
}
resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
druid_ds = db.session.query(DruidDatasource).filter_by(
datasource_name="test_click").first()
# columns and metrics are not deleted if config is changed as
# user could define his own dimensions / metrics and want to keep them
assert set([c.column_name for c in druid_ds.columns]) == set(
["affiliate_id", "campaign", "first_seen", "second_seen"])
assert set([m.metric_name for m in druid_ds.metrics
]) == set(["count", "sum", "unique"])
# metric type will not be overridden, sum stays instead of bla
assert set([m.metric_type for m in druid_ds.metrics
]) == set(["longSum", "sum", "unique"])
assert resp.status_code == 201
