def __init__(self, *args, **kwargs): if ( self.requires_examples and not os.environ.get('SOLO_TEST') and not os.environ.get('examples_loaded') ): cli.load_examples(load_test_data=True) utils.init(caravel) os.environ['examples_loaded'] = '1' super(CaravelTestCase, self).__init__(*args, **kwargs) self.client = app.test_client() self.maxDiff = None utils.init(caravel) admin = appbuilder.sm.find_user('admin') if not admin: appbuilder.sm.add_user( 'admin', 'admin', ' user', '*****@*****.**', appbuilder.sm.find_role('Admin'), password='******') gamma = appbuilder.sm.find_user('gamma') if not gamma: appbuilder.sm.add_user( 'gamma', 'gamma', 'user', '*****@*****.**', appbuilder.sm.find_role('Gamma'), password='******') alpha = appbuilder.sm.find_user('alpha') if not alpha: appbuilder.sm.add_user( 'alpha', 'alpha', 'user', '*****@*****.**', appbuilder.sm.find_role('Alpha'), password='******') # create druid cluster and druid datasources session = db.session cluster = session.query(models.DruidCluster).filter_by( cluster_name="druid_test").first() if not cluster: cluster = models.DruidCluster(cluster_name="druid_test") session.add(cluster) session.commit() druid_datasource1 = models.DruidDatasource( datasource_name='druid_ds_1', cluster_name='druid_test' ) session.add(druid_datasource1) druid_datasource2 = models.DruidDatasource( datasource_name='druid_ds_2', cluster_name='druid_test' ) session.add(druid_datasource2) session.commit() utils.init(caravel)
def test_druid_sync_from_config(self): cluster = models.DruidCluster(cluster_name="new_druid") db.session.add(cluster) db.session.commit() cfg = { "user": "******", "cluster": "new_druid", "config": { "name": "test_click", "dimensions": ["affiliate_id", "campaign", "first_seen"], "metrics_spec": [{ "type": "count", "name": "count" }, { "type": "sum", "name": "sum" }], "batch_ingestion": { "sql": "SELECT * FROM clicks WHERE d='{{ ds }}'", "ts_column": "d", "sources": [{ "table": "clicks", "partition": "d='{{ ds }}'" }] } } } resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg)) druid_ds = db.session.query(DruidDatasource).filter_by( datasource_name="test_click").first() assert set([c.column_name for c in druid_ds.columns ]) == set(["affiliate_id", "campaign", "first_seen"]) assert set([m.metric_name for m in druid_ds.metrics]) == set(["count", "sum"]) assert resp.status_code == 201 # datasource exists, not changes required resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg)) druid_ds = db.session.query(DruidDatasource).filter_by( datasource_name="test_click").first() assert set([c.column_name for c in druid_ds.columns ]) == set(["affiliate_id", "campaign", "first_seen"]) assert set([m.metric_name for m in druid_ds.metrics]) == set(["count", "sum"]) assert resp.status_code == 201 # datasource exists, add new metrics and dimentions cfg = { "user": "******", "cluster": "new_druid", "config": { "name": "test_click", "dimensions": ["affiliate_id", "second_seen"], "metrics_spec": [{ "type": "bla", "name": "sum" }, { "type": "unique", "name": "unique" }], } } resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg)) druid_ds = db.session.query(DruidDatasource).filter_by( datasource_name="test_click").first() # columns and metrics are not deleted if config is changed as # user could define his own dimensions / metrics and want to keep them assert set([c.column_name for c in druid_ds.columns]) == set( ["affiliate_id", "campaign", "first_seen", "second_seen"]) assert set([m.metric_name for m in druid_ds.metrics ]) == set(["count", "sum", "unique"]) # metric type will not be overridden, sum stays instead of bla assert set([m.metric_type for m in druid_ds.metrics ]) == set(["longSum", "sum", "unique"]) assert resp.status_code == 201