def build_query(self, query_type, args): """ Build query based on given query type and arguments. :param string query_type: a type of query :param dict args: the dict of args to be sent :return: the resulting query :rtype: Query """ query_dict = {'queryType': query_type} for key, val in six.iteritems(args): if key == 'aggregations': query_dict[key] = build_aggregators(val) elif key == 'post_aggregations': query_dict['postAggregations'] = Postaggregator.build_post_aggregators(val) elif key == 'datasource': query_dict['dataSource'] = val elif key == 'paging_spec': query_dict['pagingSpec'] = val elif key == 'limit_spec': query_dict['limitSpec'] = val elif key == "filter": query_dict[key] = Filter.build_filter(val) elif key == "having": query_dict[key] = Having.build_having(val) elif key == 'dimension': query_dict[key] = build_dimension(val) elif key == 'dimensions': query_dict[key] = [build_dimension(v) for v in val] else: query_dict[key] = val self.last_query = Query(query_dict, query_type) return self.last_query
def test_build_aggregators(self): agg_input = { 'agg1': aggregators.count('metric1'), 'agg2': aggregators.longsum('metric2'), 'agg3': aggregators.doublesum('metric3'), 'agg4': aggregators.doublemin('metric4'), 'agg5': aggregators.doublemax('metric5'), 'agg6': aggregators.hyperunique('metric6'), 'agg7': aggregators.cardinality('dim1'), 'agg8': aggregators.cardinality(['dim1', 'dim2'], by_row=True), 'agg9': aggregators.thetasketch('dim1'), 'agg10': aggregators.thetasketch('metric7'), 'agg11': aggregators.thetasketch('metric8', isinputthetasketch = True, size=8192) } built_agg = aggregators.build_aggregators(agg_input) expected = [ {'name': 'agg1', 'type': 'count', 'fieldName': 'metric1'}, {'name': 'agg2', 'type': 'longSum', 'fieldName': 'metric2'}, {'name': 'agg3', 'type': 'doubleSum', 'fieldName': 'metric3'}, {'name': 'agg4', 'type': 'doubleMin', 'fieldName': 'metric4'}, {'name': 'agg5', 'type': 'doubleMax', 'fieldName': 'metric5'}, {'name': 'agg6', 'type': 'hyperUnique', 'fieldName': 'metric6'}, {'name': 'agg7', 'type': 'cardinality', 'fieldNames': ['dim1'], 'byRow': False}, {'name': 'agg8', 'type': 'cardinality', 'fieldNames': ['dim1', 'dim2'], 'byRow': True}, {'name': 'agg9', 'type': 'thetaSketch', 'fieldName': 'dim1', 'isInputThetaSketch': False, 'size': 16384}, {'name': 'agg10', 'type': 'thetaSketch', 'fieldName': 'metric7', 'isInputThetaSketch': False, 'size': 16384}, {'name': 'agg11', 'type': 'thetaSketch', 'fieldName': 'metric8', 'isInputThetaSketch': True, 'size': 8192} ] assert (sorted(built_agg, key=itemgetter('name')) == sorted(expected, key=itemgetter('name')))
def test_build_filtered_aggregator(self): filter_ = filters.Filter(dimension="dim", value="val") agg_input = { "agg1": aggregators.filtered(filter_, aggregators.count("metric1")), "agg2": aggregators.filtered(filter_, aggregators.longsum("metric2")), "agg3": aggregators.filtered(filter_, aggregators.doublesum("metric3")), "agg4": aggregators.filtered(filter_, aggregators.min("metric4")), "agg5": aggregators.filtered(filter_, aggregators.max("metric5")), "agg6": aggregators.filtered(filter_, aggregators.hyperunique("metric6")), "agg7": aggregators.filtered(filter_, aggregators.cardinality("dim1")), "agg8": aggregators.filtered(filter_, aggregators.cardinality(["dim1", "dim2"], by_row=True)), } base = {"type": "filtered", "filter": {"type": "selector", "dimension": "dim", "value": "val"}} aggs = [ {"name": "agg1", "type": "count", "fieldName": "metric1"}, {"name": "agg2", "type": "longSum", "fieldName": "metric2"}, {"name": "agg3", "type": "doubleSum", "fieldName": "metric3"}, {"name": "agg4", "type": "min", "fieldName": "metric4"}, {"name": "agg5", "type": "max", "fieldName": "metric5"}, {"name": "agg6", "type": "hyperUnique", "fieldName": "metric6"}, {"name": "agg7", "type": "cardinality", "fieldNames": ["dim1"], "byRow": False}, {"name": "agg8", "type": "cardinality", "fieldNames": ["dim1", "dim2"], "byRow": True}, ] expected = [] for agg in aggs: exp = deepcopy(base) exp.update({"aggregator": agg}) expected.append(exp) built_agg = aggregators.build_aggregators(agg_input) expected = sorted(built_agg, key=lambda k: itemgetter("name")(itemgetter("aggregator")(k))) actual = sorted(expected, key=lambda k: itemgetter("name")(itemgetter("aggregator")(k))) assert expected == actual
def test_build_filtered_aggregator(self): filter_ = filters.Filter(dimension='dim', value='val') agg_input = { 'agg1': aggregators.filtered(filter_, aggregators.count('metric1')), 'agg2': aggregators.filtered(filter_, aggregators.longsum('metric2')), 'agg3': aggregators.filtered(filter_, aggregators.doublesum('metric3')), 'agg4': aggregators.filtered(filter_, aggregators.min('metric4')), 'agg5': aggregators.filtered(filter_, aggregators.max('metric5')), 'agg6': aggregators.filtered(filter_, aggregators.hyperunique('metric6')), 'agg7': aggregators.filtered(filter_, aggregators.cardinality('dim1')), 'agg8': aggregators.filtered(filter_, aggregators.cardinality(['dim1', 'dim2'], by_row=True)), } base = { 'type': 'filtered', 'filter': { 'type': 'selector', 'dimension': 'dim', 'value': 'val' } } aggs = [ {'name': 'agg1', 'type': 'count', 'fieldName': 'metric1'}, {'name': 'agg2', 'type': 'longSum', 'fieldName': 'metric2'}, {'name': 'agg3', 'type': 'doubleSum', 'fieldName': 'metric3'}, {'name': 'agg4', 'type': 'min', 'fieldName': 'metric4'}, {'name': 'agg5', 'type': 'max', 'fieldName': 'metric5'}, {'name': 'agg6', 'type': 'hyperUnique', 'fieldName': 'metric6'}, {'name': 'agg7', 'type': 'cardinality', 'fieldNames': ['dim1'], 'byRow': False}, {'name': 'agg8', 'type': 'cardinality', 'fieldNames': ['dim1', 'dim2'], 'byRow': True}, ] expected = [] for agg in aggs: exp = deepcopy(base) exp.update({'aggregator': agg}) expected.append(exp) built_agg = aggregators.build_aggregators(agg_input) expected = sorted(built_agg, key=lambda k: itemgetter('name')( itemgetter('aggregator')(k))) actual = sorted(expected, key=lambda k: itemgetter('name')( itemgetter('aggregator')(k))) assert expected == actual
def test_nested_filtered_aggregator(self): filter1 = filters.Filter(dimension='dim1', value='val') filter2 = filters.Filter(dimension='dim2', value='val') agg = aggregators.filtered(filter1, aggregators.filtered(filter2, aggregators.count('metric1'))) actual = aggregators.build_aggregators({'agg_name': agg}) # the innermost aggregation must have 'agg_name' expected = [{ 'type': 'filtered', 'aggregator': { 'type': 'filtered', 'aggregator': {'fieldName': 'metric1', 'type': 'count', 'name': 'agg_name'}, 'filter': {'dimension': 'dim2', 'value': 'val', 'type': 'selector'}}, 'filter': {'dimension': 'dim1', 'value': 'val', 'type': 'selector'} }] assert expected == actual
def test_nested_filtered_aggregator(self): filter1 = filters.Filter(dimension="dim1", value="val") filter2 = filters.Filter(dimension="dim2", value="val") agg = aggregators.filtered(filter1, aggregators.filtered(filter2, aggregators.count("metric1"))) actual = aggregators.build_aggregators({"agg_name": agg}) # the innermost aggregation must have 'agg_name' expected = [ { "type": "filtered", "aggregator": { "type": "filtered", "aggregator": {"fieldName": "metric1", "type": "count", "name": "agg_name"}, "filter": {"dimension": "dim2", "value": "val", "type": "selector"}, }, "filter": {"dimension": "dim1", "value": "val", "type": "selector"}, } ] assert expected == actual
def test_build_aggregators(self): agg_input = { 'agg1': aggregators.count('metric1'), 'agg2': aggregators.longsum('metric2'), 'agg3': aggregators.doublesum('metric3'), 'agg4': aggregators.min('metric4'), 'agg5': aggregators.max('metric5'), 'agg6': aggregators.hyperunique('metric6') } built_agg = aggregators.build_aggregators(agg_input) expected = [ {'name': 'agg1', 'type': 'count', 'fieldName': 'metric1'}, {'name': 'agg2', 'type': 'longSum', 'fieldName': 'metric2'}, {'name': 'agg3', 'type': 'doubleSum', 'fieldName': 'metric3'}, {'name': 'agg4', 'type': 'min', 'fieldName': 'metric4'}, {'name': 'agg5', 'type': 'max', 'fieldName': 'metric5'}, {'name': 'agg6', 'type': 'hyperUnique', 'fieldName': 'metric6'}, ] assert (sorted(built_agg, key=itemgetter('name')) == sorted(expected, key=itemgetter('name')))
def build_query(self, query_type, args): """ Build query based on given query type and arguments. :param string query_type: a type of query :param dict args: the dict of args to be sent :return: the resulting query :rtype: Query """ query_dict = {"queryType": query_type} for key, val in args.items(): if key == "aggregations": query_dict[key] = build_aggregators(val) elif key == "post_aggregations": query_dict[ "postAggregations"] = Postaggregator.build_post_aggregators( val) elif key == "context": query_dict["context"] = val elif key == "datasource": query_dict["dataSource"] = self.parse_datasource( val, query_type) elif key == "paging_spec": query_dict["pagingSpec"] = val elif key == "limit_spec": query_dict["limitSpec"] = val elif key == "filter" and val is not None: query_dict[key] = Filter.build_filter(val) elif key == "having" and val is not None: query_dict[key] = Having.build_having(val) elif key == "dimension" and val is not None: query_dict[key] = build_dimension(val) elif key == "dimensions": query_dict[key] = [build_dimension(v) for v in val] else: query_dict[key] = val self.last_query = Query(query_dict, query_type) return self.last_query
def build_query(self, query_type, args): """ Build query based on given query type and arguments. :param string query_type: a type of query :param dict args: the dict of args to be sent :return: the resulting query :rtype: Query """ query_dict = {'queryType': query_type} for key, val in six.iteritems(args): if key == 'aggregations': query_dict[key] = build_aggregators(val) elif key == 'post_aggregations': query_dict['postAggregations'] = \ Postaggregator.build_post_aggregators(val) elif key == 'context': query_dict['context'] = val elif key == 'datasource': query_dict['dataSource'] = self.parse_datasource( val, query_type) elif key == 'paging_spec': query_dict['pagingSpec'] = val elif key == 'limit_spec': query_dict['limitSpec'] = val elif key == "filter" and val is not None: query_dict[key] = Filter.build_filter(val) elif key == "having" and val is not None: query_dict[key] = Having.build_having(val) elif key == 'dimension' and val is not None: query_dict[key] = build_dimension(val) elif key == 'dimensions': query_dict[key] = [build_dimension(v) for v in val] else: query_dict[key] = val self.last_query = Query(query_dict, query_type) return self.last_query
def build_query_dict(self, query_type, args): query_dict = {"queryType": query_type} for key, val in six.iteritems(args): if key == "aggregations": query_dict[key] = build_aggregators(val) elif key == "post_aggregations": query_dict[ "postAggregations"] = Postaggregator.build_post_aggregators( val) elif key == "context": query_dict["context"] = val elif key == "datasource": query_dict["dataSource"] = self.parse_datasource( val, query_type) elif key == "paging_spec": query_dict["pagingSpec"] = val elif key == "limit_spec": query_dict["limitSpec"] = val elif key == "filter" and val is not None: query_dict[key] = Filter.build_filter(val) elif key == "having" and val is not None: query_dict[key] = Having.build_having(val) elif key == "dimension" and val is not None: query_dict[key] = build_dimension(val) elif key == "dimensions": query_dict[key] = [build_dimension(v) for v in val] elif key == 'virtualColumns': query_dict[key] = [ VirtualColumn.build_virtual_column(v) for v in val ] elif key == 'sub_query' and val is not None: query_dict['dataSource'] = { 'type': 'query', 'query': self.build_query_dict(query_type, val) } elif val is not None: query_dict[key] = val return query_dict
def test_build_aggregators(self): agg_input = { "agg1": aggregators.count("metric1"), "agg2": aggregators.longsum("metric2"), "agg3": aggregators.doublesum("metric3"), "agg4": aggregators.min("metric4"), "agg5": aggregators.max("metric5"), "agg6": aggregators.hyperunique("metric6"), "agg7": aggregators.cardinality("dim1"), "agg8": aggregators.cardinality(["dim1", "dim2"], by_row=True), } built_agg = aggregators.build_aggregators(agg_input) expected = [ {"name": "agg1", "type": "count", "fieldName": "metric1"}, {"name": "agg2", "type": "longSum", "fieldName": "metric2"}, {"name": "agg3", "type": "doubleSum", "fieldName": "metric3"}, {"name": "agg4", "type": "min", "fieldName": "metric4"}, {"name": "agg5", "type": "max", "fieldName": "metric5"}, {"name": "agg6", "type": "hyperUnique", "fieldName": "metric6"}, {"name": "agg7", "type": "cardinality", "fieldNames": ["dim1"], "byRow": False}, {"name": "agg8", "type": "cardinality", "fieldNames": ["dim1", "dim2"], "byRow": True}, ] assert sorted(built_agg, key=itemgetter("name")) == sorted(expected, key=itemgetter("name"))
def test_build_aggregators(self): agg_input = { 'agg1': aggregators.count('metric1'), 'agg2': aggregators.longsum('metric2'), 'agg3': aggregators.doublesum('metric3'), 'agg4': aggregators.min('metric4'), 'agg5': aggregators.max('metric5'), 'agg6': aggregators.hyperunique('metric6'), 'agg7': aggregators.cardinality('dim1'), 'agg8': aggregators.cardinality(['dim1', 'dim2'], by_row=True) } built_agg = aggregators.build_aggregators(agg_input) expected = [ {'name': 'agg1', 'type': 'count', 'fieldName': 'metric1'}, {'name': 'agg2', 'type': 'longSum', 'fieldName': 'metric2'}, {'name': 'agg3', 'type': 'doubleSum', 'fieldName': 'metric3'}, {'name': 'agg4', 'type': 'min', 'fieldName': 'metric4'}, {'name': 'agg5', 'type': 'max', 'fieldName': 'metric5'}, {'name': 'agg6', 'type': 'hyperUnique', 'fieldName': 'metric6'}, {'name': 'agg7', 'type': 'cardinality', 'fieldNames': ['dim1'], 'byRow': False}, {'name': 'agg8', 'type': 'cardinality', 'fieldNames': ['dim1', 'dim2'], 'byRow': True}, ] assert (sorted(built_agg, key=itemgetter('name')) == sorted(expected, key=itemgetter('name')))
def test_build_aggregators(self): agg_input = { "agg1": aggregators.count("metric1"), "agg2": aggregators.longsum("metric2"), "agg3": aggregators.doublesum("metric3"), "agg4": aggregators.doublemin("metric4"), "agg5": aggregators.doublemax("metric5"), "agg6": aggregators.hyperunique("metric6"), "agg7": aggregators.cardinality("dim1"), "agg8": aggregators.cardinality(["dim1", "dim2"], by_row=True), "agg9": aggregators.thetasketch("dim1"), "agg10": aggregators.thetasketch("metric7"), "agg11": aggregators.thetasketch("metric8", isinputthetasketch=True, size=8192), } built_agg = aggregators.build_aggregators(agg_input) expected = [ { "name": "agg1", "type": "count", "fieldName": "metric1" }, { "name": "agg2", "type": "longSum", "fieldName": "metric2" }, { "name": "agg3", "type": "doubleSum", "fieldName": "metric3" }, { "name": "agg4", "type": "doubleMin", "fieldName": "metric4" }, { "name": "agg5", "type": "doubleMax", "fieldName": "metric5" }, { "name": "agg6", "type": "hyperUnique", "fieldName": "metric6" }, { "name": "agg7", "type": "cardinality", "fieldNames": ["dim1"], "byRow": False, }, { "name": "agg8", "type": "cardinality", "fieldNames": ["dim1", "dim2"], "byRow": True, }, { "name": "agg9", "type": "thetaSketch", "fieldName": "dim1", "isInputThetaSketch": False, "size": 16384, }, { "name": "agg10", "type": "thetaSketch", "fieldName": "metric7", "isInputThetaSketch": False, "size": 16384, }, { "name": "agg11", "type": "thetaSketch", "fieldName": "metric8", "isInputThetaSketch": True, "size": 8192, }, ] assert sorted(built_agg, key=itemgetter("name")) == sorted(expected, key=itemgetter("name"))
def test_build_filtered_aggregator(self): filter_ = filters.Filter(dimension="dim", value="val") agg_input = { "agg1": aggregators.filtered(filter_, aggregators.count("metric1")), "agg2": aggregators.filtered(filter_, aggregators.longsum("metric2")), "agg3": aggregators.filtered(filter_, aggregators.doublesum("metric3")), "agg4": aggregators.filtered(filter_, aggregators.doublemin("metric4")), "agg5": aggregators.filtered(filter_, aggregators.doublemax("metric5")), "agg6": aggregators.filtered(filter_, aggregators.hyperunique("metric6")), "agg7": aggregators.filtered(filter_, aggregators.cardinality("dim1")), "agg8": aggregators.filtered( filter_, aggregators.cardinality(["dim1", "dim2"], by_row=True)), "agg9": aggregators.filtered(filter_, aggregators.thetasketch("dim1")), "agg10": aggregators.filtered(filter_, aggregators.thetasketch("metric7")), "agg11": aggregators.filtered( filter_, aggregators.thetasketch("metric8", isinputthetasketch=True, size=8192), ), } base = { "type": "filtered", "filter": { "type": "selector", "dimension": "dim", "value": "val" }, } aggs = [ { "name": "agg1", "type": "count", "fieldName": "metric1" }, { "name": "agg2", "type": "longSum", "fieldName": "metric2" }, { "name": "agg3", "type": "doubleSum", "fieldName": "metric3" }, { "name": "agg4", "type": "doubleMin", "fieldName": "metric4" }, { "name": "agg5", "type": "doubleMax", "fieldName": "metric5" }, { "name": "agg6", "type": "hyperUnique", "fieldName": "metric6" }, { "name": "agg7", "type": "cardinality", "fieldNames": ["dim1"], "byRow": False, }, { "name": "agg8", "type": "cardinality", "fieldNames": ["dim1", "dim2"], "byRow": True, }, { "name": "agg9", "type": "thetaSketch", "fieldName": "dim1", "isInputThetaSketch": False, "size": 16384, }, { "name": "agg10", "type": "thetaSketch", "fieldName": "metric7", "isInputThetaSketch": False, "size": 16384, }, { "name": "agg11", "type": "thetaSketch", "fieldName": "metric8", "isInputThetaSketch": True, "size": 8192, }, ] expected = [] for agg in aggs: exp = deepcopy(base) exp.update({"aggregator": agg}) expected.append(exp) built_agg = aggregators.build_aggregators(agg_input) expected = sorted(built_agg, key=lambda k: itemgetter("name") (itemgetter("aggregator")(k))) actual = sorted(expected, key=lambda k: itemgetter("name") (itemgetter("aggregator")(k))) assert expected == actual
def test_build_filtered_aggregator(self): filter_ = filters.Filter(dimension='dim', value='val') agg_input = { 'agg1': aggregators.filtered(filter_, aggregators.count('metric1')), 'agg2': aggregators.filtered(filter_, aggregators.longsum('metric2')), 'agg3': aggregators.filtered(filter_, aggregators.doublesum('metric3')), 'agg4': aggregators.filtered(filter_, aggregators.doublemin('metric4')), 'agg5': aggregators.filtered(filter_, aggregators.doublemax('metric5')), 'agg6': aggregators.filtered(filter_, aggregators.hyperunique('metric6')), 'agg7': aggregators.filtered(filter_, aggregators.cardinality('dim1')), 'agg8': aggregators.filtered(filter_, aggregators.cardinality(['dim1', 'dim2'], by_row=True)), 'agg9': aggregators.filtered(filter_, aggregators.thetasketch('dim1')), 'agg10': aggregators.filtered(filter_, aggregators.thetasketch('metric7')), 'agg11': aggregators.filtered(filter_, aggregators.thetasketch('metric8', isinputthetasketch = True, size=8192)), } base = { 'type': 'filtered', 'filter': { 'type': 'selector', 'dimension': 'dim', 'value': 'val' } } aggs = [ {'name': 'agg1', 'type': 'count', 'fieldName': 'metric1'}, {'name': 'agg2', 'type': 'longSum', 'fieldName': 'metric2'}, {'name': 'agg3', 'type': 'doubleSum', 'fieldName': 'metric3'}, {'name': 'agg4', 'type': 'doubleMin', 'fieldName': 'metric4'}, {'name': 'agg5', 'type': 'doubleMax', 'fieldName': 'metric5'}, {'name': 'agg6', 'type': 'hyperUnique', 'fieldName': 'metric6'}, {'name': 'agg7', 'type': 'cardinality', 'fieldNames': ['dim1'], 'byRow': False}, {'name': 'agg8', 'type': 'cardinality', 'fieldNames': ['dim1', 'dim2'], 'byRow': True}, {'name': 'agg9', 'type': 'thetaSketch', 'fieldName': 'dim1', 'isInputThetaSketch': False, 'size': 16384}, {'name': 'agg10', 'type': 'thetaSketch', 'fieldName': 'metric7', 'isInputThetaSketch': False, 'size': 16384}, {'name': 'agg11', 'type': 'thetaSketch', 'fieldName': 'metric8', 'isInputThetaSketch': True, 'size': 8192} ] expected = [] for agg in aggs: exp = deepcopy(base) exp.update({'aggregator': agg}) expected.append(exp) built_agg = aggregators.build_aggregators(agg_input) expected = sorted(built_agg, key=lambda k: itemgetter('name')( itemgetter('aggregator')(k))) actual = sorted(expected, key=lambda k: itemgetter('name')( itemgetter('aggregator')(k))) assert expected == actual