def test_build_query_none_type(self):
# given
expected_query_dict = {
"queryType": None,
"dataSource": "things",
"aggregations": [{"fieldName": "thing", "name": "count", "type": "count"}],
"filter": {"dimension": "one", "type": "selector", "value": 1},
"having": {"aggregation": "sum", "type": "greaterThan", "value": 1},
"dimension": "dim1",
}
builder = QueryBuilder()
# when
builder_dict = {
"datasource": "things",
"aggregations": {"count": aggregators.count("thing")},
"filter": filters.Dimension("one") == 1,
"having": having.Aggregation("sum") > 1,
"dimension": "dim1",
}
query = builder.build_query(None, builder_dict)
# then
assert query.query_dict == expected_query_dict
# you should be able to pass `None` to dimension/having/filter
for v in ["dimension", "having", "filter"]:
expected_query_dict[v] = None
builder_dict[v] = None
query = builder.build_query(None, builder_dict)
assert query.query_dict == expected_query_dict
def test_ne_dimension(self):
d = filters.Dimension('dim')
actual = filters.Filter.build_filter(d != 'val')
expected = {
'field': {
'dimension': 'dim',
'type': 'selector',
'value': 'val'
},
'type': 'not'
}
assert actual == expected
def test_ne_dimension(self):
d = filters.Dimension("dim")
actual = filters.Filter.build_filter(d != "val")
expected = {
"field": {
"dimension": "dim",
"type": "selector",
"value": "val"
},
"type": "not",
}
assert actual == expected
def test_build_query_none_type(self):
# given
expected_query_dict = {
'queryType':
None,
'dataSource':
'things',
'aggregations': [{
'fieldName': 'thing',
'name': 'count',
'type': 'count'
}],
'filter': {
'dimension': 'one',
'type': 'selector',
'value': 1
},
'having': {
'aggregation': 'sum',
'type': 'greaterThan',
'value': 1
},
'dimension':
'dim1',
}
builder = QueryBuilder()
# when
builder_dict = {
'datasource': 'things',
'aggregations': {
'count': aggregators.count('thing'),
},
'filter': filters.Dimension('one') == 1,
'having': having.Aggregation('sum') > 1,
'dimension': 'dim1',
}
query = builder.build_query(None, builder_dict)
# then
assert query.query_dict == expected_query_dict
# you should be able to pass `None` to dimension/having/filter
for v in ['dimension', 'having', 'filter']:
expected_query_dict[v] = None
builder_dict[v] = None
query = builder.build_query(None, builder_dict)
assert query.query_dict == expected_query_dict
def test_build_query(self):
# given
expected_query_dict = {
'queryType': None,
'dataSource': 'things',
'aggregations': [{'fieldName': 'thing', 'name': 'count', 'type': 'count'}],
'postAggregations': [{
'fields': [{
'fieldName': 'sum', 'type': 'fieldAccess',
}, {
'fieldName': 'count', 'type': 'fieldAccess',
}],
'fn': '/',
'name': 'avg',
'type': 'arithmetic',
}],
'pagingSpec': {'pagingIdentifies': {}, 'threshold': 1},
'filter': {'dimension': 'one', 'type': 'selector', 'value': 1},
'having': {'aggregation': 'sum', 'type': 'greaterThan', 'value': 1},
'new_key': 'value',
'virtualColumns': [{
'type': 'expression', 'name': 'foo', 'expression': "concat('foo' + page)", 'outputType': 'STRING'
}],
}
builder = QueryBuilder()
# when
query = builder.build_query(None, {
'datasource': 'things',
'aggregations': {
'count': aggregators.count('thing'),
},
'post_aggregations': {
'avg': (postaggregator.Field('sum') /
postaggregator.Field('count')),
},
'paging_spec': {
'pagingIdentifies': {},
'threshold': 1,
},
'filter': filters.Dimension('one') == 1,
'having': having.Aggregation('sum') > 1,
'new_key': 'value',
'virtualColumns':
[VirtualColumn(type='expression', name='foo', expression="concat('foo' + page)", outputType='STRING')]
})
# then
assert query.query_dict == expected_query_dict
def test_build_query(self):
# given
expected_query_dict = {
"queryType": None,
"dataSource": "things",
"aggregations": [{"fieldName": "thing", "name": "count", "type": "count"}],
"postAggregations": [
{
"fields": [
{"fieldName": "sum", "type": "fieldAccess"},
{"fieldName": "count", "type": "fieldAccess"},
],
"fn": "/",
"name": "avg",
"type": "arithmetic",
}
],
"pagingSpec": {"pagingIdentifies": {}, "threshold": 1},
"filter": {"dimension": "one", "type": "selector", "value": 1},
"having": {"aggregation": "sum", "type": "greaterThan", "value": 1},
"new_key": "value",
}
builder = QueryBuilder()
# when
query = builder.build_query(
None,
{
"datasource": "things",
"aggregations": {"count": aggregators.count("thing")},
"post_aggregations": {
"avg": (postaggregator.Field("sum") / postaggregator.Field("count"))
},
"paging_spec": {"pagingIdentifies": {}, "threshold": 1},
"filter": filters.Dimension("one") == 1,
"having": having.Aggregation("sum") > 1,
"new_key": "value",
},
)
# then
assert query.query_dict == expected_query_dict
def test_build_subquery(self):
# given
expected_query_dict = {
"query": {
"queryType": "groupBy",
"dataSource": "things",
"aggregations": [
{"fieldName": "thing", "name": "count", "type": "count"}
],
"postAggregations": [
{
"fields": [
{"fieldName": "sum", "type": "fieldAccess"},
{"fieldName": "count", "type": "fieldAccess"},
],
"fn": "/",
"name": "avg",
"type": "arithmetic",
}
],
"filter": {"dimension": "one", "type": "selector", "value": 1},
"having": {"aggregation": "sum", "type": "greaterThan", "value": 1},
},
"type": "query",
}
builder = QueryBuilder()
# when
subquery_dict = builder.subquery(
{
"datasource": "things",
"aggregations": {"count": aggregators.count("thing")},
"post_aggregations": {
"avg": (postaggregator.Field("sum") / postaggregator.Field("count"))
},
"filter": filters.Dimension("one") == 1,
"having": having.Aggregation("sum") > 1,
}
)
# then
assert subquery_dict == expected_query_dict
def druid_simple_groupby(dimensions, filter_list=[], filter_type="and",
datasource=settings.DRUID_SPRAYDAY_DATASOURCE):
"""
Inputs:
dimensions => list of dimensions to group by
filter_list => list of list of things to filter with e.g.
filter_list=[['target_area_id', operator.ne, 1],
['sprayable', operator.eq, "true"],
['dimension', operator, "value"]])
filter_type => type of Druid filter to perform
"""
query = PyDruid(get_druid_broker_url(), 'druid/v2')
params = dict(
datasource=datasource,
granularity='all',
intervals=settings.DRUID_INTERVAL,
limit_spec={
"type": "default",
"limit": 50000,
}
)
params['dimensions'] = dimensions
if filter_list:
fields = []
for this_filter in filter_list:
compare_dim = filters.Dimension(this_filter[0])
comparison_operator = this_filter[1] # e.g. operator.eq
compare_dim_value = this_filter[2]
fields.append(comparison_operator(compare_dim, compare_dim_value))
params['filter'] = filters.Filter(
type=filter_type,
fields=fields
)
try:
request = query.groupby(**params)
except OSError:
pass
else:
return request.result
return []
def test_build_subquery(self):
# given
expected_query_dict = {
'query': {
'queryType': 'groupBy',
'dataSource': 'things',
'aggregations': [{'fieldName': 'thing', 'name': 'count', 'type': 'count'}],
'postAggregations': [{
'fields': [{
'fieldName': 'sum', 'type': 'fieldAccess',
}, {
'fieldName': 'count', 'type': 'fieldAccess',
}],
'fn': '/',
'name': 'avg',
'type': 'arithmetic',
}],
'filter': {'dimension': 'one', 'type': 'selector', 'value': 1},
'having': {'aggregation': 'sum', 'type': 'greaterThan', 'value': 1},
},
'type': 'query'
}
builder = QueryBuilder()
# when
subquery_dict = builder.subquery({
'datasource': 'things',
'aggregations': {
'count': aggregators.count('thing'),
},
'post_aggregations': {
'avg': (postaggregator.Field('sum') /
postaggregator.Field('count')),
},
'filter': filters.Dimension('one') == 1,
'having': having.Aggregation('sum') > 1,
})
# then
assert subquery_dict == expected_query_dict
def get_druid_data(dimensions=None, filter_list=[], filter_type="and",
order_by=["target_area_name"],
datasource=settings.DRUID_SPRAYDAY_DATASOURCE):
"""
Runs a query against Druid, returns data with metrics
Inputs:
dimensions => list of dimensions to group by
filter_list => list of list of things to filter with e.g.
filter_list=[['target_area_id', operator.ne, 1],
['sprayable', operator.eq, "true"],
['dimension', operator, "value"]])
filter_type => type of Druid filter to perform,
order_by => field(s) to order the data by
"""
query = PyDruid(get_druid_broker_url(), 'druid/v2')
params = dict(
datasource=datasource,
granularity='all',
intervals=settings.DRUID_INTERVAL,
aggregations={
'num_not_sprayable': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('sprayable') == 'false']
),
aggregators.longsum('count')
),
'num_not_sprayed': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('sprayable') == 'true',
filters.Dimension('sprayed') ==
settings.MSPRAY_WAS_NOT_SPRAYED_VALUE]
),
aggregators.longsum('count')
),
'num_sprayed': aggregators.filtered(
filters.Dimension('sprayed') ==
settings.MSPRAY_WAS_SPRAYED_VALUE,
aggregators.longsum('count')
),
'num_new': aggregators.filtered(
filters.Dimension('is_new') == 'true',
aggregators.longsum('count')
),
'num_new_no_duplicates': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('is_duplicate') == 'false',
filters.Dimension('is_new') == 'true']
),
aggregators.longsum('count')
),
'num_duplicate': aggregators.filtered(
filters.Dimension('is_duplicate') == 'true',
aggregators.longsum('count')
),
'num_sprayed_no_duplicates': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('is_duplicate') == 'false',
filters.Dimension('sprayed') ==
settings.MSPRAY_WAS_SPRAYED_VALUE]
),
aggregators.longsum('count')
),
'num_not_sprayed_no_duplicates': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('is_duplicate') == 'false',
filters.Dimension('sprayable') == 'true',
filters.Dimension('sprayed') ==
settings.MSPRAY_WAS_NOT_SPRAYED_VALUE]
),
aggregators.longsum('count')
),
'num_sprayed_duplicates': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('is_duplicate') == 'true',
filters.Dimension('sprayable') == 'true',
filters.Dimension('sprayed') ==
settings.MSPRAY_WAS_SPRAYED_VALUE]
),
aggregators.longsum('count')
),
'num_not_sprayable_no_duplicates': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('is_duplicate') == 'false',
filters.Dimension('sprayable') == 'false']
),
aggregators.longsum('count')
),
'num_refused': aggregators.filtered(
filters.Filter(
type='and',
fields=[filters.Dimension('is_duplicate') == 'false',
filters.Dimension('is_refused') == 'true',
filters.Dimension('sprayed') ==
settings.MSPRAY_WAS_NOT_SPRAYED_VALUE]
),
aggregators.longsum('count')
),
},
post_aggregations={
'num_found': Field('num_sprayed_no_duplicates') +
Field('num_sprayed_duplicates') +
Field('num_not_sprayed_no_duplicates')
},
limit_spec={
"type": "default",
"limit": 50000,
"columns": order_by
}
)
if filter_list:
fields = []
for this_filter in filter_list:
compare_dim = filters.Dimension(this_filter[0])
comparison_operator = this_filter[1] # e.g. operator.eq
compare_dim_value = this_filter[2]
fields.append(comparison_operator(compare_dim, compare_dim_value))
params['filter'] = filters.Filter(
type=filter_type,
fields=fields
)
if dimensions is None:
params['dimensions'] = ['target_area_id', 'target_area_name',
'target_area_structures']
else:
params['dimensions'] = dimensions
try:
request = query.groupby(**params)
except OSError:
return []
else:
return request.result
def test_dimension(self):
d = filters.Dimension('dim')
actual = filters.Filter.build_filter(d == 'val')
expected = {'type': 'selector', 'dimension': 'dim', 'value': 'val'}
assert actual == expected
def test_dimension(self):
d = filters.Dimension("dim")
actual = filters.Filter.build_filter(d == "val")
expected = {"type": "selector", "dimension": "dim", "value": "val"}
assert actual == expected
