def append_query(self, es_query, start):
self.start = start
parts = self.edge.domain.partitions
filters = []
notty = []
for p in parts:
w = p.where
filters.append(AndOp("and", [w] + notty).to_esfilter(self.schema))
notty.append(NotOp("not", w))
missing_filter = None
if self.edge.allowNulls: # TODO: Use Expression.missing().esfilter() TO GET OPTIMIZED FILTER
missing_filter = set_default(
{"filter": AndOp("and", notty).to_esfilter(self.schema)},
es_query
)
return wrap({"aggs": {
"_match": set_default(
{"filters": {"filters": filters}},
es_query
),
"_missing": missing_filter
}})
def append_query(self, es_query, start):
self.start = start
domain = self.domain
domain_key = domain.key
include, text_include = transpose(*(
(
float(v) if isinstance(v, (int, float)) else v,
text_type(float(v)) if isinstance(v, (int, float)) else v
)
for v in (p[domain_key] for p in domain.partitions)
))
value = self.edge.value
exists = AndOp("and", [
value.exists(),
InOp("in", [value, Literal("literal", include)])
]).partial_eval()
limit = coalesce(self.limit, len(domain.partitions))
if isinstance(value, Variable):
es_field = self.query.frum.schema.leaves(value.var)[0].es_column # ALREADY CHECKED THERE IS ONLY ONE
terms = set_default({"terms": {
"field": es_field,
"size": limit,
"order": {"_term": self.sorted} if self.sorted else None
}}, es_query)
else:
terms = set_default({"terms": {
"script": {
"lang": "painless",
"inline": value.to_es_script(self.schema).script(self.schema)
},
"size": limit
}}, es_query)
if self.edge.allowNulls:
missing = set_default(
{"filter": NotOp("not", exists).to_esfilter(self.schema)},
es_query
)
else:
missing = None
return wrap({"aggs": {
"_match": {
"filter": exists.to_esfilter(self.schema),
"aggs": {
"_filter": terms
}
},
"_missing": missing
}})
def append_query(self, es_query, start):
self.start = start
domain = self.domain
domain_key = domain.key
include, text_include = zip(*(
(
float(v) if isinstance(v, (int, float)) else v,
text_type(float(v)) if isinstance(v, (int, float)) else v
)
for v in (p[domain_key] for p in domain.partitions)
))
value = self.edge.value
exists = AndOp("and", [
value.exists(),
InOp("in", [value, Literal("literal", include)])
]).partial_eval()
limit = coalesce(self.limit, len(domain.partitions))
if isinstance(value, Variable):
es_field = self.query.frum.schema.leaves(value.var)[0].es_column # ALREADY CHECKED THERE IS ONLY ONE
terms = set_default({"terms": {
"field": es_field,
"size": limit,
"order": {"_term": self.sorted} if self.sorted else None
}}, es_query)
else:
terms = set_default({"terms": {
"script": {
"lang": "painless",
"inline": value.to_painless(self.schema).script(self.schema)
},
"size": limit
}}, es_query)
if self.edge.allowNulls:
missing = set_default(
{"filter": NotOp("not", exists).to_esfilter(self.schema)},
es_query
)
else:
missing = None
return wrap({"aggs": {
"_match": {
"filter": exists.to_esfilter(self.schema),
"aggs": {
"_filter": terms
}
},
"_missing": missing
}})
def append_query(self, query_path, es_query):
parts = self.edge.domain.partitions
filters = []
notty = []
for p in parts:
w = p.where
filters.append(AndOp([w] + notty))
notty.append(NotOp(w))
output = Aggs().add(FiltersAggs("_match", filters, self).add(es_query))
if self.edge.allowNulls: # TODO: Use Expression.missing().esfilter() TO GET OPTIMIZED FILTER
output.add(FilterAggs("_missing", AndOp(notty), self).add(es_query))
return output
def _range_composer(edge, domain, es_query, to_float, schema):
# USE RANGES
_min = coalesce(domain.min, MIN(domain.partitions.min))
_max = coalesce(domain.max, MAX(domain.partitions.max))
if edge.allowNulls:
missing_filter = set_default(
{
"filter": NotOp("not", AndOp("and", [
edge.value.exists(),
InequalityOp("gte", [edge.value, Literal(None, to_float(_min))]),
InequalityOp("lt", [edge.value, Literal(None, to_float(_max))])
]).partial_eval()).to_esfilter(schema)
},
es_query
)
else:
missing_filter = None
if isinstance(edge.value, Variable):
calc = {"field": schema.leaves(edge.value.var)[0].es_column}
else:
calc = {"script": edge.value.to_painless(schema).script(schema)}
return wrap({"aggs": {
"_match": set_default(
{"range": calc},
{"range": {"ranges": [{"from": to_float(p.min), "to": to_float(p.max)} for p in domain.partitions]}},
es_query
),
"_missing": missing_filter
}})
def _range_composer(self, edge, domain, es_query, to_float, schema):
# USE RANGES
_min = coalesce(domain.min, MIN(domain.partitions.min))
_max = coalesce(domain.max, MAX(domain.partitions.max))
output = Aggs()
if edge.allowNulls:
output.add(
FilterAggs(
"_missing",
NotOp(
AndOp([
edge.value.exists(),
GteOp([edge.value, Literal(to_float(_min))]),
LtOp([edge.value, Literal(to_float(_max))])
]).partial_eval()), self).add(es_query))
if is_op(edge.value, Variable):
calc = {"field": first(schema.leaves(edge.value.var)).es_column}
else:
calc = {"script": text_type(Painless[edge.value].to_es_script(schema))}
calc['ranges'] = [{
"from": to_float(p.min),
"to": to_float(p.max)
} for p in domain.partitions]
return output.add(RangeAggs("_match", calc, self).add(es_query))
def build_es_query(select, query_path, schema, query):
acc = extract_aggs(select, query_path, schema)
acc = NestedAggs(query_path).add(acc)
split_decoders = get_decoders_by_path(query)
split_wheres = split_expression_by_path(query.where,
schema=schema,
lang=ES52)
start = 0
decoders = [None] * (len(query.edges) + len(query.groupby))
paths = list(
reversed(sorted(set(split_wheres.keys())
| set(split_decoders.keys()))))
for path in paths:
decoder = split_decoders.get(path, Null)
where = split_wheres.get(path, Null)
for d in decoder:
decoders[d.edge.dim] = d
acc = d.append_query(path, acc)
start += d.num_columns
if where:
acc = FilterAggs("_filter", AndOp(where), None).add(acc)
acc = NestedAggs(path).add(acc)
acc = NestedAggs('.').add(acc)
acc = simplify(acc)
es_query = wrap(acc.to_es(schema))
es_query.size = 0
return acc, decoders, es_query
def append_query(self, query_path, es_query):
domain = self.domain
domain_key = domain.key
value = self.edge.value
cnv = pull_functions[value.type]
include = tuple(cnv(p[domain_key]) for p in domain.partitions)
exists = AndOp("and", [
InOp("in", [value, Literal("literal", include)])
]).partial_eval()
limit = coalesce(self.limit, len(domain.partitions))
if isinstance(value, Variable):
es_field = first(self.query.frum.schema.leaves(value.var)).es_column # ALREADY CHECKED THERE IS ONLY ONE
match = TermsAggs(
"_match",
{
"field": es_field,
"size": limit,
"order": {"_term": self.sorted} if self.sorted else None
},
self
)
else:
match = TermsAggs(
"_match",
{
"script": {
"lang": "painless",
"inline": value.to_es_script(self.schema).script(self.schema)
},
"size": limit
},
self
)
output = Aggs().add(FilterAggs("_filter", exists, None).add(match.add(es_query)))
if self.edge.allowNulls:
# FIND NULLS AT EACH NESTED LEVEL
for p in self.schema.query_path:
if p == query_path:
# MISSING AT THE QUERY DEPTH
output.add(
NestedAggs(p).add(FilterAggs("_missing0", NotOp(None, exists), self).add(es_query))
)
else:
# PARENT HAS NO CHILDREN, SO MISSING
column = first(self.schema.values(query_path, (OBJECT, EXISTS)))
output.add(
NestedAggs(column.nested_path[0]).add(
FilterAggs(
"_missing1",
NotOp(None, ExistsOp(None, Variable(column.es_column.replace(NESTED_TYPE, EXISTS_TYPE)))),
self
).add(es_query)
)
)
return output
def append_query(self, es_query, start):
self.start = start
edge = self.edge
range = edge.range
domain = edge.domain
aggs = {}
for i, p in enumerate(domain.partitions):
filter_ = AndOp("and", [
InequalityOp("lte", [range.min, Literal("literal", self.to_float(p.min))]),
InequalityOp("gt", [range.max, Literal("literal", self.to_float(p.min))])
])
aggs["_join_" + text_type(i)] = set_default(
{"filter": filter_.to_esfilter(self.schema)},
es_query
)
return wrap({"aggs": aggs})
def append_query(self, es_query, start):
self.start = start
edge = self.edge
range = edge.range
domain = edge.domain
aggs = {}
for i, p in enumerate(domain.partitions):
filter_ = AndOp("and", [
InequalityOp("lte", [range.min, Literal("literal", self.to_float(p.min))]),
InequalityOp("gt", [range.max, Literal("literal", self.to_float(p.min))])
])
aggs["_join_" + text_type(i)] = set_default(
{"filter": filter_.to_esfilter(self.schema)},
es_query
)
return wrap({"aggs": aggs})
def append_query(self, query_path, es_query):
edge = self.edge
range = edge.range
domain = edge.domain
aggs = Aggs()
for i, p in enumerate(domain.partitions):
filter_ = AndOp([
LteOp([range.min, Literal(self.to_float(p.min))]),
GtOp([range.max, Literal(self.to_float(p.min))])
])
aggs.add(FilterAggs("_match" + text(i), filter_, self).add(es_query))
return aggs
def append_query(self, query_path, es_query):
edge = self.edge
range = edge.range
domain = edge.domain
aggs = Aggs()
for i, p in enumerate(domain.partitions):
filter_ = AndOp("and", [
InequalityOp("lte", [range.min, Literal("literal", self.to_float(p.min))]),
InequalityOp("gt", [range.max, Literal("literal", self.to_float(p.min))])
])
aggs.add(FilterAggs("_match" + text_type(i), filter_, self).add(es_query))
return aggs
def es_query_proto(path, selects, wheres, schema):
"""
RETURN TEMPLATE AND PATH-TO-FILTER AS A 2-TUPLE
:param path: THE NESTED PATH (NOT INCLUDING TABLE NAME)
:param wheres: MAP FROM path TO LIST OF WHERE CONDITIONS
:return: (es_query, filters_map) TUPLE
"""
output = None
last_where = MATCH_ALL
for p in reversed(sorted(wheres.keys() | set(selects.keys()))):
where = wheres.get(p)
select = selects.get(p)
if where:
where = AndOp(where).partial_eval().to_esfilter(schema)
if output:
where = es_or([es_and([output, where]), where])
else:
if output:
if last_where is MATCH_ALL:
where = es_or([output, MATCH_ALL])
else:
where = output
else:
where = MATCH_ALL
if p == ".":
output = set_default(
{
"from": 0,
"size": 0,
"sort": [],
"query": where
}, select.to_es())
else:
output = {
"nested": {
"path":
p,
"inner_hits":
set_default({"size": 100000}, select.to_es())
if select else None,
"query":
where
}
}
last_where = where
return output
def append_query(self, query_path, es_query):
domain = self.domain
domain_key = domain.key
value = Painless[self.edge.value]
cnv = pull_functions[value.type]
include = tuple(cnv(p[domain_key]) for p in domain.partitions)
schema = self.schema
exists = Painless[AndOp([InOp([value,
Literal(include)])])].partial_eval()
limit = coalesce(self.limit, len(domain.partitions))
if is_op(value, Variable):
es_field = first(schema.leaves(
value.var)).es_column # ALREADY CHECKED THERE IS ONLY ONE
match = TermsAggs(
"_match", {
"field": es_field,
"size": limit,
"order": {
"_term": self.sorted
} if self.sorted else None
}, self)
else:
match = TermsAggs("_match", {
"script": text(value.to_es_script(schema)),
"size": limit
}, self)
output = Aggs().add(
FilterAggs("_filter", exists, None).add(match.add(es_query)))
if self.edge.allowNulls:
# IF ALL NESTED COLUMNS ARE NULL, DOES THE FILTER PASS?
# MISSING AT THE QUERY DEPTH
op, split = split_expression_by_path(NotOp(exists), schema)
for i, p in enumerate(reversed(sorted(split.keys()))):
e = split.get(p)
if e:
not_match = NestedAggs(p).add(
FilterAggs("_missing" + text(i), e,
self).add(es_query))
output.add(not_match)
return output
def _range_composer(self, edge, domain, es_query, to_float, schema):
# USE RANGES
_min = coalesce(domain.min, MIN(domain.partitions.min))
_max = coalesce(domain.max, MAX(domain.partitions.max))
output = Aggs()
if edge.allowNulls:
output.add(FilterAggs(
"_missing",
NotOp("not", AndOp("and", [
edge.value.exists(),
InequalityOp("gte", [edge.value, Literal(None, to_float(_min))]),
InequalityOp("lt", [edge.value, Literal(None, to_float(_max))])
]).partial_eval()),
self
).add(es_query))
if isinstance(edge.value, Variable):
calc = {"field": first(schema.leaves(edge.value.var)).es_column}
else:
calc = {"script": edge.value.to_es_script(schema).script(schema)}
calc['ranges'] = [{"from": to_float(p.min), "to": to_float(p.max)} for p in domain.partitions]
return output.add(RangeAggs("_match", calc, self).add(es_query))
def es_deepop(es, query):
schema = query.frum.schema
query_path = schema.query_path[0]
# TODO: FIX THE GREAT SADNESS CAUSED BY EXECUTING post_expressions
# THE EXPRESSIONS SHOULD BE PUSHED TO THE CONTAINER: ES ALLOWS
# {"inner_hit":{"script_fields":[{"script":""}...]}}, BUT THEN YOU
# LOOSE "_source" BUT GAIN "fields", FORCING ALL FIELDS TO BE EXPLICIT
post_expressions = {}
es_query, es_filters = es_query_template(query_path)
# SPLIT WHERE CLAUSE BY DEPTH
wheres = split_expression_by_depth(query.where, schema)
for f, w in zip_longest(es_filters, wheres):
script = ES52[AndOp(w)].partial_eval().to_esfilter(schema)
set_default(f, script)
if not wheres[1]:
# INCLUDE DOCS WITH NO NESTED DOCS
more_filter = {
"bool": {
"filter": [AndOp(wheres[0]).partial_eval().to_esfilter(schema)],
"must_not": {
"nested": {
"path": query_path,
"query": MATCH_ALL
}
}
}
}
else:
more_filter = None
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
map_to_es_columns = schema.map_to_es()
query_for_es = query.map(map_to_es_columns)
es_query.sort = jx_sort_to_es_sort(query_for_es.sort, schema)
es_query.stored_fields = []
is_list = is_list_(query.select)
selects = wrap([unwrap(s.copy()) for s in listwrap(query.select)])
new_select = FlatList()
put_index = 0
for select in selects:
if is_op(select.value, LeavesOp) and is_op(select.value.term, Variable):
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
leaves = schema.leaves(select.value.term.var)
col_names = set()
for c in leaves:
if c.nested_path[0] == ".":
if c.jx_type == NESTED:
continue
es_query.stored_fields += [c.es_column]
c_name = untype_path(relative_field(c.name, query_path))
col_names.add(c_name)
new_select.append({
"name": concat_field(select.name, c_name),
"nested_path": c.nested_path[0],
"put": {"name": concat_field(select.name, literal_field(c_name)), "index": put_index, "child": "."},
"pull": get_pull_function(c)
})
put_index += 1
# REMOVE DOTS IN PREFIX IF NAME NOT AMBIGUOUS
for n in new_select:
if n.name.startswith("..") and n.name.lstrip(".") not in col_names:
n.put.name = n.name = n.name.lstrip(".")
col_names.add(n.name)
elif is_op(select.value, Variable):
net_columns = schema.leaves(select.value.var)
if not net_columns:
new_select.append({
"name": select.name,
"nested_path": ".",
"put": {"name": select.name, "index": put_index, "child": "."},
"pull": NULL
})
else:
for n in net_columns:
pull = get_pull_function(n)
if n.nested_path[0] == ".":
if n.jx_type == NESTED:
continue
es_query.stored_fields += [n.es_column]
# WE MUST FIGURE OUT WHICH NAMESSPACE s.value.var IS USING SO WE CAN EXTRACT THE child
for np in n.nested_path:
c_name = untype_path(relative_field(n.name, np))
if startswith_field(c_name, select.value.var):
# PREFER THE MOST-RELATIVE NAME
child = relative_field(c_name, select.value.var)
break
else:
continue
new_select.append({
"name": select.name,
"pull": pull,
"nested_path": n.nested_path[0],
"put": {
"name": select.name,
"index": put_index,
"child": child
}
})
put_index += 1
else:
expr = select.value
for v in expr.vars():
for c in schema[v.var]:
if c.nested_path[0] == ".":
es_query.stored_fields += [c.es_column]
# else:
# Log.error("deep field not expected")
pull_name = EXPRESSION_PREFIX + select.name
map_to_local = MapToLocal(schema)
pull = jx_expression_to_function(pull_name)
post_expressions[pull_name] = jx_expression_to_function(expr.map(map_to_local))
new_select.append({
"name": select.name if is_list else ".",
"pull": pull,
"value": expr.__data__(),
"put": {"name": select.name, "index": put_index, "child": "."}
})
put_index += 1
es_query.stored_fields = sorted(es_query.stored_fields)
# <COMPLICATED> ES needs two calls to get all documents
more = []
def get_more(please_stop):
more.append(es_post(
es,
Data(
query=more_filter,
stored_fields=es_query.stored_fields
),
query.limit
))
if more_filter:
need_more = Thread.run("get more", target=get_more)
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
# EACH A HIT IS RETURNED MULTIPLE TIMES FOR EACH INNER HIT, WITH INNER HIT INCLUDED
def inners():
for t in data.hits.hits:
for i in t.inner_hits[literal_field(query_path)].hits.hits:
t._inner = i._source
for k, e in post_expressions.items():
t[k] = e(t)
yield t
if more_filter:
Thread.join(need_more)
for t in more[0].hits.hits:
yield t
# </COMPLICATED>
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(inners(), new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def es_aggsop(es, frum, query):
query = query.copy() # WE WILL MARK UP THIS QUERY
schema = frum.schema
select = listwrap(query.select)
es_query = Data()
new_select = Data() # MAP FROM canonical_name (USED FOR NAMES IN QUERY) TO SELECT MAPPING
formula = []
for s in select:
if s.aggregate == "count" and isinstance(s.value, Variable) and s.value.var == ".":
if schema.query_path == ".":
s.pull = jx_expression_to_function("doc_count")
else:
s.pull = jx_expression_to_function({"coalesce": ["_nested.doc_count", "doc_count", 0]})
elif isinstance(s.value, Variable):
if s.aggregate == "count":
new_select["count_"+literal_field(s.value.var)] += [s]
else:
new_select[literal_field(s.value.var)] += [s]
elif s.aggregate:
formula.append(s)
for canonical_name, many in new_select.items():
for s in many:
columns = frum.schema.values(s.value.var)
if s.aggregate == "count":
canonical_names = []
for column in columns:
cn = literal_field(column.es_column + "_count")
if column.jx_type == EXISTS:
canonical_names.append(cn + ".doc_count")
es_query.aggs[cn].filter.range = {column.es_column: {"gt": 0}}
else:
canonical_names.append(cn+ ".value")
es_query.aggs[cn].value_count.field = column.es_column
if len(canonical_names) == 1:
s.pull = jx_expression_to_function(canonical_names[0])
else:
s.pull = jx_expression_to_function({"add": canonical_names})
elif s.aggregate == "median":
if len(columns) > 1:
Log.error("Do not know how to count columns with more than one type (script probably)")
# ES USES DIFFERENT METHOD FOR PERCENTILES
key = literal_field(canonical_name + " percentile")
es_query.aggs[key].percentiles.field = columns[0].es_column
es_query.aggs[key].percentiles.percents += [50]
s.pull = jx_expression_to_function(key + ".values.50\\.0")
elif s.aggregate == "percentile":
if len(columns) > 1:
Log.error("Do not know how to count columns with more than one type (script probably)")
# ES USES DIFFERENT METHOD FOR PERCENTILES
key = literal_field(canonical_name + " percentile")
if isinstance(s.percentile, text_type) or s.percetile < 0 or 1 < s.percentile:
Log.error("Expecting percentile to be a float from 0.0 to 1.0")
percent = Math.round(s.percentile * 100, decimal=6)
es_query.aggs[key].percentiles.field = columns[0].es_column
es_query.aggs[key].percentiles.percents += [percent]
s.pull = jx_expression_to_function(key + ".values." + literal_field(text_type(percent)))
elif s.aggregate == "cardinality":
canonical_names = []
for column in columns:
cn = literal_field(column.es_column + "_cardinality")
canonical_names.append(cn)
es_query.aggs[cn].cardinality.field = column.es_column
if len(columns) == 1:
s.pull = jx_expression_to_function(canonical_names[0] + ".value")
else:
s.pull = jx_expression_to_function({"add": [cn + ".value" for cn in canonical_names], "default": 0})
elif s.aggregate == "stats":
if len(columns) > 1:
Log.error("Do not know how to count columns with more than one type (script probably)")
# REGULAR STATS
stats_name = literal_field(canonical_name)
es_query.aggs[stats_name].extended_stats.field = columns[0].es_column
# GET MEDIAN TOO!
median_name = literal_field(canonical_name + "_percentile")
es_query.aggs[median_name].percentiles.field = columns[0].es_column
es_query.aggs[median_name].percentiles.percents += [50]
s.pull = get_pull_stats(stats_name, median_name)
elif s.aggregate == "union":
pulls = []
for column in columns:
script = {"scripted_metric": {
'init_script': 'params._agg.terms = new HashSet()',
'map_script': 'for (v in doc['+quote(column.es_column)+'].values) params._agg.terms.add(v)',
'combine_script': 'return params._agg.terms.toArray()',
'reduce_script': 'HashSet output = new HashSet(); for (a in params._aggs) { if (a!=null) for (v in a) {output.add(v)} } return output.toArray()',
}}
stats_name = encode_property(column.es_column)
if column.nested_path[0] == ".":
es_query.aggs[stats_name] = script
pulls.append(jx_expression_to_function(stats_name + ".value"))
else:
es_query.aggs[stats_name] = {
"nested": {"path": column.nested_path[0]},
"aggs": {"_nested": script}
}
pulls.append(jx_expression_to_function(stats_name + "._nested.value"))
if len(pulls) == 0:
s.pull = NULL
elif len(pulls) == 1:
s.pull = pulls[0]
else:
s.pull = lambda row: UNION(p(row) for p in pulls)
else:
if len(columns) > 1:
Log.error("Do not know how to count columns with more than one type (script probably)")
elif len(columns) <1:
# PULL VALUE OUT OF THE stats AGGREGATE
s.pull = jx_expression_to_function({"null":{}})
else:
# PULL VALUE OUT OF THE stats AGGREGATE
es_query.aggs[literal_field(canonical_name)].extended_stats.field = columns[0].es_column
s.pull = jx_expression_to_function({"coalesce": [literal_field(canonical_name) + "." + aggregates[s.aggregate], s.default]})
for i, s in enumerate(formula):
canonical_name = literal_field(s.name)
if isinstance(s.value, TupleOp):
if s.aggregate == "count":
# TUPLES ALWAYS EXIST, SO COUNTING THEM IS EASY
s.pull = "doc_count"
elif s.aggregate in ('max', 'maximum', 'min', 'minimum'):
if s.aggregate in ('max', 'maximum'):
dir = 1
op = "max"
else:
dir = -1
op = 'min'
nully = TupleOp("tuple", [NULL]*len(s.value.terms)).partial_eval().to_es_script(schema).expr
selfy = s.value.partial_eval().to_es_script(schema).expr
script = {"scripted_metric": {
'init_script': 'params._agg.best = ' + nully + ';',
'map_script': 'params._agg.best = ' + expand_template(MAX_OF_TUPLE, {"expr1": "params._agg.best", "expr2": selfy, "dir": dir, "op": op}) + ";",
'combine_script': 'return params._agg.best',
'reduce_script': 'return params._aggs.stream().max(' + expand_template(COMPARE_TUPLE, {"dir": dir, "op": op}) + ').get()',
}}
if schema.query_path[0] == ".":
es_query.aggs[canonical_name] = script
s.pull = jx_expression_to_function(literal_field(canonical_name) + ".value")
else:
es_query.aggs[canonical_name] = {
"nested": {"path": schema.query_path[0]},
"aggs": {"_nested": script}
}
s.pull = jx_expression_to_function(literal_field(canonical_name) + "._nested.value")
else:
Log.error("{{agg}} is not a supported aggregate over a tuple", agg=s.aggregate)
elif s.aggregate == "count":
es_query.aggs[literal_field(canonical_name)].value_count.script = s.value.partial_eval().to_es_script(schema).script(schema)
s.pull = jx_expression_to_function(literal_field(canonical_name) + ".value")
elif s.aggregate == "median":
# ES USES DIFFERENT METHOD FOR PERCENTILES THAN FOR STATS AND COUNT
key = literal_field(canonical_name + " percentile")
es_query.aggs[key].percentiles.script = s.value.to_es_script(schema).script(schema)
es_query.aggs[key].percentiles.percents += [50]
s.pull = jx_expression_to_function(key + ".values.50\\.0")
elif s.aggregate == "percentile":
# ES USES DIFFERENT METHOD FOR PERCENTILES THAN FOR STATS AND COUNT
key = literal_field(canonical_name + " percentile")
percent = Math.round(s.percentile * 100, decimal=6)
es_query.aggs[key].percentiles.script = s.value.to_es_script(schema).script(schema)
es_query.aggs[key].percentiles.percents += [percent]
s.pull = jx_expression_to_function(key + ".values." + literal_field(text_type(percent)))
elif s.aggregate == "cardinality":
# ES USES DIFFERENT METHOD FOR CARDINALITY
key = canonical_name + " cardinality"
es_query.aggs[key].cardinality.script = s.value.to_es_script(schema).script(schema)
s.pull = jx_expression_to_function(key + ".value")
elif s.aggregate == "stats":
# REGULAR STATS
stats_name = literal_field(canonical_name)
es_query.aggs[stats_name].extended_stats.script = s.value.to_es_script(schema).script(schema)
# GET MEDIAN TOO!
median_name = literal_field(canonical_name + " percentile")
es_query.aggs[median_name].percentiles.script = s.value.to_es_script(schema).script(schema)
es_query.aggs[median_name].percentiles.percents += [50]
s.pull = get_pull_stats(stats_name, median_name)
elif s.aggregate == "union":
# USE TERMS AGGREGATE TO SIMULATE union
stats_name = literal_field(canonical_name)
es_query.aggs[stats_name].terms.script_field = s.value.to_es_script(schema).script(schema)
s.pull = jx_expression_to_function(stats_name + ".buckets.key")
else:
# PULL VALUE OUT OF THE stats AGGREGATE
s.pull = jx_expression_to_function(canonical_name + "." + aggregates[s.aggregate])
es_query.aggs[canonical_name].extended_stats.script = s.value.to_es_script(schema).script(schema)
decoders = get_decoders_by_depth(query)
start = 0
#<TERRIBLE SECTION> THIS IS WHERE WE WEAVE THE where CLAUSE WITH nested
split_where = split_expression_by_depth(query.where, schema=frum.schema)
if len(split_field(frum.name)) > 1:
if any(split_where[2::]):
Log.error("Where clause is too deep")
for d in decoders[1]:
es_query = d.append_query(es_query, start)
start += d.num_columns
if split_where[1]:
#TODO: INCLUDE FILTERS ON EDGES
filter_ = AndOp("and", split_where[1]).to_esfilter(schema)
es_query = Data(
aggs={"_filter": set_default({"filter": filter_}, es_query)}
)
es_query = wrap({
"aggs": {"_nested": set_default(
{"nested": {"path": schema.query_path[0]}},
es_query
)}
})
else:
if any(split_where[1::]):
Log.error("Where clause is too deep")
if decoders:
for d in jx.reverse(decoders[0]):
es_query = d.append_query(es_query, start)
start += d.num_columns
if split_where[0]:
#TODO: INCLUDE FILTERS ON EDGES
filter = AndOp("and", split_where[0]).to_esfilter(schema)
es_query = Data(
aggs={"_filter": set_default({"filter": filter}, es_query)}
)
# </TERRIBLE SECTION>
if not es_query:
es_query = wrap({"query": {"match_all": {}}})
es_query.size = 0
with Timer("ES query time") as es_duration:
result = es_post(es, es_query, query.limit)
try:
format_time = Timer("formatting")
with format_time:
decoders = [d for ds in decoders for d in ds]
result.aggregations.doc_count = coalesce(result.aggregations.doc_count, result.hits.total) # IT APPEARS THE OLD doc_count IS GONE
formatter, groupby_formatter, aggop_formatter, mime_type = format_dispatch[query.format]
if query.edges:
output = formatter(decoders, result.aggregations, start, query, select)
elif query.groupby:
output = groupby_formatter(decoders, result.aggregations, start, query, select)
else:
output = aggop_formatter(decoders, result.aggregations, start, query, select)
output.meta.timing.formatting = format_time.duration
output.meta.timing.es_search = es_duration.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
if query.format not in format_dispatch:
Log.error("Format {{format|quote}} not supported yet", format=query.format, cause=e)
Log.error("Some problem", cause=e)
def es_aggsop(es, frum, query):
query = query.copy() # WE WILL MARK UP THIS QUERY
schema = frum.schema
query_path = schema.query_path[0]
select = listwrap(query.select)
new_select = Data(
) # MAP FROM canonical_name (USED FOR NAMES IN QUERY) TO SELECT MAPPING
formula = []
for s in select:
if is_op(s.value, Variable_):
s.query_path = query_path
if s.aggregate == "count":
new_select["count_" + literal_field(s.value.var)] += [s]
else:
new_select[literal_field(s.value.var)] += [s]
elif s.aggregate:
split_select = split_expression_by_path(s.value,
schema,
lang=Painless)
for si_key, si_value in split_select.items():
if si_value:
if s.query_path:
Log.error(
"can not handle more than one depth per select")
s.query_path = si_key
formula.append(s)
acc = Aggs()
for _, many in new_select.items():
for s in many:
canonical_name = s.name
if s.aggregate in ("value_count", "count"):
columns = frum.schema.values(s.value.var,
exclude_type=(OBJECT, NESTED))
else:
columns = frum.schema.values(s.value.var)
if s.aggregate == "count":
canonical_names = []
for column in columns:
es_name = column.es_column + "_count"
if column.jx_type == EXISTS:
if column.nested_path[0] == query_path:
canonical_names.append("doc_count")
acc.add(
NestedAggs(column.nested_path[0]).add(
CountAggs(s)))
else:
canonical_names.append("value")
acc.add(
NestedAggs(column.nested_path[0]).add(
ExprAggs(es_name, {
"value_count": {
"field": column.es_column
}
}, s)))
if len(canonical_names) == 1:
s.pull = jx_expression_to_function(canonical_names[0])
else:
s.pull = jx_expression_to_function(
{"add": canonical_names})
elif s.aggregate == "median":
columns = [
c for c in columns if c.jx_type in (NUMBER, INTEGER)
]
if len(columns) != 1:
Log.error(
"Do not know how to perform median on columns with more than one type (script probably)"
)
# ES USES DIFFERENT METHOD FOR PERCENTILES
key = canonical_name + " percentile"
acc.add(
ExprAggs(
key, {
"percentiles": {
"field": first(columns).es_column,
"percents": [50]
}
}, s))
s.pull = jx_expression_to_function("values.50\\.0")
elif s.aggregate == "percentile":
columns = [
c for c in columns if c.jx_type in (NUMBER, INTEGER)
]
if len(columns) != 1:
Log.error(
"Do not know how to perform percentile on columns with more than one type (script probably)"
)
# ES USES DIFFERENT METHOD FOR PERCENTILES
key = canonical_name + " percentile"
if is_text(
s.percentile) or s.percetile < 0 or 1 < s.percentile:
Log.error(
"Expecting percentile to be a float from 0.0 to 1.0")
percent = mo_math.round(s.percentile * 100, decimal=6)
acc.add(
ExprAggs(
key, {
"percentiles": {
"field": first(columns).es_column,
"percents": [percent],
"tdigest": {
"compression": 2
}
}
}, s))
s.pull = jx_expression_to_function(
join_field(["values", text_type(percent)]))
elif s.aggregate == "cardinality":
for column in columns:
path = column.es_column + "_cardinality"
acc.add(
ExprAggs(path,
{"cardinality": {
"field": column.es_column
}}, s))
s.pull = jx_expression_to_function("value")
elif s.aggregate == "stats":
columns = [
c for c in columns if c.jx_type in (NUMBER, INTEGER)
]
if len(columns) != 1:
Log.error(
"Do not know how to perform stats on columns with more than one type (script probably)"
)
# REGULAR STATS
acc.add(
ExprAggs(canonical_name, {
"extended_stats": {
"field": first(columns).es_column
}
}, s))
s.pull = get_pull_stats()
# GET MEDIAN TOO!
select_median = s.copy()
select_median.pull = jx_expression_to_function(
{"select": [{
"name": "median",
"value": "values.50\\.0"
}]})
acc.add(
ExprAggs(
canonical_name + "_percentile", {
"percentiles": {
"field": first(columns).es_column,
"percents": [50]
}
}, select_median))
elif s.aggregate == "union":
for column in columns:
script = {
"scripted_metric": {
'init_script':
'params._agg.terms = new HashSet()',
'map_script':
'for (v in doc[' + quote(column.es_column) +
'].values) params._agg.terms.add(v);',
'combine_script':
'return params._agg.terms.toArray()',
'reduce_script':
'HashSet output = new HashSet(); for (a in params._aggs) { if (a!=null) for (v in a) {output.add(v)} } return output.toArray()',
}
}
stats_name = column.es_column
acc.add(
NestedAggs(column.nested_path[0]).add(
ExprAggs(stats_name, script, s)))
s.pull = jx_expression_to_function("value")
elif s.aggregate == "count_values":
# RETURN MAP FROM VALUE TO THE NUMBER OF TIMES FOUND IN THE DOCUMENTS
# NOT A NESTED DOC, RATHER A MULTIVALUE FIELD
for column in columns:
script = {
"scripted_metric": {
'params': {
"_agg": {}
},
'init_script':
'params._agg.terms = new HashMap()',
'map_script':
'for (v in doc[' + quote(column.es_column) +
'].values) params._agg.terms.put(v, Optional.ofNullable(params._agg.terms.get(v)).orElse(0)+1);',
'combine_script':
'return params._agg.terms',
'reduce_script':
'''
HashMap output = new HashMap();
for (agg in params._aggs) {
if (agg!=null){
for (e in agg.entrySet()) {
String key = String.valueOf(e.getKey());
output.put(key, e.getValue() + Optional.ofNullable(output.get(key)).orElse(0));
}
}
}
return output;
'''
}
}
stats_name = encode_property(column.es_column)
acc.add(
NestedAggs(column.nested_path[0]).add(
ExprAggs(stats_name, script, s)))
s.pull = jx_expression_to_function("value")
else:
if not columns:
s.pull = jx_expression_to_function(NULL)
else:
for c in columns:
acc.add(
NestedAggs(c.nested_path[0]).add(
ExprAggs(
canonical_name,
{"extended_stats": {
"field": c.es_column
}}, s)))
s.pull = jx_expression_to_function(aggregates[s.aggregate])
for i, s in enumerate(formula):
s_path = [
k for k, v in split_expression_by_path(
s.value, schema=schema, lang=Painless).items() if v
]
if len(s_path) == 0:
# FOR CONSTANTS
nest = NestedAggs(query_path)
acc.add(nest)
elif len(s_path) == 1:
nest = NestedAggs(first(s_path))
acc.add(nest)
else:
Log.error("do not know how to handle")
canonical_name = s.name
if is_op(s.value, TupleOp):
if s.aggregate == "count":
# TUPLES ALWAYS EXIST, SO COUNTING THEM IS EASY
s.pull = jx_expression_to_function("doc_count")
elif s.aggregate in ('max', 'maximum', 'min', 'minimum'):
if s.aggregate in ('max', 'maximum'):
dir = 1
op = "max"
else:
dir = -1
op = 'min'
nully = Painless[TupleOp(
[NULL] *
len(s.value.terms))].partial_eval().to_es_script(schema)
selfy = text_type(
Painless[s.value].partial_eval().to_es_script(schema))
script = {
"scripted_metric": {
'init_script':
'params._agg.best = ' + nully + ';',
'map_script':
'params._agg.best = ' + expand_template(
MAX_OF_TUPLE, {
"expr1": "params._agg.best",
"expr2": selfy,
"dir": dir,
"op": op
}) + ";",
'combine_script':
'return params._agg.best',
'reduce_script':
'return params._aggs.stream().' + op + '(' +
expand_template(COMPARE_TUPLE, {
"dir": dir,
"op": op
}) + ').get()',
}
}
nest.add(
NestedAggs(query_path).add(
ExprAggs(canonical_name, script, s)))
s.pull = jx_expression_to_function("value")
else:
Log.error("{{agg}} is not a supported aggregate over a tuple",
agg=s.aggregate)
elif s.aggregate == "count":
nest.add(
ExprAggs(
canonical_name, {
"value_count": {
"script":
text_type(Painless[
s.value].partial_eval().to_es_script(schema))
}
}, s))
s.pull = jx_expression_to_function("value")
elif s.aggregate == "median":
# ES USES DIFFERENT METHOD FOR PERCENTILES THAN FOR STATS AND COUNT
key = literal_field(canonical_name + " percentile")
nest.add(
ExprAggs(
key, {
"percentiles": {
"script":
text_type(Painless[s.value].to_es_script(schema)),
"percents": [50]
}
}, s))
s.pull = jx_expression_to_function(join_field(["50.0"]))
elif s.aggregate == "percentile":
# ES USES DIFFERENT METHOD FOR PERCENTILES THAN FOR STATS AND COUNT
key = literal_field(canonical_name + " percentile")
percent = mo_math.round(s.percentile * 100, decimal=6)
nest.add(
ExprAggs(
key, {
"percentiles": {
"script":
text_type(Painless[s.value].to_es_script(schema)),
"percents": [percent]
}
}, s))
s.pull = jx_expression_to_function(
join_field(["values", text_type(percent)]))
elif s.aggregate == "cardinality":
# ES USES DIFFERENT METHOD FOR CARDINALITY
key = canonical_name + " cardinality"
nest.add(
ExprAggs(
key, {
"cardinality": {
"script":
text_type(Painless[s.value].to_es_script(schema))
}
}, s))
s.pull = jx_expression_to_function("value")
elif s.aggregate == "stats":
# REGULAR STATS
nest.add(
ExprAggs(
canonical_name, {
"extended_stats": {
"script":
text_type(Painless[s.value].to_es_script(schema))
}
}, s))
s.pull = get_pull_stats()
# GET MEDIAN TOO!
select_median = s.copy()
select_median.pull = jx_expression_to_function(
{"select": [{
"name": "median",
"value": "values.50\\.0"
}]})
nest.add(
ExprAggs(
canonical_name + "_percentile", {
"percentiles": {
"script":
text_type(Painless[s.value].to_es_script(schema)),
"percents": [50]
}
}, select_median))
s.pull = get_pull_stats()
elif s.aggregate == "union":
# USE TERMS AGGREGATE TO SIMULATE union
nest.add(
TermsAggs(
canonical_name, {
"script_field":
text_type(Painless[s.value].to_es_script(schema))
}, s))
s.pull = jx_expression_to_function("key")
else:
# PULL VALUE OUT OF THE stats AGGREGATE
s.pull = jx_expression_to_function(aggregates[s.aggregate])
nest.add(
ExprAggs(
canonical_name, {
"extended_stats": {
"script":
text_type(
NumberOp(s.value).partial_eval().to_es_script(
schema))
}
}, s))
acc = NestedAggs(query_path).add(acc)
split_decoders = get_decoders_by_path(query)
split_wheres = split_expression_by_path(query.where,
schema=frum.schema,
lang=ES52)
start = 0
decoders = [None] * (len(query.edges) + len(query.groupby))
paths = list(reversed(sorted(split_wheres.keys() | split_decoders.keys())))
for path in paths:
literal_path = literal_field(path)
decoder = split_decoders[literal_path]
where = split_wheres[literal_path]
for d in decoder:
decoders[d.edge.dim] = d
acc = d.append_query(path, acc)
start += d.num_columns
if where:
acc = FilterAggs("_filter", AndOp(where), None).add(acc)
acc = NestedAggs(path).add(acc)
acc = NestedAggs('.').add(acc)
acc = simplify(acc)
es_query = wrap(acc.to_es(schema))
es_query.size = 0
with Timer("ES query time", silent=not DEBUG) as es_duration:
result = es_post(es, es_query, query.limit)
try:
format_time = Timer("formatting", silent=not DEBUG)
with format_time:
# result.aggregations.doc_count = coalesce(result.aggregations.doc_count, result.hits.total) # IT APPEARS THE OLD doc_count IS GONE
aggs = unwrap(result.aggregations)
formatter, groupby_formatter, aggop_formatter, mime_type = format_dispatch[
query.format]
if query.edges:
output = formatter(aggs, acc, query, decoders, select)
elif query.groupby:
output = groupby_formatter(aggs, acc, query, decoders, select)
else:
output = aggop_formatter(aggs, acc, query, decoders, select)
output.meta.timing.formatting = format_time.duration
output.meta.timing.es_search = es_duration.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
if query.format not in format_dispatch:
Log.error("Format {{format|quote}} not supported yet",
format=query.format,
cause=e)
Log.error("Some problem", cause=e)