def get_more(please_stop):
more.append(
es_post(
es,
Data(query={"filtered": {
"filter": more_filter
}},
fields=es_query.fields), query.limit))
def es_fieldop(es, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
FromES.query = {
"bool": {
"query": {
"match_all": {}
},
"filter": jx_expression(query.where).to_esfilter()
}
}
FromES.size = coalesce(query.limit, 200000)
FromES.fields = FlatList()
for s in select.value:
if s == "*":
FromES.fields = None
elif isinstance(s, list):
FromES.fields.extend(s)
elif isinstance(s, Mapping):
FromES.fields.extend(s.values())
else:
FromES.fields.append(s)
FromES.sort = [{
s.field: "asc" if s.sort >= 0 else "desc"
} for s in query.sort]
data = es_post(es, FromES, query.limit)
T = data.hits.hits
matricies = {}
for s in select:
if s.value == "*":
matricies[s.name] = Matrix.wrap([t._source for t in T])
elif isinstance(s.value, Mapping):
# for k, v in s.value.items():
# matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T])
matricies[s.name] = Matrix.wrap([{
k: unwrap(t.fields).get(v, None)
for k, v in s.value.items()
} for t in T])
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([
tuple(unwrap(t.fields).get(ss, None) for ss in s.value)
for t in T
])
elif not s.value:
matricies[s.name] = Matrix.wrap(
[unwrap(t.fields).get(s.value, None) for t in T])
else:
try:
matricies[s.name] = Matrix.wrap(
[unwrap(t.fields).get(s.value, None) for t in T])
except Exception as e:
Log.error("", e)
cube = Cube(query.select, query.edges, matricies, frum=query)
cube.frum = query
return cube
def _es_terms2(es, mvel, query):
"""
WE ASSUME THERE ARE JUST TWO EDGES, AND EACH HAS A SIMPLE value
"""
# REQUEST VALUES IN FIRST DIMENSION
q1 = query.copy()
q1.edges = query.edges[0:1:]
values1 = es_terms(es, mvel, q1).edges[0].domain.partitions.value
select = listwrap(query.select)
FromES = build_es_query(query)
for s in select:
for i, v in enumerate(values1):
FromES.facets[s.name + "," + str(i)] = {
"terms": {
"field": query.edges[1].value,
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter({"and": [
query.where,
{"term": {query.edges[0].value: v}}
]})
}
data = es_post(es, FromES, query.limit)
# UNION ALL TERMS FROM SECOND DIMENSION
values2 = set()
for k, f in data.facets.items():
values2.update(f.terms.term)
values2 = jx.sort(values2)
term2index = {v: i for i, v in enumerate(values2)}
query.edges[1].domain.partitions = FlatList([{"name": v, "value": v} for v in values2])
# MAKE CUBE
output = {}
dims = [len(values1), len(values2)]
for s in select:
output[s.name] = Matrix(*dims)
# FILL CUBE
# EXPECTING ONLY SELECT CLAUSE FACETS
for facetName, facet in data.facets.items():
coord = facetName.split(",")
s = [s for s in select if s.name == coord[0]][0]
i1 = int(coord[1])
for term in facet.terms:
i2 = term2index[term.term]
output[s.name][(i1, i2)] = term[aggregates[s.aggregate]]
cube = Cube(query.select, query.edges, output)
cube.query = query
return cube
def es_deepop(es, mvel, query):
FromES = es09.util.build_es_query(query)
select = query.edges
temp_query = query.copy()
temp_query.select = select
temp_query.edges = FlatList()
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(temp_query),
"size": query.limit
},
"facet_filter": jx_expression(query.where).to_esfilter()
}
data = es_post(es, FromES, query.limit)
rows = unpack_terms(data.facets.mvel, query.edges)
terms = zip(*rows)
# NUMBER ALL EDGES FOR JSON EXPRESSION INDEXING
edges = query.edges
for f, e in enumerate(edges):
for r in terms[f]:
e.domain.getPartByKey(r)
e.index = f
for p, part in enumerate(e.domain.partitions):
part.dataIndex = p
e.domain.NULL.dataIndex = len(e.domain.partitions)
# MAKE CUBE
dims = [len(e.domain.partitions) for e in query.edges]
output = Matrix(*dims)
# FILL CUBE
for r in rows:
term_coord = [
e.domain.getPartByKey(r[i]).dataIndex for i, e in enumerate(edges)
]
output[term_coord] = SUM(output[term_coord], r[-1])
cube = Cube(query.select, query.edges, {query.select.name: output})
cube.frum = query
return cube
def es_countop(es, mvel, query):
"""
RETURN SINGLE COUNT
"""
select = listwrap(query.select)
FromES = build_es_query(query)
for s in select:
if is_variable_name(s.value):
FromES.facets[s.name] = {
"terms": {
"field": s.value,
"size": query.limit,
},
"facet_filter": {
"exists": {
"field": s.value
}
}
}
else:
# COMPLICATED value IS PROBABLY A SCRIPT, USE IT
FromES.facets[s.name] = {
"terms": {
"script_field":
es09.expressions.compile_expression(s.value, query),
"size":
200000
}
}
data = es_post(es, FromES, query.limit)
matricies = {}
for s in select:
matricies[s.name] = Matrix(value=data.hits.facets[s.name].total)
cube = Cube(query.select, query.edges, matricies)
cube.frum = query
return cube
def es_aggop(es, mvel, query):
select = listwrap(query.select)
FromES = build_es_query(query)
isSimple = AND(aggregates[s.aggregate] == "count" for s in select)
if isSimple:
return es_countop(es, query) # SIMPLE, USE TERMS FACET INSTEAD
value2facet = dict() # ONLY ONE FACET NEEDED PER
name2facet = dict() # MAP name TO FACET WITH STATS
for s in select:
if s.value not in value2facet:
if isinstance(s.value, Variable):
unwrap(FromES.facets)[s.name] = {
"statistical": {
"field": s.value.var
},
"facet_filter": query.where.to_esfilter()
}
else:
unwrap(FromES.facets)[s.name] = {
"statistical": {
"script": jx_expression_to_function(s.value)
},
"facet_filter": query.where.to_es_filter()
}
value2facet[s.value] = s.name
name2facet[s.name] = value2facet[s.value]
data = es_post(es, FromES, query.limit)
matricies = {
s.name: Matrix(value=fix_es_stats(data.facets[literal_field(s.name)])[
aggregates[s.aggregate]])
for s in select
}
cube = Cube(query.select, [], matricies)
cube.frum = query
return cube
def es_aggsop(es, frum, query):
query = query.copy() # WE WILL MARK UP THIS QUERY
schema = frum.schema
select = listwrap(query.select)
es_query = Data()
new_select = Data(
) # MAP FROM canonical_name (USED FOR NAMES IN QUERY) TO SELECT MAPPING
formula = []
for s in select:
if s.aggregate == "count" and isinstance(
s.value, Variable) and s.value.var == ".":
if schema.query_path == ".":
s.pull = jx_expression_to_function("doc_count")
else:
s.pull = jx_expression_to_function(
{"coalesce": ["_nested.doc_count", "doc_count", 0]})
elif isinstance(s.value, Variable):
if s.aggregate == "count":
new_select["count_" + literal_field(s.value.var)] += [s]
else:
new_select[literal_field(s.value.var)] += [s]
elif s.aggregate:
formula.append(s)
for canonical_name, many in new_select.items():
for s in many:
es_cols = frum.schema.values(s.value.var)
if s.aggregate == "count":
canonical_names = []
for es_col in es_cols:
cn = literal_field(es_col.es_column + "_count")
if es_col.type == EXISTS:
canonical_names.append(cn + ".doc_count")
es_query.aggs[cn].filter.range = {
es_col.es_column: {
"gt": 0
}
}
else:
canonical_names.append(cn + ".value")
es_query.aggs[cn].value_count.field = es_col.es_column
if len(es_cols) == 1:
s.pull = jx_expression_to_function(canonical_names[0])
else:
s.pull = jx_expression_to_function(
{"add": canonical_names})
elif s.aggregate == "median":
if len(es_cols) > 1:
Log.error(
"Do not know how to count columns with more than one type (script probably)"
)
# ES USES DIFFERENT METHOD FOR PERCENTILES
key = literal_field(canonical_name + " percentile")
es_query.aggs[key].percentiles.field = es_cols[0].es_column
es_query.aggs[key].percentiles.percents += [50]
s.pull = jx_expression_to_function(key + ".values.50\.0")
elif s.aggregate == "percentile":
if len(es_cols) > 1:
Log.error(
"Do not know how to count columns with more than one type (script probably)"
)
# ES USES DIFFERENT METHOD FOR PERCENTILES
key = literal_field(canonical_name + " percentile")
if isinstance(
s.percentile,
text_type) or s.percetile < 0 or 1 < s.percentile:
Log.error(
"Expecting percentile to be a float from 0.0 to 1.0")
percent = Math.round(s.percentile * 100, decimal=6)
es_query.aggs[key].percentiles.field = es_cols[0].es_column
es_query.aggs[key].percentiles.percents += [percent]
s.pull = jx_expression_to_function(
key + ".values." + literal_field(text_type(percent)))
elif s.aggregate == "cardinality":
canonical_names = []
for es_col in es_cols:
cn = literal_field(es_col.es_column + "_cardinality")
canonical_names.append(cn)
es_query.aggs[cn].cardinality.field = es_col.es_column
if len(es_cols) == 1:
s.pull = jx_expression_to_function(canonical_names[0] +
".value")
else:
s.pull = jx_expression_to_function({
"add": [cn + ".value" for cn in canonical_names],
"default":
0
})
elif s.aggregate == "stats":
if len(es_cols) > 1:
Log.error(
"Do not know how to count columns with more than one type (script probably)"
)
# REGULAR STATS
stats_name = literal_field(canonical_name)
es_query.aggs[stats_name].extended_stats.field = es_cols[
0].es_column
# GET MEDIAN TOO!
median_name = literal_field(canonical_name + "_percentile")
es_query.aggs[median_name].percentiles.field = es_cols[
0].es_column
es_query.aggs[median_name].percentiles.percents += [50]
s.pull = get_pull_stats(stats_name, median_name)
elif s.aggregate == "union":
pulls = []
for es_col in es_cols:
script = {
"scripted_metric": {
'init_script':
'params._agg.terms = new HashSet()',
'map_script':
'for (v in doc[' + quote(es_col.es_column) +
'].values) params._agg.terms.add(v)',
'combine_script':
'return params._agg.terms.toArray()',
'reduce_script':
'HashSet output = new HashSet(); for (a in params._aggs) { if (a!=null) for (v in a) {output.add(v)} } return output.toArray()',
}
}
stats_name = encode_property(es_col.es_column)
if es_col.nested_path[0] == ".":
es_query.aggs[stats_name] = script
pulls.append(
jx_expression_to_function(stats_name + ".value"))
else:
es_query.aggs[stats_name] = {
"nested": {
"path": es_col.nested_path[0]
},
"aggs": {
"_nested": script
}
}
pulls.append(
jx_expression_to_function(stats_name +
"._nested.value"))
if len(pulls) == 0:
s.pull = NULL
elif len(pulls) == 1:
s.pull = pulls[0]
else:
s.pull = lambda row: UNION(p(row) for p in pulls)
else:
if len(es_cols) > 1:
Log.error(
"Do not know how to count columns with more than one type (script probably)"
)
# PULL VALUE OUT OF THE stats AGGREGATE
es_query.aggs[literal_field(
canonical_name
)].extended_stats.field = es_cols[0].es_column
s.pull = jx_expression_to_function({
"coalesce": [
literal_field(canonical_name) + "." +
aggregates[s.aggregate], s.default
]
})
for i, s in enumerate(formula):
canonical_name = literal_field(s.name)
if isinstance(s.value, TupleOp):
if s.aggregate == "count":
# TUPLES ALWAYS EXIST, SO COUNTING THEM IS EASY
s.pull = "doc_count"
else:
Log.error("{{agg}} is not a supported aggregate over a tuple",
agg=s.aggregate)
elif s.aggregate == "count":
es_query.aggs[literal_field(
canonical_name)].value_count.script = s.value.partial_eval(
).to_painless(schema).script(schema)
s.pull = jx_expression_to_function(
literal_field(canonical_name) + ".value")
elif s.aggregate == "median":
# ES USES DIFFERENT METHOD FOR PERCENTILES THAN FOR STATS AND COUNT
key = literal_field(canonical_name + " percentile")
es_query.aggs[key].percentiles.script = s.value.to_painless(
schema).script(schema)
es_query.aggs[key].percentiles.percents += [50]
s.pull = jx_expression_to_function(key + ".values.50\.0")
elif s.aggregate == "percentile":
# ES USES DIFFERENT METHOD FOR PERCENTILES THAN FOR STATS AND COUNT
key = literal_field(canonical_name + " percentile")
percent = Math.round(s.percentile * 100, decimal=6)
es_query.aggs[key].percentiles.script = s.value.to_painless(
schema).script(schema)
es_query.aggs[key].percentiles.percents += [percent]
s.pull = jx_expression_to_function(
key + ".values." + literal_field(text_type(percent)))
elif s.aggregate == "cardinality":
# ES USES DIFFERENT METHOD FOR CARDINALITY
key = canonical_name + " cardinality"
es_query.aggs[key].cardinality.script = s.value.to_painless(
schema).script(schema)
s.pull = jx_expression_to_function(key + ".value")
elif s.aggregate == "stats":
# REGULAR STATS
stats_name = literal_field(canonical_name)
es_query.aggs[
stats_name].extended_stats.script = s.value.to_painless(
schema).script(schema)
# GET MEDIAN TOO!
median_name = literal_field(canonical_name + " percentile")
es_query.aggs[
median_name].percentiles.script = s.value.to_painless(
schema).script(schema)
es_query.aggs[median_name].percentiles.percents += [50]
s.pull = get_pull_stats(stats_name, median_name)
elif s.aggregate == "union":
# USE TERMS AGGREGATE TO SIMULATE union
stats_name = literal_field(canonical_name)
es_query.aggs[stats_name].terms.script_field = s.value.to_painless(
schema).script(schema)
s.pull = jx_expression_to_function(stats_name + ".buckets.key")
else:
# PULL VALUE OUT OF THE stats AGGREGATE
s.pull = jx_expression_to_function(canonical_name + "." +
aggregates[s.aggregate])
es_query.aggs[
canonical_name].extended_stats.script = s.value.to_painless(
schema).script(schema)
decoders = get_decoders_by_depth(query)
start = 0
#<TERRIBLE SECTION> THIS IS WHERE WE WEAVE THE where CLAUSE WITH nested
split_where = split_expression_by_depth(query.where, schema=frum.schema)
if len(split_field(frum.name)) > 1:
if any(split_where[2::]):
Log.error("Where clause is too deep")
for d in decoders[1]:
es_query = d.append_query(es_query, start)
start += d.num_columns
if split_where[1]:
#TODO: INCLUDE FILTERS ON EDGES
filter_ = AndOp("and", split_where[1]).to_esfilter(schema)
es_query = Data(
aggs={"_filter": set_default({"filter": filter_}, es_query)})
es_query = wrap({
"aggs": {
"_nested":
set_default({"nested": {
"path": schema.query_path
}}, es_query)
}
})
else:
if any(split_where[1::]):
Log.error("Where clause is too deep")
if decoders:
for d in jx.reverse(decoders[0]):
es_query = d.append_query(es_query, start)
start += d.num_columns
if split_where[0]:
#TODO: INCLUDE FILTERS ON EDGES
filter = AndOp("and", split_where[0]).to_esfilter(schema)
es_query = Data(
aggs={"_filter": set_default({"filter": filter}, es_query)})
# </TERRIBLE SECTION>
if not es_query:
es_query = wrap({"query": {"match_all": {}}})
es_query.size = 0
with Timer("ES query time") as es_duration:
result = es_post(es, es_query, query.limit)
try:
format_time = Timer("formatting")
with format_time:
decoders = [d for ds in decoders for d in ds]
result.aggregations.doc_count = coalesce(
result.aggregations.doc_count,
result.hits.total) # IT APPEARS THE OLD doc_count IS GONE
formatter, groupby_formatter, aggop_formatter, mime_type = format_dispatch[
query.format]
if query.edges:
output = formatter(decoders, result.aggregations, start, query,
select)
elif query.groupby:
output = groupby_formatter(decoders, result.aggregations,
start, query, select)
else:
output = aggop_formatter(decoders, result.aggregations, start,
query, select)
output.meta.timing.formatting = format_time.duration
output.meta.timing.es_search = es_duration.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
if query.format not in format_dispatch:
Log.error("Format {{format|quote}} not supported yet",
format=query.format,
cause=e)
Log.error("Some problem", cause=e)
def es_terms(es, mvel, query):
"""
RETURN LIST OF ALL EDGE QUERIES
EVERY FACET IS NAMED <select.name>, <c1>, ... <cN> WHERE <ci> ARE THE ELEMENT COORDINATES
WE TRY TO PACK DIMENSIONS INTO THE TERMS TO MINIMIZE THE CROSS-PRODUCT EXPLOSION
"""
if len(query.edges) == 2:
return _es_terms2(es, mvel, query)
select = listwrap(query.select)
FromES = build_es_query(query)
packed_term = compileEdges2Term(mvel, query.edges, wrap([]))
for s in select:
FromES.facets[s.name] = {
"terms": {
"field": packed_term.field,
"script_field": packed_term.expression,
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
term2Parts = packed_term.term2parts
data = es_post(es, FromES, query.limit)
# GETTING ALL PARTS WILL EXPAND THE EDGES' DOMAINS
# BUT HOW TO UNPACK IT FROM THE term FASTER IS UNKNOWN
for k, f in data.facets.items():
for t in f.terms:
term2Parts(t.term)
# NUMBER ALL EDGES FOR jx INDEXING
for f, e in enumerate(query.edges):
e.index = f
if e.domain.type in ["uid", "default"]:
# e.domain.partitions = jx.sort(e.domain.partitions, "value")
for p, part in enumerate(e.domain.partitions):
part.dataIndex = p
e.domain.NULL.dataIndex = len(e.domain.partitions)
# MAKE CUBE
output = {}
dims = [len(e.domain.partitions) + (1 if e.allowNulls else 0) for e in query.edges]
for s in select:
output[s.name] = Matrix(*dims)
# FILL CUBE
# EXPECTING ONLY SELECT CLAUSE FACETS
for facetName, facet in data.facets.items():
for term in facet.terms:
term_coord = term2Parts(term.term).dataIndex
for s in select:
try:
output[s.name][term_coord] = term[aggregates[s.aggregate]]
except Exception as e:
# USUALLY CAUSED BY output[s.name] NOT BEING BIG ENOUGH TO HANDLE NULL COUNTS
pass
cube = Cube(query.select, query.edges, output)
cube.query = query
return cube
def es_deepop(es, query):
schema = query.frum.schema
columns = schema.columns
query_path = schema.query_path
# TODO: FIX THE GREAT SADNESS CAUSED BY EXECUTING post_expressions
# THE EXPRESSIONS SHOULD BE PUSHED TO THE CONTAINER: ES ALLOWS
# {"inner_hit":{"script_fields":[{"script":""}...]}}, BUT THEN YOU
# LOOSE "_source" BUT GAIN "fields", FORCING ALL FIELDS TO BE EXPLICIT
post_expressions = {}
es_query, es_filters = es_query_template(query_path)
# SPLIT WHERE CLAUSE BY DEPTH
wheres = split_expression_by_depth(query.where, schema)
for i, f in enumerate(es_filters):
script = AndOp("and", wheres[i]).partial_eval().to_esfilter(schema)
set_default(f, script)
if not wheres[1]:
more_filter = {
"bool": {
"must":
[AndOp("and", wheres[0]).partial_eval().to_esfilter(schema)],
"must_not": {
"nested": {
"path": query_path,
"query": {
"match_all": {}
}
}
}
}
}
else:
more_filter = None
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
# es_query.sort = jx_sort_to_es_sort(query.sort)
map_to_es_columns = schema.map_to_es()
# {c.names["."]: c.es_column for c in schema.leaves(".")}
query_for_es = query.map(map_to_es_columns)
es_query.sort = jx_sort_to_es_sort(query_for_es.sort, schema)
es_query.stored_fields = []
is_list = isinstance(query.select, list)
new_select = FlatList()
i = 0
for s in listwrap(query.select):
if isinstance(s.value, LeavesOp) and isinstance(
s.value.term, Variable):
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
leaves = schema.leaves(s.value.term.var)
col_names = set()
for c in leaves:
if c.nested_path[0] == ".":
if c.type == NESTED:
continue
es_query.stored_fields += [c.es_column]
c_name = untype_path(c.names[query_path])
col_names.add(c_name)
new_select.append({
"name": concat_field(s.name, c_name),
"nested_path": c.nested_path[0],
"put": {
"name": concat_field(s.name, literal_field(c_name)),
"index": i,
"child": "."
},
"pull": get_pull_function(c)
})
i += 1
# REMOVE DOTS IN PREFIX IF NAME NOT AMBIGUOUS
for n in new_select:
if n.name.startswith("..") and n.name.lstrip(
".") not in col_names:
n.put.name = n.name = n.name.lstrip(".")
col_names.add(n.name)
elif isinstance(s.value, Variable):
net_columns = schema.leaves(s.value.var)
if not net_columns:
new_select.append({
"name": s.name,
"nested_path": ".",
"put": {
"name": s.name,
"index": i,
"child": "."
},
"pull": NULL
})
else:
for n in net_columns:
pull = get_pull_function(n)
if n.nested_path[0] == ".":
if n.type == NESTED:
continue
es_query.stored_fields += [n.es_column]
# WE MUST FIGURE OUT WHICH NAMESSPACE s.value.var IS USING SO WE CAN EXTRACT THE child
for np in n.nested_path:
c_name = untype_path(n.names[np])
if startswith_field(c_name, s.value.var):
child = relative_field(c_name, s.value.var)
break
else:
child = relative_field(
untype_path(n.names[n.nested_path[0]]),
s.value.var)
new_select.append({
"name": s.name,
"pull": pull,
"nested_path": n.nested_path[0],
"put": {
"name": s.name,
"index": i,
"child": child
}
})
i += 1
else:
expr = s.value
for v in expr.vars():
for c in schema[v]:
if c.nested_path[0] == ".":
es_query.stored_fields += [c.es_column]
# else:
# Log.error("deep field not expected")
pull_name = EXPRESSION_PREFIX + s.name
map_to_local = {
untype_path(k): get_pull(cc)
for k, c in schema.lookup.items() for cc in c
if cc.type not in STRUCT
}
pull = jx_expression_to_function(pull_name)
post_expressions[pull_name] = compile_expression(
expr.map(map_to_local).to_python())
new_select.append({
"name": s.name if is_list else ".",
"pull": pull,
"value": expr.__data__(),
"put": {
"name": s.name,
"index": i,
"child": "."
}
})
i += 1
# <COMPLICATED> ES needs two calls to get all documents
more = []
def get_more(please_stop):
more.append(
es_post(
es,
Data(query=more_filter, stored_fields=es_query.stored_fields),
query.limit))
if more_filter:
need_more = Thread.run("get more", target=get_more)
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
# EACH A HIT IS RETURNED MULTIPLE TIMES FOR EACH INNER HIT, WITH INNER HIT INCLUDED
def inners():
for t in data.hits.hits:
for i in t.inner_hits[literal_field(query_path)].hits.hits:
t._inner = i._source
for k, e in post_expressions.items():
t[k] = e(t)
yield t
if more_filter:
Thread.join(need_more)
for t in more[0].hits.hits:
yield t
#</COMPLICATED>
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(inners(), new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def get_more(please_stop):
more.append(
es_post(
es,
Data(query=more_filter, stored_fields=es_query.stored_fields),
query.limit))
def es_setop(es, query):
schema = query.frum.schema
es_query, filters = es_query_template(schema.query_path)
nested_filter = None
set_default(filters[0], query.where.partial_eval().to_esfilter(schema))
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
es_query.fields = FlatList()
selects = wrap([s.copy() for s in listwrap(query.select)])
new_select = FlatList()
schema = query.frum.schema
# columns = schema.columns
# nested_columns = set(c.names["."] for c in columns if c.nested_path[0] != ".")
es_query.sort = jx_sort_to_es_sort(query.sort, schema)
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if isinstance(select.value, LeavesOp) and isinstance(
select.value.term, Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
full_name = concat_field(
select.name,
relative_field(untype_path(c.names["."]), term.var))
if c.type == NESTED:
es_query.fields = ["_source"]
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": "."
},
"pull": get_pull_source(c.es_column)
})
put_index += 1
elif c.nested_path[0] != ".":
es_query.fields = ["_source"]
else:
es_query.fields += [c.es_column]
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": "."
}
})
put_index += 1
elif isinstance(select.value, Variable):
s_column = select.value.var
# LEAVES OF OBJECT
leaves = schema.leaves(s_column)
nested_selects = {}
if leaves:
if any(c.type == NESTED for c in leaves):
# PULL WHOLE NESTED ARRAYS
es_query.fields = ["_source"]
for c in leaves:
if len(c.nested_path) == 1:
jx_name = untype_path(c.names["."])
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child": relative_field(jx_name, s_column)
},
"pull":
get_pull_source(c.es_column)
})
else:
# PULL ONLY WHAT'S NEEDED
for c in leaves:
if len(c.nested_path) == 1:
jx_name = untype_path(c.names["."])
if c.type == NESTED:
es_query.fields = ["_source"]
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child":
relative_field(jx_name, s_column)
},
"pull":
get_pull_source(c.es_column)
})
else:
es_query.fields += [c.es_column]
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child":
relative_field(jx_name, s_column)
}
})
else:
if not nested_filter:
where = filters[0].copy()
nested_filter = [where]
for k in filters[0].keys():
filters[0][k] = None
set_default(
filters[0],
{"and": [where, {
"or": nested_filter
}]})
nested_path = c.nested_path[0]
if nested_path not in nested_selects:
where = nested_selects[nested_path] = Data()
nested_filter += [where]
where.nested.path = nested_path
where.nested.query.match_all = {}
where.nested.inner_hits._source = False
where.nested.inner_hits.fields += [c.es_column]
child = relative_field(
untype_path(c.names[schema.query_path]),
s_column)
pull = accumulate_nested_doc(
nested_path,
Variable(
relative_field(
s_column,
unnest_path(nested_path))))
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": child
},
"pull": pull
})
else:
nested_selects[
nested_path].nested.inner_hits.fields += [
c.es_column
]
else:
new_select.append({
"name": select.name,
"value": Variable("$dummy"),
"put": {
"name": select.name,
"index": put_index,
"child": "."
}
})
put_index += 1
else:
painless = select.value.partial_eval().to_ruby(schema)
es_query.script_fields[literal_field(select.name)] = {
"script": painless.script(schema)
}
new_select.append({
"name":
select.name,
"pull":
jx_expression_to_function("fields." +
literal_field(select.name)),
"put": {
"name": select.name,
"index": put_index,
"child": "."
}
})
put_index += 1
for n in new_select:
if n.pull:
continue
elif isinstance(n.value, Variable):
if es_query.fields[0] == "_source":
es_query.fields = ["_source"]
n.pull = get_pull_source(n.value.var)
else:
n.pull = jx_expression_to_function(
concat_field("fields", literal_field(n.value.var)))
else:
Log.error("Do not know what to do")
with Timer("call to ES") as call_timer:
Log.note("{{data}}", data=es_query)
data = es_post(es, es_query, query.limit)
T = data.hits.hits
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def es_setop(es, mvel, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
isDeep = len(split_field(
query.frum.name)) > 1 # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
isComplex = OR([
s.value == None and s.aggregate not in ("count", "none")
for s in select
]) # CONVERTING esfilter DEFINED PARTS WILL REQUIRE SCRIPT
if not isDeep and not isComplex:
if len(select) == 1 and isinstance(select[0].value, LeavesOp):
FromES = wrap({
"query": {
"bool": {
"query": {
"match_all": {}
},
"filter": query.where.to_esfilter()
}
},
"sort": query.sort,
"size": 0
})
elif all(isinstance(v, Variable) for v in select.value):
FromES = wrap({
"query": {
"bool": {
"query": {
"match_all": {}
},
"filter": query.where.to_esfilter()
}
},
"fields": select.value,
"sort": query.sort,
"size": coalesce(query.limit, 200000)
})
elif not isDeep:
simple_query = query.copy()
simple_query.where = TRUE # THE FACET FILTER IS FASTER
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(simple_query),
"size": coalesce(simple_query.limit, 200000)
},
"facet_filter": jx_expression(query.where).to_esfilter()
}
else:
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(query),
"size": coalesce(query.limit, 200000)
},
"facet_filter": jx_expression(query.where).to_esfilter()
}
data = es_post(es, FromES, query.limit)
if len(select) == 1 and isinstance(select[0].value, LeavesOp):
# SPECIAL CASE FOR SINGLE COUNT
cube = wrap(data).hits.hits._source
elif isinstance(select[0].value, Variable):
# SPECIAL CASE FOR SINGLE TERM
cube = wrap(data).hits.hits.fields
else:
data_list = unpack_terms(data.facets.mvel, select)
if not data_list:
cube = Cube(select, [], {s.name: Matrix.wrap([]) for s in select})
else:
output = zip(*data_list)
cube = Cube(
select, [],
{s.name: Matrix(list=output[i])
for i, s in enumerate(select)})
return Data(meta={"esquery": FromES}, data=cube)