def __init__(self, query_path, snowflake):
if not is_list(snowflake.query_paths[0]):
Log.error("Snowflake query paths should be a list of string tuples (well, technically, a list of lists of strings)")
self.snowflake = snowflake
try:
path = [
p
for p in snowflake.query_paths
if untype_path(p[0]) == query_path
]
if path:
# WE DO NOT NEED TO LOOK INTO MULTI-VALUED FIELDS AS A TABLE
self.multi = None
self.query_path = path[0]
else:
# LOOK INTO A SPECIFIC MULTI VALUED COLUMN
try:
self.multi = [
c
for c in self.snowflake.columns
if untype_path(c.name) == query_path and c.multi > 1
][0]
self.query_path = [self.multi.name] + self.multi.nested_path
except Exception as e:
# PROBLEM WITH METADATA UPDATE
self.multi = None
self.query_path = [query_path] + ["."]
Log.warning("Problem getting query path {{path|quote}} in snowflake {{sf|quote}}", path=query_path, sf=snowflake.name, cause=e)
if not is_list(self.query_path) or self.query_path[len(self.query_path) - 1] != ".":
Log.error("error")
except Exception as e:
Log.error("logic error", cause=e)
def _normalize_group(edge, dim_index, limit, schema=None):
"""
:param edge: Not normalized groupby
:param dim_index: Dimensions are ordered; this is this groupby's index into that order
:param schema: for context
:return: a normalized groupby
"""
if isinstance(edge, text_type):
if edge.endswith(".*"):
prefix = edge[:-2]
if schema:
output = wrap([
{
"name": concat_field(prefix, literal_field(relative_field(untype_path(c.names["."]), prefix))),
"put": {"name": literal_field(untype_path(c.names["."]))},
"value": jx_expression(c.es_column, schema=schema),
"allowNulls": True,
"domain": {"type": "default"}
}
for c in schema.leaves(prefix)
])
return output
else:
return wrap([{
"name": untype_path(prefix),
"put": {"name": literal_field(untype_path(prefix))},
"value": jx_expression(prefix, schema=schema),
"allowNulls": True,
"dim":dim_index,
"domain": {"type": "default"}
}])
return wrap([{
"name": edge,
"value": jx_expression(edge, schema=schema),
"allowNulls": True,
"dim": dim_index,
"domain": Domain(type="default", limit=limit)
}])
else:
edge = wrap(edge)
if (edge.domain and edge.domain.type != "default") or edge.allowNulls != None:
Log.error("groupby does not accept complicated domains")
if not edge.name and not isinstance(edge.value, text_type):
Log.error("You must name compound edges: {{edge}}", edge= edge)
return wrap([{
"name": coalesce(edge.name, edge.value),
"value": jx_expression(edge.value, schema=schema),
"allowNulls": True,
"dim":dim_index,
"domain": {"type": "default"}
}])
def not_monitor(self, please_stop):
Log.alert("metadata scan has been disabled")
please_stop.on_go(lambda: self.todo.add(THREAD_STOP))
while not please_stop:
column = self.todo.pop()
if column == THREAD_STOP:
break
if column.jx_type in STRUCT or split_field(column.es_column)[-1] == EXISTS_TYPE:
DEBUG and Log.note("{{column.es_column}} is a struct", column=column)
column.last_updated = Date.now()
continue
elif column.last_updated > Date.now() - TOO_OLD and column.cardinality is not None:
# DO NOT UPDATE FRESH COLUMN METADATA
DEBUG and Log.note("{{column.es_column}} is still fresh ({{ago}} ago)", column=column, ago=(Date.now()-Date(column.last_updated)).seconds)
continue
with Timer("Update {{col.es_index}}.{{col.es_column}}", param={"col": column}, silent=not DEBUG, too_long=0.05):
if untype_path(column.name) in ["build.type", "run.type"]:
try:
self._update_cardinality(column)
except Exception as e:
Log.warning("problem getting cardinality for {{column.name}}", column=column, cause=e)
else:
column.last_updated = Date.now()
def denormalized(self):
"""
THE INTERNAL STRUCTURE FOR THE COLUMN METADATA IS VERY DIFFERENT FROM
THE DENORMALIZED PERSPECITVE. THIS PROVIDES THAT PERSPECTIVE FOR QUERIES
"""
with self.locker:
self._update_meta()
output = [
{
"table": c.es_index,
"name": untype_path(c.name),
"cardinality": c.cardinality,
"es_column": c.es_column,
"es_index": c.es_index,
"last_updated": c.last_updated,
"count": c.count,
"nested_path": [unnest_path(n) for n in c.nested_path],
"es_type": c.es_type,
"type": c.jx_type,
}
for tname, css in self.data.items()
for cname, cs in css.items()
for c in cs
if c.jx_type not in STRUCT # and c.es_column != "_id"
]
from jx_python.containers.list_usingPythonList import ListContainer
return ListContainer(
self.name,
data=output,
schema=jx_base.Schema("meta.columns", SIMPLE_METADATA_COLUMNS),
)
def values(self, column_name, exclude_type=STRUCT):
"""
RETURN ALL COLUMNS THAT column_name REFERS TO
"""
column_name = unnest_path(column_name)
columns = self.columns
output = []
for path in self.query_path:
full_path = untype_path(concat_field(path, column_name))
for c in columns:
if c.jx_type in exclude_type:
continue
# if c.cardinality == 0:
# continue
if untype_path(c.name) == full_path:
output.append(c)
if output:
return output
return []
def __init__(self, query_path, snowflake):
if not isinstance(snowflake.query_paths[0], list):
Log.error("Snowflake query paths should be a list of string tuples (well, technically, a list of lists of strings)")
try:
self.query_path = [
p
for p in snowflake.query_paths
if untype_path(p[0]) == query_path
][0]
self.snowflake = snowflake
except Exception as e:
Log.error("logic error", cause=e)
def output(doc):
acc = []
for h in doc.inner_hits[name].hits.hits:
i = h._nested.offset
obj = Data()
for f, v in h.fields.items():
local_path = untype_path(relative_field(f, nested_path))
obj[local_path] = unwraplist(v)
# EXTEND THE LIST TO THE LENGTH WE REQUIRE
for _ in range(len(acc), i+1):
acc.append(None)
acc[i] = expr(obj)
return acc
def map_to_es(self):
"""
RETURN A MAP FROM THE NAMESPACE TO THE es_column NAME
"""
output = {}
for path in self.query_path:
set_default(
output,
{
k: c.es_column
for c in self.snowflake.columns
if c.jx_type not in STRUCT
for rel_name in [c.names[path]]
for k in [rel_name, untype_path(rel_name), unnest_path(rel_name)]
}
)
return output
def values(self, column_name):
"""
RETURN ALL COLUMNS THAT column_name REFERES TO
"""
column_name = unnest_path(column_name)
columns = self.columns
deep_path = self.query_path[0]
for path in self.query_path:
output = [
c
for c in columns
if (
c.jx_type not in STRUCT and
untype_path(c.names[path]) == column_name
)
]
if output:
return output
return output
def __init__(self, edge, query, limit):
AggsDecoder.__init__(self, edge, query, limit)
if isinstance(edge.value, LeavesOp):
prefix = edge.value.term.var
flatter = lambda k: literal_field(relative_field(k, prefix))
else:
prefix = edge.value.var
flatter = lambda k: relative_field(k, prefix)
self.put, self.fields = transpose(*[
(flatter(untype_path(c.names["."])), c.es_column)
for c in query.frum.schema.leaves(prefix)
])
self.domain = self.edge.domain = wrap({"dimension": {"fields": self.fields}})
self.domain.limit = Math.min(coalesce(self.domain.limit, query.limit, 10), MAX_LIMIT)
self.parts = list()
self.key2index = {}
self.computed_domain = False
def _normalize_group(edge, dim_index, limit, schema=None):
"""
:param edge: Not normalized groupby
:param dim_index: Dimensions are ordered; this is this groupby's index into that order
:param schema: for context
:return: a normalized groupby
"""
if is_text(edge):
if edge.endswith(".*"):
prefix = edge[:-2]
if schema:
output = wrap([
{ # BECASUE THIS IS A GROUPBY, EARLY SPLIT INTO LEAVES WORKS JUST FINE
"name":
concat_field(
prefix,
literal_field(
relative_field(untype_path(c.name), prefix))),
"put": {
"name": literal_field(untype_path(c.name))
},
"value":
jx_expression(c.es_column, schema=schema),
"allowNulls":
True,
"domain": {
"type": "default"
}
} for c in schema.leaves(prefix)
])
return output
else:
return wrap([{
"name": untype_path(prefix),
"put": {
"name": literal_field(untype_path(prefix))
},
"value": LeavesOp(Variable(prefix)),
"allowNulls": True,
"dim": dim_index,
"domain": {
"type": "default"
}
}])
return wrap([{
"name": edge,
"value": jx_expression(edge, schema=schema),
"allowNulls": True,
"dim": dim_index,
"domain": Domain(type="default", limit=limit)
}])
else:
edge = wrap(edge)
if (edge.domain and edge.domain.type != "default"):
Log.error("groupby does not accept complicated domains")
if not edge.name and not is_text(edge.value):
Log.error("You must name compound edges: {{edge}}", edge=edge)
return wrap([{
"name": coalesce(edge.name, edge.value),
"value": jx_expression(edge.value, schema=schema),
"allowNulls": True,
"dim": dim_index,
"domain": {
"type": "default"
}
}])
def es_deepop(es, query):
schema = query.frum.schema
query_path = schema.query_path[0]
# TODO: FIX THE GREAT SADNESS CAUSED BY EXECUTING post_expressions
# THE EXPRESSIONS SHOULD BE PUSHED TO THE CONTAINER: ES ALLOWS
# {"inner_hit":{"script_fields":[{"script":""}...]}}, BUT THEN YOU
# LOOSE "_source" BUT GAIN "fields", FORCING ALL FIELDS TO BE EXPLICIT
post_expressions = {}
es_query, es_filters = es_query_template(query_path)
# SPLIT WHERE CLAUSE BY DEPTH
wheres = split_expression_by_depth(query.where, schema)
for f, w in zip_longest(es_filters, wheres):
script = ES52[AndOp(w)].partial_eval().to_esfilter(schema)
set_default(f, script)
if not wheres[1]:
# INCLUDE DOCS WITH NO NESTED DOCS
more_filter = {
"bool": {
"filter": [AndOp(wheres[0]).partial_eval().to_esfilter(schema)],
"must_not": {
"nested": {
"path": query_path,
"query": MATCH_ALL
}
}
}
}
else:
more_filter = None
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
map_to_es_columns = schema.map_to_es()
query_for_es = query.map(map_to_es_columns)
es_query.sort = jx_sort_to_es_sort(query_for_es.sort, schema)
es_query.stored_fields = []
is_list = is_list_(query.select)
selects = wrap([unwrap(s.copy()) for s in listwrap(query.select)])
new_select = FlatList()
put_index = 0
for select in selects:
if is_op(select.value, LeavesOp) and is_op(select.value.term, Variable):
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
leaves = schema.leaves(select.value.term.var)
col_names = set()
for c in leaves:
if c.nested_path[0] == ".":
if c.jx_type == NESTED:
continue
es_query.stored_fields += [c.es_column]
c_name = untype_path(relative_field(c.name, query_path))
col_names.add(c_name)
new_select.append({
"name": concat_field(select.name, c_name),
"nested_path": c.nested_path[0],
"put": {"name": concat_field(select.name, literal_field(c_name)), "index": put_index, "child": "."},
"pull": get_pull_function(c)
})
put_index += 1
# REMOVE DOTS IN PREFIX IF NAME NOT AMBIGUOUS
for n in new_select:
if n.name.startswith("..") and n.name.lstrip(".") not in col_names:
n.put.name = n.name = n.name.lstrip(".")
col_names.add(n.name)
elif is_op(select.value, Variable):
net_columns = schema.leaves(select.value.var)
if not net_columns:
new_select.append({
"name": select.name,
"nested_path": ".",
"put": {"name": select.name, "index": put_index, "child": "."},
"pull": NULL
})
else:
for n in net_columns:
pull = get_pull_function(n)
if n.nested_path[0] == ".":
if n.jx_type == NESTED:
continue
es_query.stored_fields += [n.es_column]
# WE MUST FIGURE OUT WHICH NAMESSPACE s.value.var IS USING SO WE CAN EXTRACT THE child
for np in n.nested_path:
c_name = untype_path(relative_field(n.name, np))
if startswith_field(c_name, select.value.var):
# PREFER THE MOST-RELATIVE NAME
child = relative_field(c_name, select.value.var)
break
else:
continue
new_select.append({
"name": select.name,
"pull": pull,
"nested_path": n.nested_path[0],
"put": {
"name": select.name,
"index": put_index,
"child": child
}
})
put_index += 1
else:
expr = select.value
for v in expr.vars():
for c in schema[v.var]:
if c.nested_path[0] == ".":
es_query.stored_fields += [c.es_column]
# else:
# Log.error("deep field not expected")
pull_name = EXPRESSION_PREFIX + select.name
map_to_local = MapToLocal(schema)
pull = jx_expression_to_function(pull_name)
post_expressions[pull_name] = jx_expression_to_function(expr.map(map_to_local))
new_select.append({
"name": select.name if is_list else ".",
"pull": pull,
"value": expr.__data__(),
"put": {"name": select.name, "index": put_index, "child": "."}
})
put_index += 1
es_query.stored_fields = sorted(es_query.stored_fields)
# <COMPLICATED> ES needs two calls to get all documents
more = []
def get_more(please_stop):
more.append(es_post(
es,
Data(
query=more_filter,
stored_fields=es_query.stored_fields
),
query.limit
))
if more_filter:
need_more = Thread.run("get more", target=get_more)
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
# EACH A HIT IS RETURNED MULTIPLE TIMES FOR EACH INNER HIT, WITH INNER HIT INCLUDED
def inners():
for t in data.hits.hits:
for i in t.inner_hits[literal_field(query_path)].hits.hits:
t._inner = i._source
for k, e in post_expressions.items():
t[k] = e(t)
yield t
if more_filter:
Thread.join(need_more)
for t in more[0].hits.hits:
yield t
# </COMPLICATED>
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(inners(), new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def es_deepop(es, query):
schema = query.frum.schema
query_path = schema.query_path[0]
# TODO: FIX THE GREAT SADNESS CAUSED BY EXECUTING post_expressions
# THE EXPRESSIONS SHOULD BE PUSHED TO THE CONTAINER: ES ALLOWS
# {"inner_hit":{"script_fields":[{"script":""}...]}}, BUT THEN YOU
# LOOSE "_source" BUT GAIN "fields", FORCING ALL FIELDS TO BE EXPLICIT
post_expressions = {}
es_query, es_filters = es_query_template(query_path)
# SPLIT WHERE CLAUSE BY DEPTH
wheres = split_expression_by_depth(query.where, schema)
for i, f in enumerate(es_filters):
script = AndOp("and", wheres[i]).partial_eval().to_esfilter(schema)
set_default(f, script)
if not wheres[1]:
# WITHOUT NESTED CONDITIONS, WE MUST ALSO RETURN DOCS WITH NO NESTED RECORDS
more_filter = {
"and": [
es_filters[0],
{"missing": {"field": untype_path(query_path) + "." + EXISTS_TYPE}}
]
}
else:
more_filter = None
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
# es_query.sort = jx_sort_to_es_sort(query.sort)
map_to_es_columns = schema.map_to_es()
# {c.names["."]: c.es_column for c in schema.leaves(".")}
query_for_es = query.map(map_to_es_columns)
es_query.sort = jx_sort_to_es_sort(query_for_es.sort, schema)
es_query.fields = []
is_list = isinstance(query.select, list)
new_select = FlatList()
i = 0
for s in listwrap(query.select):
if isinstance(s.value, LeavesOp) and isinstance(s.value.term, Variable):
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
leaves = schema.leaves(s.value.term.var)
col_names = set()
for c in leaves:
if c.nested_path[0] == ".":
if c.jx_type == NESTED:
continue
es_query.fields += [c.es_column]
c_name = untype_path(c.names[query_path])
col_names.add(c_name)
new_select.append({
"name": concat_field(s.name, c_name),
"nested_path": c.nested_path[0],
"put": {"name": concat_field(s.name, literal_field(c_name)), "index": i, "child": "."},
"pull": get_pull_function(c)
})
i += 1
# REMOVE DOTS IN PREFIX IF NAME NOT AMBIGUOUS
for n in new_select:
if n.name.startswith("..") and n.name.lstrip(".") not in col_names:
n.put.name = n.name = n.name.lstrip(".")
col_names.add(n.name)
elif isinstance(s.value, Variable):
net_columns = schema.leaves(s.value.var)
if not net_columns:
new_select.append({
"name": s.name,
"nested_path": ".",
"put": {"name": s.name, "index": i, "child": "."},
"pull": NULL
})
else:
for n in net_columns:
pull = get_pull_function(n)
if n.nested_path[0] == ".":
if n.jx_type == NESTED:
continue
es_query.fields += [n.es_column]
# WE MUST FIGURE OUT WHICH NAMESSPACE s.value.var IS USING SO WE CAN EXTRACT THE child
for np in n.nested_path:
c_name = untype_path(n.names[np])
if startswith_field(c_name, s.value.var):
child = relative_field(c_name, s.value.var)
break
else:
child = relative_field(untype_path(n.names[n.nested_path[0]]), s.value.var)
new_select.append({
"name": s.name,
"pull": pull,
"nested_path": n.nested_path[0],
"put": {
"name": s.name,
"index": i,
"child": child
}
})
i += 1
else:
expr = s.value
for v in expr.vars():
for c in schema[v.var]:
if c.nested_path[0] == ".":
es_query.fields += [c.es_column]
# else:
# Log.error("deep field not expected")
pull_name = EXPRESSION_PREFIX + s.name
map_to_local = MapToLocal(schema)
pull = jx_expression_to_function(pull_name)
post_expressions[pull_name] = compile_expression(expr.map(map_to_local).to_python())
new_select.append({
"name": s.name if is_list else ".",
"pull": pull,
"value": expr.__data__(),
"put": {"name": s.name, "index": i, "child": "."}
})
i += 1
# <COMPLICATED> ES needs two calls to get all documents
more = []
def get_more(please_stop):
more.append(es_post(
es,
Data(
query={"filtered": {"filter": more_filter}},
fields=es_query.fields
),
query.limit
))
if more_filter:
need_more = Thread.run("get more", target=get_more)
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
# EACH A HIT IS RETURNED MULTIPLE TIMES FOR EACH INNER HIT, WITH INNER HIT INCLUDED
def inners():
for t in data.hits.hits:
for i in t.inner_hits[literal_field(query_path)].hits.hits:
t._inner = i._source
for k, e in post_expressions.items():
t[k] = e(t)
yield t
if more_filter:
Thread.join(need_more)
for t in more[0].hits.hits:
yield t
#</COMPLICATED>
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(inners(), new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def es_deepop(es, query):
schema = query.frum.schema
columns = schema.columns
query_path = schema.query_path
# TODO: FIX THE GREAT SADNESS CAUSED BY EXECUTING post_expressions
# THE EXPRESSIONS SHOULD BE PUSHED TO THE CONTAINER: ES ALLOWS
# {"inner_hit":{"script_fields":[{"script":""}...]}}, BUT THEN YOU
# LOOSE "_source" BUT GAIN "fields", FORCING ALL FIELDS TO BE EXPLICIT
post_expressions = {}
es_query, es_filters = es_query_template(query_path)
# SPLIT WHERE CLAUSE BY DEPTH
wheres = split_expression_by_depth(query.where, schema)
for i, f in enumerate(es_filters):
script = AndOp("and", wheres[i]).partial_eval().to_esfilter(schema)
set_default(f, script)
if not wheres[1]:
# WITHOUT NESTED CONDITIONS, WE MUST ALSO RETURN DOCS WITH NO NESTED RECORDS
more_filter = {
"and": [
es_filters[0], {
"missing": {
"field": untype_path(query_path) + "." + EXISTS_TYPE
}
}
]
}
else:
more_filter = None
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
# es_query.sort = jx_sort_to_es_sort(query.sort)
map_to_es_columns = schema.map_to_es()
# {c.names["."]: c.es_column for c in schema.leaves(".")}
query_for_es = query.map(map_to_es_columns)
es_query.sort = jx_sort_to_es_sort(query_for_es.sort, schema)
es_query.fields = []
is_list = isinstance(query.select, list)
new_select = FlatList()
i = 0
for s in listwrap(query.select):
if isinstance(s.value, LeavesOp) and isinstance(
s.value.term, Variable):
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
leaves = schema.leaves(s.value.term.var)
col_names = set()
for c in leaves:
if c.nested_path[0] == ".":
if c.type == NESTED:
continue
es_query.fields += [c.es_column]
c_name = untype_path(c.names[query_path])
col_names.add(c_name)
new_select.append({
"name": concat_field(s.name, c_name),
"nested_path": c.nested_path[0],
"put": {
"name": concat_field(s.name, literal_field(c_name)),
"index": i,
"child": "."
},
"pull": get_pull_function(c)
})
i += 1
# REMOVE DOTS IN PREFIX IF NAME NOT AMBIGUOUS
for n in new_select:
if n.name.startswith("..") and n.name.lstrip(
".") not in col_names:
n.put.name = n.name = n.name.lstrip(".")
col_names.add(n.name)
elif isinstance(s.value, Variable):
net_columns = schema.leaves(s.value.var)
if not net_columns:
new_select.append({
"name": s.name,
"nested_path": ".",
"put": {
"name": s.name,
"index": i,
"child": "."
},
"pull": NULL
})
else:
for n in net_columns:
pull = get_pull_function(n)
if n.nested_path[0] == ".":
if n.type == NESTED:
continue
es_query.fields += [n.es_column]
# WE MUST FIGURE OUT WHICH NAMESSPACE s.value.var IS USING SO WE CAN EXTRACT THE child
for np in n.nested_path:
c_name = untype_path(n.names[np])
if startswith_field(c_name, s.value.var):
child = relative_field(c_name, s.value.var)
break
else:
child = relative_field(
untype_path(n.names[n.nested_path[0]]),
s.value.var)
new_select.append({
"name": s.name,
"pull": pull,
"nested_path": n.nested_path[0],
"put": {
"name": s.name,
"index": i,
"child": child
}
})
i += 1
else:
expr = s.value
for v in expr.vars():
for c in schema[v]:
if c.nested_path[0] == ".":
es_query.fields += [c.es_column]
# else:
# Log.error("deep field not expected")
pull_name = EXPRESSION_PREFIX + s.name
map_to_local = {
untype_path(k): get_pull(cc)
for k, c in schema.lookup.items() for cc in c
if cc.type not in STRUCT
}
pull = jx_expression_to_function(pull_name)
post_expressions[pull_name] = compile_expression(
expr.map(map_to_local).to_python())
new_select.append({
"name": s.name if is_list else ".",
"pull": pull,
"value": expr.__data__(),
"put": {
"name": s.name,
"index": i,
"child": "."
}
})
i += 1
# <COMPLICATED> ES needs two calls to get all documents
more = []
def get_more(please_stop):
more.append(
es_post(
es,
Data(query={"filtered": {
"filter": more_filter
}},
fields=es_query.fields), query.limit))
if more_filter:
need_more = Thread.run("get more", target=get_more)
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
# EACH A HIT IS RETURNED MULTIPLE TIMES FOR EACH INNER HIT, WITH INNER HIT INCLUDED
def inners():
for t in data.hits.hits:
for i in t.inner_hits[literal_field(query_path)].hits.hits:
t._inner = i._source
for k, e in post_expressions.items():
t[k] = e(t)
yield t
if more_filter:
Thread.join(need_more)
for t in more[0].hits.hits:
yield t
#</COMPLICATED>
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(inners(), new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def doc_to_column(doc):
now = Date.now()
try:
doc = to_data(untyped(doc))
# I HAVE MANAGED TO MAKE MANY MISTAKES WRITING COLUMNS TO ES. HERE ARE THE FIXES
# FIX
if not doc.last_updated:
doc.last_updated = Date.now() - YEAR
# FIX
if doc.es_type == None:
if doc.jx_type == OBJECT:
doc.es_type = "object"
else:
Log.warning("{{doc}} has no es_type", doc=doc)
# FIX
if doc.es_type == "nested":
doc.multi = 1001
if doc.multi == None:
doc.multi = 1
# FIX
if doc.es_column.endswith("." + NESTED_TYPE):
if doc.jx_type == OBJECT:
doc.jx_type = NESTED
doc.last_updated = now
if doc.es_type == "nested":
doc.es_type = "nested"
doc.last_updated = now
# FIX
doc.nested_path = tuple(listwrap(doc.nested_path))
if last(split_field(
doc.es_column)) == NESTED_TYPE and doc.es_type != "nested":
doc.es_type = "nested"
doc.jx_type = NESTED
doc.multi = 1001
doc.last_updated = now
# FIX
expected_nested_path = get_nested_path(doc.es_column)
if len(doc.nested_path) > 1 and doc.nested_path[-2] == '.':
doc.nested_path = doc.nested_path[:-1]
doc.last_updated = now
# FIX
if untype_path(doc.es_column) == doc.es_column:
if doc.nested_path != (".", ):
if doc.es_index in {"repo"}:
pass
else:
Log.note("not expected")
doc.nested_path = expected_nested_path
doc.last_updated = now
else:
if doc.nested_path != expected_nested_path:
doc.nested_path = expected_nested_path
doc.last_updated = now
# FIX
if last(split_field(doc.es_column)) == EXISTS_TYPE:
if doc.jx_type != EXISTS:
doc.jx_type = EXISTS
doc.last_updated = now
if doc.cardinality == None:
doc.cardinality = 1
doc.last_updated = now
# FIX
if doc.jx_type in STRUCT:
if doc.cardinality not in [0, 1]:
doc.cardinality = 1 # DO NOT KNOW IF EXISTS OR NOT
doc.last_updated = now
return Column(**doc)
except Exception as e:
try:
mark_as_deleted(Column(**doc), now)
except Exception:
pass
return None
def get_selects(query):
schema = query.frum.schema
split_select = {".": ESSelectOp(".")}
def get_select(path):
es_select = split_select.get(path)
if not es_select:
es_select = split_select[path] = ESSelectOp(path)
return es_select
selects = list_to_data([unwrap(s.copy()) for s in listwrap(query.select)])
new_select = FlatList()
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if is_op(select.value, LeavesOp) and is_op(select.value.term,
Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
full_name = concat_field(
select.name, relative_field(untype_path(c.name), term.var))
if c.jx_type == NESTED:
get_select(".").get_source = True
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": ".",
},
"pull": get_pull_source(c.es_column),
})
put_index += 1
else:
get_select(c.nested_path[0]).fields.append(c.es_column)
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": ".",
},
})
put_index += 1
elif is_op(select.value, Variable):
s_column = select.value.var
if s_column == ".":
# PULL ALL SOURCE
get_select(".").get_source = True
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
"pull": get_pull_source("."),
})
continue
leaves = schema.leaves(s_column) # LEAVES OF OBJECT
# nested_selects = {}
if leaves:
if any(c.jx_type == NESTED for c in leaves):
# PULL WHOLE NESTED ARRAYS
get_select(".").get_source = True
for c in leaves:
if len(
c.nested_path
) == 1: # NESTED PROPERTIES ARE IGNORED, CAPTURED BY THESE FIRST LEVEL PROPERTIES
pre_child = join_field(
decode_property(n)
for n in split_field(c.name))
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name":
select.name,
"index":
put_index,
"child":
untype_path(
relative_field(pre_child, s_column)),
},
"pull":
get_pull_source(c.es_column),
})
else:
# PULL ONLY WHAT'S NEEDED
for c in leaves:
c_nested_path = c.nested_path[0]
if c_nested_path == ".":
if c.es_column == "_id":
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child": ".",
},
"pull":
lambda row: row._id,
})
elif c.jx_type == NESTED:
get_select(".").get_source = True
pre_child = join_field(
decode_property(n)
for n in split_field(c.name))
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name":
select.name,
"index":
put_index,
"child":
untype_path(
relative_field(
pre_child, s_column)),
},
"pull":
get_pull_source(c.es_column),
})
else:
get_select(c_nested_path).fields.append(
c.es_column)
pre_child = join_field(
decode_property(n)
for n in split_field(c.name))
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name":
select.name,
"index":
put_index,
"child":
untype_path(
relative_field(
pre_child, s_column)),
},
})
else:
es_select = get_select(c_nested_path)
es_select.fields.append(c.es_column)
child = relative_field(
untype_path(
relative_field(c.name,
schema.query_path[0])),
s_column,
)
pull = accumulate_nested_doc(
c_nested_path,
Variable(
relative_field(
s_column, unnest_path(c_nested_path))),
)
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": child,
},
"pull": pull,
})
else:
new_select.append({
"name": select.name,
"value": Variable("$dummy"),
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
})
put_index += 1
else:
op, split_scripts = split_expression_by_path(select.value,
schema,
lang=Painless)
for p, script in split_scripts.items():
es_select = get_select(p)
es_select.scripts[select.name] = {
"script":
text(Painless[script].partial_eval().to_es_script(schema))
}
new_select.append({
"name":
select.name,
"pull":
jx_expression_to_function("fields." +
literal_field(select.name)),
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
})
put_index += 1
for n in new_select:
if n.pull:
continue
elif is_op(n.value, Variable):
if get_select(".").get_source:
n.pull = get_pull_source(n.value.var)
elif n.value == "_id":
n.pull = jx_expression_to_function("_id")
else:
n.pull = jx_expression_to_function(
concat_field("fields", literal_field(n.value.var)))
else:
Log.error("Do not know what to do")
return new_select, split_select
def es_setop(es, query):
schema = query.frum.schema
es_query, filters = es_query_template(schema.query_path[0])
nested_filter = None
set_default(filters[0], query.where.partial_eval().to_esfilter(schema))
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
es_query.stored_fields = FlatList()
selects = wrap([s.copy() for s in listwrap(query.select)])
new_select = FlatList()
schema = query.frum.schema
# columns = schema.columns
# nested_columns = set(c.names["."] for c in columns if c.nested_path[0] != ".")
es_query.sort = jx_sort_to_es_sort(query.sort, schema)
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if isinstance(select.value, LeavesOp) and isinstance(
select.value.term, Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
full_name = concat_field(
select.name,
relative_field(untype_path(c.names["."]), term.var))
if c.jx_type == NESTED:
es_query.stored_fields = ["_source"]
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": "."
},
"pull": get_pull_source(c.es_column)
})
put_index += 1
elif c.nested_path[0] != ".":
pass # THE NESTED PARENT WILL CAPTURE THIS
else:
es_query.stored_fields += [c.es_column]
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": "."
}
})
put_index += 1
elif isinstance(select.value, Variable):
s_column = select.value.var
# LEAVES OF OBJECT
leaves = schema.leaves(s_column)
nested_selects = {}
if leaves:
if s_column == '.':
# PULL ALL SOURCE
es_query.stored_fields = ["_source"]
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
"pull": get_pull_source(".")
})
elif any(c.jx_type == NESTED for c in leaves):
# PULL WHOLE NESTED ARRAYS
es_query.stored_fields = ["_source"]
for c in leaves:
if len(
c.nested_path
) == 1: # NESTED PROPERTIES ARE IGNORED, CAPTURED BY THESE FIRT LEVEL PROPERTIES
jx_name = untype_path(c.names["."])
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child": relative_field(jx_name, s_column)
},
"pull":
get_pull_source(c.es_column)
})
else:
# PULL ONLY WHAT'S NEEDED
for c in leaves:
if len(c.nested_path) == 1:
jx_name = untype_path(c.names["."])
if c.jx_type == NESTED:
es_query.stored_fields = ["_source"]
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child":
relative_field(jx_name, s_column)
},
"pull":
get_pull_source(c.es_column)
})
else:
es_query.stored_fields += [c.es_column]
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child":
relative_field(jx_name, s_column)
}
})
else:
if not nested_filter:
where = filters[0].copy()
nested_filter = [where]
for k in filters[0].keys():
filters[0][k] = None
set_default(
filters[0],
es_and([where, es_or(nested_filter)]))
nested_path = c.nested_path[0]
if nested_path not in nested_selects:
where = nested_selects[nested_path] = Data()
nested_filter += [where]
where.nested.path = nested_path
where.nested.query.match_all = {}
where.nested.inner_hits._source = False
where.nested.inner_hits.stored_fields += [
c.es_column
]
child = relative_field(
untype_path(c.names[schema.query_path[0]]),
s_column)
pull = accumulate_nested_doc(
nested_path,
Variable(
relative_field(
s_column,
unnest_path(nested_path))))
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": child
},
"pull": pull
})
else:
nested_selects[
nested_path].nested.inner_hits.stored_fields += [
c.es_column
]
else:
new_select.append({
"name": select.name,
"value": Variable("$dummy"),
"put": {
"name": select.name,
"index": put_index,
"child": "."
}
})
put_index += 1
else:
painless = select.value.partial_eval().to_es_script(schema)
es_query.script_fields[literal_field(select.name)] = es_script(
painless.script(schema))
new_select.append({
"name":
select.name,
"pull":
jx_expression_to_function("fields." +
literal_field(select.name)),
"put": {
"name": select.name,
"index": put_index,
"child": "."
}
})
put_index += 1
for n in new_select:
if n.pull:
continue
elif isinstance(n.value, Variable):
if es_query.stored_fields[0] == "_source":
es_query.stored_fields = ["_source"]
n.pull = get_pull_source(n.value.var)
elif n.value == "_id":
n.pull = jx_expression_to_function("_id")
else:
n.pull = jx_expression_to_function(
concat_field("fields", literal_field(n.value.var)))
else:
Log.error("Do not know what to do")
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
T = data.hits.hits
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
with Timer("formatter"):
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def _normalize_group(edge, dim_index, limit, schema=None):
"""
:param edge: Not normalized groupby
:param dim_index: Dimensions are ordered; this is this groupby's index into that order
:param schema: for context
:return: a normalized groupby
"""
if isinstance(edge, text_type):
if edge.endswith(".*"):
prefix = edge[:-2]
if schema:
output = wrap([{
"name":
concat_field(
prefix,
literal_field(
relative_field(untype_path(c.names["."]),
prefix))),
"put": {
"name": literal_field(untype_path(c.names["."]))
},
"value":
jx_expression(c.es_column),
"allowNulls":
True,
"domain": {
"type": "default"
}
} for c in schema.leaves(prefix)])
return output
else:
return wrap([{
"name": untype_path(prefix),
"put": {
"name": literal_field(untype_path(prefix))
},
"value": jx_expression(prefix),
"allowNulls": True,
"dim": dim_index,
"domain": {
"type": "default"
}
}])
return wrap([{
"name": edge,
"value": jx_expression(edge),
"allowNulls": True,
"dim": dim_index,
"domain": {
"type": "default"
}
}])
else:
edge = wrap(edge)
if (edge.domain
and edge.domain.type != "default") or edge.allowNulls != None:
Log.error("groupby does not accept complicated domains")
if not edge.name and not isinstance(edge.value, text_type):
Log.error("You must name compound edges: {{edge}}", edge=edge)
return wrap([{
"name": coalesce(edge.name, edge.value),
"value": jx_expression(edge.value),
"allowNulls": True,
"dim": dim_index,
"domain": {
"type": "default"
}
}])
def _indexer(columns, query_path):
all_names = set(unnest_path(n) for c in columns for n in c.names.values()) | {"."}
lookup_leaves = {} # ALL LEAF VARIABLES
for full_name in all_names:
for c in columns:
cname = c.names[query_path]
nfp = unnest_path(cname)
if (
startswith_field(nfp, full_name) and
c.es_type not in [EXISTS, OBJECT, NESTED] and
(c.es_column != "_id" or full_name == "_id")
):
cs = lookup_leaves.setdefault(full_name, set())
cs.add(c)
cs = lookup_leaves.setdefault(untype_path(full_name), set())
cs.add(c)
lookup_variables = {} # ALL NOT-NESTED VARIABLES
for full_name in all_names:
for c in columns:
cname = c.names[query_path]
nfp = unnest_path(cname)
if (
startswith_field(nfp, full_name) and
c.es_type not in [EXISTS, OBJECT] and
(c.es_column != "_id" or full_name == "_id") and
startswith_field(c.nested_path[0], query_path)
):
cs = lookup_variables.setdefault(full_name, set())
cs.add(c)
cs = lookup_variables.setdefault(untype_path(full_name), set())
cs.add(c)
relative_lookup = {}
for c in columns:
try:
cname = c.names[query_path]
cs = relative_lookup.setdefault(cname, set())
cs.add(c)
ucname = untype_path(cname)
cs = relative_lookup.setdefault(ucname, set())
cs.add(c)
except Exception as e:
Log.error("Should not happen", cause=e)
if query_path != ".":
# ADD ABSOLUTE NAMES TO THE NAMESAPCE
absolute_lookup, more_leaves, more_variables = _indexer(columns, ".")
for k, cs in absolute_lookup.items():
if k not in relative_lookup:
relative_lookup[k] = cs
for k, cs in more_leaves.items():
if k not in lookup_leaves:
lookup_leaves[k] = cs
for k, cs in more_variables.items():
if k not in lookup_variables:
lookup_variables[k] = cs
return relative_lookup, lookup_leaves, lookup_variables
def es_setop(es, query):
schema = query.frum.schema
es_query, filters = es_query_template(schema.query_path[0])
nested_filter = None
set_default(filters[0], query.where.partial_eval().to_esfilter(schema))
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
es_query.stored_fields = FlatList()
selects = wrap([s.copy() for s in listwrap(query.select)])
new_select = FlatList()
schema = query.frum.schema
# columns = schema.columns
# nested_columns = set(c.names["."] for c in columns if c.nested_path[0] != ".")
es_query.sort = jx_sort_to_es_sort(query.sort, schema)
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if isinstance(select.value, LeavesOp) and isinstance(select.value.term, Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
full_name = concat_field(select.name, relative_field(untype_path(c.names["."]), term.var))
if c.jx_type == NESTED:
es_query.stored_fields = ["_source"]
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {"name": literal_field(full_name), "index": put_index, "child": "."},
"pull": get_pull_source(c.es_column)
})
put_index += 1
elif c.nested_path[0] != ".":
pass # THE NESTED PARENT WILL CAPTURE THIS
else:
es_query.stored_fields += [c.es_column]
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {"name": literal_field(full_name), "index": put_index, "child": "."}
})
put_index += 1
elif isinstance(select.value, Variable):
s_column = select.value.var
# LEAVES OF OBJECT
leaves = schema.leaves(s_column)
nested_selects = {}
if leaves:
if s_column == '.':
# PULL ALL SOURCE
es_query.stored_fields = ["_source"]
new_select.append({
"name": select.name,
"value": select.value,
"put": {"name": select.name, "index": put_index, "child": "."},
"pull": get_pull_source(".")
})
elif any(c.jx_type == NESTED for c in leaves):
# PULL WHOLE NESTED ARRAYS
es_query.stored_fields = ["_source"]
for c in leaves:
if len(c.nested_path) == 1: # NESTED PROPERTIES ARE IGNORED, CAPTURED BY THESE FIRT LEVEL PROPERTIES
jx_name = untype_path(c.names["."])
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": relative_field(jx_name, s_column)},
"pull": get_pull_source(c.es_column)
})
else:
# PULL ONLY WHAT'S NEEDED
for c in leaves:
if len(c.nested_path) == 1:
jx_name = untype_path(c.names["."])
if c.jx_type == NESTED:
es_query.stored_fields = ["_source"]
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": relative_field(jx_name, s_column)},
"pull": get_pull_source(c.es_column)
})
else:
es_query.stored_fields += [c.es_column]
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": relative_field(jx_name, s_column)}
})
else:
if not nested_filter:
where = filters[0].copy()
nested_filter = [where]
for k in filters[0].keys():
filters[0][k] = None
set_default(
filters[0],
es_and([where, es_or(nested_filter)])
)
nested_path = c.nested_path[0]
if nested_path not in nested_selects:
where = nested_selects[nested_path] = Data()
nested_filter += [where]
where.nested.path = nested_path
where.nested.query.match_all = {}
where.nested.inner_hits._source = False
where.nested.inner_hits.stored_fields += [c.es_column]
child = relative_field(untype_path(c.names[schema.query_path[0]]), s_column)
pull = accumulate_nested_doc(nested_path, Variable(relative_field(s_column, unnest_path(nested_path))))
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": child
},
"pull": pull
})
else:
nested_selects[nested_path].nested.inner_hits.stored_fields += [c.es_column]
else:
new_select.append({
"name": select.name,
"value": Variable("$dummy"),
"put": {"name": select.name, "index": put_index, "child": "."}
})
put_index += 1
else:
painless = select.value.partial_eval().to_es_script(schema)
es_query.script_fields[literal_field(select.name)] = es_script(painless.script(schema))
new_select.append({
"name": select.name,
"pull": jx_expression_to_function("fields." + literal_field(select.name)),
"put": {"name": select.name, "index": put_index, "child": "."}
})
put_index += 1
for n in new_select:
if n.pull:
continue
elif isinstance(n.value, Variable):
if es_query.stored_fields[0] == "_source":
es_query.stored_fields = ["_source"]
n.pull = get_pull_source(n.value.var)
elif n.value == "_id":
n.pull = jx_expression_to_function("_id")
else:
n.pull = jx_expression_to_function(concat_field("fields", literal_field(n.value.var)))
else:
Log.error("Do not know what to do")
with Timer("call to ES") as call_timer:
data = es_post(es, es_query, query.limit)
T = data.hits.hits
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
with Timer("formatter"):
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def es_setop(es, query):
schema = query.frum.schema
query_path = schema.query_path[0]
split_select = {".": ESSelect('.')}
def get_select(path):
es_select = split_select.get(path)
if not es_select:
es_select = split_select[path] = ESSelect(path)
return es_select
selects = wrap([unwrap(s.copy()) for s in listwrap(query.select)])
new_select = FlatList()
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if is_op(select.value, LeavesOp) and is_op(select.value.term,
Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
full_name = concat_field(
select.name, relative_field(untype_path(c.name), term.var))
if c.jx_type == NESTED:
get_select('.').use_source = True
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": "."
},
"pull": get_pull_source(c.es_column)
})
put_index += 1
else:
get_select(c.nested_path[0]).fields.append(c.es_column)
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {
"name": literal_field(full_name),
"index": put_index,
"child": "."
}
})
put_index += 1
elif is_op(select.value, Variable):
s_column = select.value.var
if s_column == ".":
# PULL ALL SOURCE
get_select('.').use_source = True
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
"pull": get_pull_source(".")
})
continue
leaves = schema.leaves(s_column) # LEAVES OF OBJECT
# nested_selects = {}
if leaves:
if any(c.jx_type == NESTED for c in leaves):
# PULL WHOLE NESTED ARRAYS
get_select('.').use_source = True
for c in leaves:
if len(
c.nested_path
) == 1: # NESTED PROPERTIES ARE IGNORED, CAPTURED BY THESE FIRST LEVEL PROPERTIES
pre_child = join_field(
decode_property(n)
for n in split_field(c.name))
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name":
select.name,
"index":
put_index,
"child":
untype_path(
relative_field(pre_child, s_column))
},
"pull":
get_pull_source(c.es_column)
})
else:
# PULL ONLY WHAT'S NEEDED
for c in leaves:
c_nested_path = c.nested_path[0]
if c_nested_path == ".":
if c.es_column == "_id":
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
"pull":
lambda row: row._id
})
elif c.jx_type == NESTED:
get_select('.').use_source = True
pre_child = join_field(
decode_property(n)
for n in split_field(c.name))
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name":
select.name,
"index":
put_index,
"child":
untype_path(
relative_field(
pre_child, s_column))
},
"pull":
get_pull_source(c.es_column)
})
else:
get_select(c_nested_path).fields.append(
c.es_column)
pre_child = join_field(
decode_property(n)
for n in split_field(c.name))
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name":
select.name,
"index":
put_index,
"child":
untype_path(
relative_field(
pre_child, s_column))
}
})
else:
es_select = get_select(c_nested_path)
es_select.fields.append(c.es_column)
child = relative_field(
untype_path(
relative_field(c.name,
schema.query_path[0])),
s_column)
pull = accumulate_nested_doc(
c_nested_path,
Variable(
relative_field(
s_column, unnest_path(c_nested_path))))
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": child
},
"pull": pull
})
else:
new_select.append({
"name": select.name,
"value": Variable("$dummy"),
"put": {
"name": select.name,
"index": put_index,
"child": "."
}
})
put_index += 1
else:
split_scripts = split_expression_by_path(select.value,
schema,
lang=Painless)
for p, script in split_scripts.items():
es_select = get_select(p)
es_select.scripts[select.name] = {
"script":
text_type(Painless[first(
script)].partial_eval().to_es_script(schema))
}
new_select.append({
"name":
select.name,
"pull":
jx_expression_to_function("fields." +
literal_field(select.name)),
"put": {
"name": select.name,
"index": put_index,
"child": "."
}
})
put_index += 1
for n in new_select:
if n.pull:
continue
elif is_op(n.value, Variable):
if get_select('.').use_source:
n.pull = get_pull_source(n.value.var)
elif n.value == "_id":
n.pull = jx_expression_to_function("_id")
else:
n.pull = jx_expression_to_function(
concat_field("fields", literal_field(n.value.var)))
else:
Log.error("Do not know what to do")
split_wheres = split_expression_by_path(query.where, schema, lang=ES52)
es_query = es_query_proto(query_path, split_select, split_wheres, schema)
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
es_query.sort = jx_sort_to_es_sort(query.sort, schema)
with Timer("call to ES", silent=DEBUG) as call_timer:
data = es_post(es, es_query, query.limit)
T = data.hits.hits
# Log.note("{{output}}", output=T)
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
with Timer("formatter", silent=True):
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def __getitem__(self, column_name):
return [c for c in self.columns if untype_path(c.name) == column_name]
def _indexer(columns, query_path):
all_names = set(unnest_path(c.name) for c in columns) | {"."}
lookup_leaves = {} # ALL LEAF VARIABLES
for full_name in all_names:
for c in columns:
cname = relative_field(c.name, query_path)
nfp = unnest_path(cname)
if (
startswith_field(nfp, full_name) and
c.es_type not in [EXISTS, OBJECT, NESTED] and
(c.es_column != "_id" or full_name == "_id")
):
cs = lookup_leaves.setdefault(full_name, set())
cs.add(c)
cs = lookup_leaves.setdefault(untype_path(full_name), set())
cs.add(c)
lookup_variables = {} # ALL NOT-NESTED VARIABLES
for full_name in all_names:
for c in columns:
cname = relative_field(c.name, query_path)
nfp = unnest_path(cname)
if (
startswith_field(nfp, full_name) and
c.es_type not in [EXISTS, OBJECT] and
(c.es_column != "_id" or full_name == "_id") and
startswith_field(c.nested_path[0], query_path)
):
cs = lookup_variables.setdefault(full_name, set())
cs.add(c)
cs = lookup_variables.setdefault(untype_path(full_name), set())
cs.add(c)
relative_lookup = {}
for c in columns:
try:
cname = relative_field(c.name, query_path)
cs = relative_lookup.setdefault(cname, set())
cs.add(c)
ucname = untype_path(cname)
cs = relative_lookup.setdefault(ucname, set())
cs.add(c)
except Exception as e:
Log.error("Should not happen", cause=e)
if query_path != ".":
# ADD ABSOLUTE NAMES TO THE NAMESAPCE
absolute_lookup, more_leaves, more_variables = _indexer(columns, ".")
for k, cs in absolute_lookup.items():
if k not in relative_lookup:
relative_lookup[k] = cs
for k, cs in more_leaves.items():
if k not in lookup_leaves:
lookup_leaves[k] = cs
for k, cs in more_variables.items():
if k not in lookup_variables:
lookup_variables[k] = cs
return relative_lookup, lookup_leaves, lookup_variables
def es_setop(es, query):
schema = query.frum.schema
query_path = schema.query_path[0]
split_select = {".": ESSelect('.')}
def get_select(path):
es_select = split_select.get(path)
if not es_select:
es_select = split_select[path] = ESSelect(path)
return es_select
selects = wrap([unwrap(s.copy()) for s in listwrap(query.select)])
new_select = FlatList()
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if is_op(select.value, LeavesOp) and is_op(select.value.term, Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
full_name = concat_field(select.name, relative_field(untype_path(c.name), term.var))
if c.jx_type == NESTED:
get_select('.').use_source = True
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {"name": literal_field(full_name), "index": put_index, "child": "."},
"pull": get_pull_source(c.es_column)
})
put_index += 1
else:
get_select(c.nested_path[0]).fields.append(c.es_column)
new_select.append({
"name": full_name,
"value": Variable(c.es_column),
"put": {"name": literal_field(full_name), "index": put_index, "child": "."}
})
put_index += 1
elif is_op(select.value, Variable):
s_column = select.value.var
if s_column == ".":
# PULL ALL SOURCE
get_select('.').use_source = True
new_select.append({
"name": select.name,
"value": select.value,
"put": {"name": select.name, "index": put_index, "child": "."},
"pull": get_pull_source(".")
})
continue
leaves = schema.leaves(s_column) # LEAVES OF OBJECT
# nested_selects = {}
if leaves:
if any(c.jx_type == NESTED for c in leaves):
# PULL WHOLE NESTED ARRAYS
get_select('.').use_source = True
for c in leaves:
if len(c.nested_path) == 1: # NESTED PROPERTIES ARE IGNORED, CAPTURED BY THESE FIRST LEVEL PROPERTIES
pre_child = join_field(decode_property(n) for n in split_field(c.name))
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": untype_path(relative_field(pre_child, s_column))},
"pull": get_pull_source(c.es_column)
})
else:
# PULL ONLY WHAT'S NEEDED
for c in leaves:
c_nested_path = c.nested_path[0]
if c_nested_path == ".":
if c.es_column == "_id":
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": "."},
"pull": lambda row: row._id
})
elif c.jx_type == NESTED:
get_select('.').use_source = True
pre_child = join_field(decode_property(n) for n in split_field(c.name))
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": untype_path(relative_field(pre_child, s_column))},
"pull": get_pull_source(c.es_column)
})
else:
get_select(c_nested_path).fields.append(c.es_column)
pre_child = join_field(decode_property(n) for n in split_field(c.name))
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {"name": select.name, "index": put_index, "child": untype_path(relative_field(pre_child, s_column))}
})
else:
es_select = get_select(c_nested_path)
es_select.fields.append(c.es_column)
child = relative_field(untype_path(relative_field(c.name, schema.query_path[0])), s_column)
pull = accumulate_nested_doc(c_nested_path, Variable(relative_field(s_column, unnest_path(c_nested_path))))
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": put_index,
"child": child
},
"pull": pull
})
else:
new_select.append({
"name": select.name,
"value": Variable("$dummy"),
"put": {"name": select.name, "index": put_index, "child": "."}
})
put_index += 1
else:
split_scripts = split_expression_by_path(select.value, schema, lang=Painless)
for p, script in split_scripts.items():
es_select = get_select(p)
es_select.scripts[select.name] = {"script": text_type(Painless[first(script)].partial_eval().to_es_script(schema))}
new_select.append({
"name": select.name,
"pull": jx_expression_to_function("fields." + literal_field(select.name)),
"put": {"name": select.name, "index": put_index, "child": "."}
})
put_index += 1
for n in new_select:
if n.pull:
continue
elif is_op(n.value, Variable):
if get_select('.').use_source:
n.pull = get_pull_source(n.value.var)
elif n.value == "_id":
n.pull = jx_expression_to_function("_id")
else:
n.pull = jx_expression_to_function(concat_field("fields", literal_field(n.value.var)))
else:
Log.error("Do not know what to do")
split_wheres = split_expression_by_path(query.where, schema, lang=ES52)
es_query = es_query_proto(query_path, split_select, split_wheres, schema)
es_query.size = coalesce(query.limit, DEFAULT_LIMIT)
es_query.sort = jx_sort_to_es_sort(query.sort, schema)
with Timer("call to ES", silent=True) as call_timer:
data = es_post(es, es_query, query.limit)
T = data.hits.hits
# Log.note("{{output}}", output=T)
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
with Timer("formatter", silent=True):
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def get_selects(query):
schema = query.frum.schema
query_level = len(schema.query_path)
query_path = schema.query_path[0]
# SPLIT select INTO ES_SELECT AND RESULTSET SELECT
split_select = OrderedDict((p, ESSelectOp(p)) for p in schema.query_path)
def expand_split_select(c_nested_path):
es_select = split_select.get(c_nested_path)
if not es_select:
temp = [(k, v) for k, v in split_select.items()]
split_select.clear()
split_select.update({c_nested_path: ESSelectOp(c_nested_path)})
split_select.update(temp)
return split_select[c_nested_path]
new_select = FlatList()
post_expressions = {}
selects = list_to_data([unwrap(s.copy()) for s in listwrap(query.select)])
# WHAT PATH IS _source USED, IF ANY?
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if is_op(select.value, LeavesOp) and is_op(select.value.term,
Variable):
term = select.value.term
leaves = schema.leaves(term.var)
if any(c.jx_type == NESTED for c in leaves):
split_select["."].source_path = "."
elif is_op(select.value, Variable):
for selected_column in schema.values(select.value.var,
exclude_type=(OBJECT,
EXISTS)):
if selected_column.jx_type == NESTED:
expand_split_select(
selected_column.es_column
).source_path = selected_column.es_column
continue
leaves = schema.leaves(selected_column.es_column)
for c in leaves:
if c.jx_type == NESTED:
split_select[c.es_column].source_path = c.es_column
# IF WE GET THE SOURCE FOR PARENT, WE ASSUME WE GOT SOURCE FOR CHILD
source_path = None
source_level = 0
for level, es_select in enumerate(reversed(list(split_select.values()))):
if source_path:
es_select.source_path = source_path
elif es_select.source_path:
source_level = level + 1
source_path = es_select.source_path
def get_pull_source(c):
nested_path = c.nested_path
nested_level = len(nested_path)
pos = text(nested_level)
if nested_level <= query_level:
if not source_level or nested_level < source_level:
field = join_field([pos, "fields", c.es_column])
return jx_expression_to_function(field)
elif nested_level == source_level:
field = relative_field(c.es_column, nested_path[0])
def pull_source(row):
return untyped(row.get(pos, Null)._source[field])
return pull_source
else:
field = relative_field(c.es_column, nested_path[0])
def pull_property(row):
return untyped(row.get(pos, Null)[field])
return pull_property
else:
pos = text(query_level)
if not source_level or nested_level < source_level:
# PULL FIELDS AND THEN AGGREGATE THEM
value = jx_expression_to_function(
join_field(["fields", c.es_column]))
name = literal_field(nested_path[0])
index = jx_expression_to_function("_nested.offset")
def pull_nested_field(doc):
hits = doc.get(pos, Null).inner_hits[name].hits.hits
if not hits:
return []
temp = [(index(h), value(h)) for h in hits]
acc = [None] * len(temp)
for i, v in temp:
acc[i] = unwraplist(v)
return acc
return pull_nested_field
else:
# PULL SOURCES
value = jx_expression_to_function(
concat_field("_source",
relative_field(c.es_column, nested_path[0])))
name = literal_field(nested_path[0])
index = jx_expression_to_function(
join_field(["_nested"] * (len(c.nested_path) - 1) +
["offset"]))
def pull_nested_source(doc):
hits = doc.get(pos, Null).inner_hits[name].hits.hits
if not hits:
return []
temp = [(index(h), value(h)) for h in hits]
acc = [None] * len(temp)
for i, v in temp:
acc[i] = untyped(v)
return acc
return pull_nested_source
put_index = 0
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if is_op(select.value, LeavesOp) and is_op(select.value.term,
Variable):
term = select.value.term
leaves = schema.leaves(term.var)
for c in leaves:
c_nested_path = c.nested_path[0]
simple_name = relative_field(c.es_column,
query_path).lstrip(".")
name = concat_field(select.name, untype_path(simple_name))
put_name = concat_field(
select.name, literal_field(untype_path(simple_name)))
split_select[c_nested_path].fields.append(c.es_column)
new_select.append({
"name": name,
"value": Variable(c.es_column),
"put": {
"name": put_name,
"index": put_index,
"child": ".",
},
"pull": get_pull_source(c),
})
put_index += 1
elif is_op(select.value, Variable):
if select.value.var == ".":
# PULL ALL SOURCE
new_select.append({
"name":
select.name,
"value":
select.value,
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
"pull":
get_pull_source(
Data(es_column=query_path,
nested_path=schema.query_path)),
})
continue
for selected_column in schema.values(select.value.var,
exclude_type=(EXISTS,
OBJECT)):
if selected_column.jx_type == NESTED:
new_select.append({
"name":
select.name,
"value":
select.value,
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
"pull":
get_pull_source(
Data(
es_column=selected_column.es_column,
nested_path=(selected_column.es_column, ) +
selected_column.nested_path,
)),
})
continue
leaves = schema.leaves(
selected_column.es_column,
exclude_type=INTERNAL) # LEAVES OF OBJECT
if leaves:
for c in leaves:
if c.es_column == "_id":
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child": ".",
},
"pull": pull_id,
})
continue
c_nested_path = c.nested_path[0]
expand_split_select(c_nested_path).fields.append(
c.es_column)
child = untype_path(
relative_field(
c.es_column,
selected_column.es_column,
))
new_select.append({
"name": select.name,
"value": Variable(c.es_column),
"put": {
"name": select.name,
"index": put_index,
"child": child,
},
"pull": get_pull_source(c),
})
else:
new_select.append({
"name": select.name,
"value": NULL,
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
})
put_index += 1
else:
op, split_scripts = split_expression_by_path(select.value,
schema,
lang=Painless)
for p, script in split_scripts.items():
es_select = split_select[p]
es_select.scripts[select.name] = {
"script":
text(Painless[script].partial_eval().to_es_script(schema))
}
new_select.append({
"name":
select.name,
"pull":
jx_expression_to_function(
join_field([
text(p),
"fields",
select.name,
])),
"put": {
"name": select.name,
"index": put_index,
"child": "."
},
})
put_index += 1
def inners(query_path, parent_pos):
"""
:param query_path:
:return: ITERATOR OVER TUPLES ROWS AS TUPLES, WHERE row[len(nested_path)] HAS INNER HITS
AND row[0] HAS post_expressions
"""
pos = text(int(parent_pos) + 1)
if not query_path:
def base_case(row):
extra = {}
for k, e in post_expressions.items():
extra[k] = e(row)
row["0"] = extra
yield row
return base_case
if pos == "1":
more = inners(query_path[:-1], "1")
def first_case(results):
for result in results:
for hit in result.hits.hits:
seed = {"0": None, pos: hit}
for row in more(seed):
yield row
return first_case
else:
more = inners(query_path[:-1], pos)
if source_path and source_path < query_path[-1]:
rel_path = relative_field(query_path[-1], source_path)
def source(acc):
for inner_row in acc[parent_pos][rel_path]:
acc[pos] = inner_row
for tt in more(acc):
yield tt
return source
else:
path = literal_field(query_path[-1])
def recurse(acc):
hits = acc[parent_pos].inner_hits[path].hits.hits
if hits:
for inner_row in hits:
acc[pos] = inner_row
for tt in more(acc):
yield tt
else:
for tt in more(acc):
yield tt
return recurse
return new_select, split_select, inners(schema.query_path, "0")
