def _convert_query(self, query):
output = QueryOp(None)
output.select = self._convert_clause(query.select)
output.where = self.convert(query.where)
output.frum = self._convert_from(query.frum)
output.edges = convert_list(self._convert_edge, query.edges)
output.window = convert_list(self._convert_window, query.window)
output.sort = self._convert_clause(query.sort)
output.format = query.format
return output
def split_expression(where, query):
"""
:param where: EXPRESSION TO CONVERT TO MULTIPLE ES QUERIES
:param query: FOR CONTEXT (WHAT IS INCLUDED)
:return: ConcatOp(InnerOp(NestedOp())))
"""
query = QueryOp(frum=query.frum, where=AndOp([where, query.where]))
all_paths, split_decoders, var_to_columns = pre_process(query)
return setop_to_inner_joins(query, all_paths, {}, var_to_columns)
def query(self, _query):
try:
query = QueryOp.wrap(_query,
container=self,
namespace=self.namespace)
self.stats.record(query)
for s in listwrap(query.select):
if s.aggregate != None and not aggregates.get(s.aggregate):
Log.error(
"ES can not aggregate {{name}} because {{aggregate|quote}} is"
" not a recognized aggregate",
name=s.name,
aggregate=s.aggregate,
)
frum = query["from"]
if is_op(frum, QueryOp):
result = self.query(frum)
q2 = query.copy()
q2.frum = result
return jx.run(q2)
if is_bulk_agg(self.es, query):
return es_bulkaggsop(self, frum, query)
if is_bulk_set(self.es, query):
return es_bulksetop(self, frum, query)
query.limit = temper_limit(query.limit, query)
if is_aggsop(self.es, query):
return es_aggsop(self.es, frum, query)
if is_setop(self.es, query):
return es_setop(self.es, query)
Log.error("Can not handle")
except Exception as cause:
cause = Except.wrap(cause)
if "Data too large, data for" in cause:
http.post(self.es.cluster.url / "_cache/clear")
Log.error("Problem (Tried to clear Elasticsearch cache)",
cause)
Log.error("problem", cause=cause)
def _convert_query(self, query):
# if not isinstance(query["from"], Container):
# Log.error('Expecting from clause to be a Container')
query = to_data(query)
output = QueryOp(None)
output["from"] = self._convert_from(query["from"])
output.format = query.format
if query.select:
output.select = convert_list(self._convert_select, query.select)
else:
if query.edges or query.groupby:
output.select = {
"name": "count",
"value": ".",
"aggregate": "count",
"default": 0
}
else:
output.select = {
"name": "__all__",
"value": "*",
"aggregate": "none"
}
if query.groupby and query.edges:
Log.error(
"You can not use both the `groupby` and `edges` clauses in the same query!"
)
elif query.edges:
output.edges = convert_list(self._convert_edge, query.edges)
output.groupby = None
elif query.groupby:
output.edges = None
output.groupby = convert_list(self._convert_group, query.groupby)
else:
output.edges = []
output.groupby = None
output.where = self.convert(query.where)
output.window = convert_list(self._convert_window, query.window)
output.sort = self._convert_sort(query.sort)
output.limit = coalesce(query.limit, DEFAULT_LIMIT)
if not mo_math.is_integer(output.limit) or output.limit < 0:
Log.error("Expecting limit >= 0")
# DEPTH ANALYSIS - LOOK FOR COLUMN REFERENCES THAT MAY BE DEEPER THAN
# THE from SOURCE IS.
vars = get_all_vars(
output, exclude_where=True) # WE WILL EXCLUDE where VARIABLES
for c in query.columns:
if c.name in vars and len(c.nested_path) != 1:
Log.error("This query, with variable {{var_name}} is too deep",
var_name=c.name)
return output
def run(query, container=Null):
"""
THIS FUNCTION IS SIMPLY SWITCHING BASED ON THE query["from"] CONTAINER,
BUT IT IS ALSO PROCESSING A list CONTAINER; SEPARATE TO A ListContainer
"""
if container == None:
container = to_data(query)["from"]
query_op = QueryOp.wrap(query,
container=container,
namespace=container.schema)
else:
query_op = QueryOp.wrap(query,
container=container,
namespace=container.namespace)
if container == None:
from jx_python.containers.list import DUAL
return DUAL.query(query_op)
elif isinstance(container, Container):
return container.query(query_op)
elif is_many(container):
container = ListContainer(name=None, data=list(container))
elif isinstance(container, Cube):
if is_aggs(query_op):
return cube_aggs(container, query_op)
elif is_op(container, QueryOp):
container = run(container)
elif is_data(container):
query = container
container = query["from"]
container = run(QueryOp.wrap(query, container, container.namespace),
container)
else:
Log.error("Do not know how to handle {{type}}",
type=container.__class__.__name__)
if is_aggs(query_op):
container = list_aggs(container, query_op)
else: # SETOP
if query_op.where is not TRUE:
container = filter(container, query_op.where)
if query_op.sort:
container = sort(container, query_op.sort, already_normalized=True)
if query_op.select:
container = select(container, query_op.select)
if query_op.window:
if isinstance(container, Cube):
container = list(container.values())
for param in query_op.window:
window(container, param)
# AT THIS POINT frum IS IN LIST FORMAT, NOW PACKAGE RESULT
if query_op.format == "cube":
container = list2cube(container)
elif query_op.format == "table":
container = list2table(container)
container.meta.format = "table"
else:
container = dict_to_data({
"meta": {
"format": "list"
},
"data": container
})
return container