def query(self, query, stacked=False):
"""
TRANSLATE qb QUERY ON SINGLE TABLE TO SQL QUERY
"""
from pyLibrary.queries.query import Query
query = Query(query)
sql, post = self._subquery(query, isolate=False, stacked=stacked)
query.data = post(sql)
return query.data
def _convert_query(self, query):
output = Query(None)
output.select = self._convert_clause(query.select)
output.where = self.convert(query.where)
output.frum = self._convert_from(query.frum)
output.edges = convert_list(self._convert_edge, query.edges)
output.having = convert_list(self._convert_having, query.having)
output.window = convert_list(self._convert_window, query.window)
output.sort = self._convert_clause(query.sort)
output.format = query.format
return output
def wrap_from(frum, schema=None):
"""
:param frum:
:param schema:
:return:
"""
if not type2container:
_delayed_imports()
frum = wrap(frum)
if isinstance(frum, basestring):
if not config.default.settings:
Log.error(
"expecting pyLibrary.queries.query.config.default.settings to contain default elasticsearch connection info"
)
settings = set_default({
"index": split_field(frum)[0],
"name": frum,
}, config.default.settings)
settings.type = None # WE DO NOT WANT TO INFLUENCE THE TYPE BECAUSE NONE IS IN THE frum STRING ANYWAY
return type2container["elasticsearch"](settings)
elif isinstance(frum, Mapping) and frum.type and type2container[frum.type]:
# TODO: Ensure the frum.name is set, so we capture the deep queries
if not frum.type:
Log.error("Expecting from clause to have a 'type' property")
return type2container[frum.type](frum.settings)
elif isinstance(frum, Mapping) and (frum["from"]
or isinstance(frum["from"],
(list, set))):
from pyLibrary.queries.query import Query
return Query(frum, schema=schema)
else:
return frum
def query(self, _query):
try:
if not self.ready:
Log.error("Must use with clause for any instance of FromES")
query = Query(_query, schema=self)
# try:
# frum = self.get_columns(query["from"])
# mvel = _MVEL(frum)
# except Exception, e:
# mvel = None
# Log.warning("TODO: Fix this", e)
#
for s in listwrap(query.select):
if not aggregates1_4[s.aggregate]:
Log.error("ES can not aggregate " + self.select[0].name +
" because '" + self.select[0].aggregate +
"' is not a recognized aggregate")
frum = query["from"]
if isinstance(frum, Query):
result = self.query(frum)
q2 = query.copy()
q2.frum = result
return qb.run(q2)
if is_aggsop(self._es, query):
return es_aggsop(self._es, frum, query)
if is_fieldop(self._es, query):
return es_fieldop(self._es, query)
if is_setop(self._es, query):
return es_setop(self._es, query)
Log.error("Can not handle")
except Exception, e:
e = Except.wrap(e)
if "Data too large, data for" in e:
http.post(self._es.cluster.path + "/_cache/clear")
Log.error("Problem (Tried to clear Elasticsearch cache)", e)
Log.error("problem", e)
def query(self, _query):
try:
query = Query(_query, schema=self)
for n in self.namespaces:
query = n.convert(query)
if self.typed:
query = Typed().convert(query)
for s in listwrap(query.select):
if not aggregates1_4.get(s.aggregate):
Log.error("ES can not aggregate " + s.name + " because '" + s.aggregate + "' is not a recognized aggregate")
frum = query["from"]
if isinstance(frum, Query):
result = self.query(frum)
q2 = query.copy()
q2.frum = result
return qb.run(q2)
if is_deepop(self._es, query):
return es_deepop(self._es, query)
if is_aggsop(self._es, query):
return es_aggsop(self._es, frum, query)
if is_setop(self._es, query):
return es_setop(self._es, query)
if es09_setop.is_setop(query):
return es09_setop.es_setop(self._es, None, query)
if es09_aggop.is_aggop(query):
return es09_aggop.es_aggop(self._es, None, query)
Log.error("Can not handle")
except Exception, e:
e = Except.wrap(e)
if "Data too large, data for" in e:
http.post(self._es.cluster.path+"/_cache/clear")
Log.error("Problem (Tried to clear Elasticsearch cache)", e)
Log.error("problem", e)
def run(query):
"""
THIS FUNCTION IS SIMPLY SWITCHING BASED ON THE query["from"] CONTAINER,
BUT IT IS ALSO PROCESSING A list CONTAINER; SEPARATE TO A ListContainer
"""
query = Query(query)
frum = query["from"]
if isinstance(frum, Container):
with frum:
return frum.query(query)
elif isinstance(frum, (list, set, GeneratorType)):
frum = wrap(list(frum))
elif isinstance(frum, Cube):
if is_aggs(query):
return cube_aggs(frum, query)
elif isinstance(frum, Query):
frum = run(frum).data
else:
Log.error("Do not know how to handle {{type}}",
type=frum.__class__.__name__)
if is_aggs(query):
frum = list_aggs(frum, query)
else: # SETOP
try:
if query.filter != None or query.esfilter != None:
Log.error("use 'where' clause")
except AttributeError, e:
pass
if query.where is not TRUE_FILTER:
frum = filter(frum, query.where)
if query.sort:
frum = sort(frum, query.sort)
if query.select:
frum = select(frum, query.select)
def _convert_query(self, query):
# if not isinstance(query["from"], Container):
# Log.error('Expecting from clause to be a Container')
query = wrap(query)
output = Query()
output["from"] = self._convert_from(query["from"])
output.format = query.format
if query.select:
output.select = convert_list(self._convert_select, query.select)
else:
if query.edges or query.groupby:
output.select = {"name": "count", "value": ".", "aggregate": "count"}
else:
output.select = {"name": "__all__", "value": "*", "aggregate": "none"}
if query.groupby and query.edges:
Log.error("You can not use both the `groupby` and `edges` clauses in the same query!")
elif query.edges:
output.edges = convert_list(self._convert_edge, query.edges)
output.groupby = None
elif query.groupby:
output.edges = None
output.groupby = convert_list(self._convert_group, query.groupby)
else:
output.edges = []
output.groupby = None
output.where = self.convert(query.where)
output.window = convert_list(self._convert_window, query.window)
output.sort = self._convert_sort(query.sort)
output.limit = coalesce(query.limit, DEFAULT_LIMIT)
if not Math.is_integer(output.limit) or output.limit < 0:
Log.error("Expecting limit >= 0")
output.isLean = query.isLean
# DEPTH ANALYSIS - LOOK FOR COLUMN REFERENCES THAT MAY BE DEEPER THAN
# THE from SOURCE IS.
vars = get_all_vars(output, exclude_where=True) # WE WILL EXCLUDE where VARIABLES
for c in query.columns:
if c.name in vars and c.nested_path:
Log.error("This query, with variable {{var_name}} is too deep", var_name=c.name)
output.having = convert_list(self._convert_having, query.having)
return output
