def to_esfilter(self, schema):
if isinstance(self.lhs, Variable) and isinstance(self.rhs, Literal):
columns = schema.values(self.lhs.var)
if len(columns) == 0:
return {"match_all": {}}
elif len(columns) == 1:
return es_not({"term": {columns[0].es_column: self.rhs.value}})
else:
Log.error("column split to multiple, not handled")
else:
lhs = self.lhs.partial_eval().to_es_script(schema)
rhs = self.rhs.partial_eval().to_es_script(schema)
if lhs.many:
if rhs.many:
return es_not(
ScriptOp(
"script",
("(" + lhs.expr + ").size()==(" + rhs.expr +
").size() && " + "(" + rhs.expr + ").containsAll(" +
lhs.expr + ")")).to_esfilter(schema))
else:
return es_not(
ScriptOp("script", "(" + lhs.expr + ").contains(" +
rhs.expr + ")").to_esfilter(schema))
else:
if rhs.many:
return es_not(
ScriptOp("script", "(" + rhs.expr + ").contains(" +
lhs.expr + ")").to_esfilter(schema))
else:
return es_not(
ScriptOp("script", "(" + lhs.expr + ") != (" + rhs.expr +
")").to_esfilter(schema))
def to_esfilter(self, schema):
if isinstance(self.term, MissingOp) and isinstance(self.term.expr, Variable):
v = self.term.expr.var
cols = schema.leaves(v)
if cols:
v = cols[0].es_column
return {"exists": {"field": v}}
else:
operand = self.term.to_esfilter(schema)
return es_not(operand)
def to_esfilter(self, schema):
if not isinstance(self.lhs, Variable) or not isinstance(self.rhs, Literal) or self.op in BinaryOp.operators:
return self.to_es_script(schema).to_esfilter(schema)
if self.op in ["eq", "term"]:
return {"term": {self.lhs.var: self.rhs.to_esfilter(schema)}}
elif self.op in ["ne", "neq"]:
return es_not({"term": {self.lhs.var: self.rhs.to_esfilter(schema)}})
elif self.op in BinaryOp.ineq_ops:
return {"range": {self.lhs.var: {self.op: self.rhs.value}}}
else:
Log.error("Logic error")
def to_esfilter(self, schema):
if isinstance(self.lhs, Variable) and isinstance(self.rhs, Literal):
columns = schema.values(self.lhs.var)
if len(columns) == 0:
return {"match_all": {}}
elif len(columns) == 1:
return es_not({"term": {columns[0].es_column: self.rhs.value}})
else:
Log.error("column split to multiple, not handled")
else:
lhs = self.lhs.partial_eval().to_es_script(schema)
rhs = self.rhs.partial_eval().to_es_script(schema)
if lhs.many:
if rhs.many:
return es_not(
ScriptOp(
"script",
(
"(" + lhs.expr + ").size()==(" + rhs.expr + ").size() && " +
"(" + rhs.expr + ").containsAll(" + lhs.expr + ")"
)
).to_esfilter(schema)
)
else:
return es_not(
ScriptOp("script", "(" + lhs.expr + ").contains(" + rhs.expr + ")").to_esfilter(schema)
)
else:
if rhs.many:
return es_not(
ScriptOp("script", "(" + rhs.expr + ").contains(" + lhs.expr + ")").to_esfilter(schema)
)
else:
return es_not(
ScriptOp("script", "(" + lhs.expr + ") != (" + rhs.expr + ")").to_esfilter(schema)
)
def to_esfilter(self, schema):
if is_op(self.lhs, Variable_) and is_literal(self.rhs):
columns = schema.values(self.lhs.var)
if len(columns) == 0:
return MATCH_ALL
elif len(columns) == 1:
return es_not(
{"term": {
first(columns).es_column: self.rhs.value
}})
else:
Log.error("column split to multiple, not handled")
else:
lhs = self.lhs.partial_eval().to_es_script(schema)
rhs = self.rhs.partial_eval().to_es_script(schema)
if lhs.many:
if rhs.many:
return es_not(
ScriptOp(("(" + lhs.expr + ").size()==(" + rhs.expr +
").size() && " + "(" + rhs.expr +
").containsAll(" + lhs.expr +
")")).to_esfilter(schema))
else:
return es_not(
ScriptOp("(" + lhs.expr + ").contains(" + rhs.expr +
")").to_esfilter(schema))
else:
if rhs.many:
return es_not(
ScriptOp("(" + rhs.expr + ").contains(" + lhs.expr +
")").to_esfilter(schema))
else:
return es_not(
ScriptOp("(" + lhs.expr + ") != (" + rhs.expr +
")").to_esfilter(schema))
def to_esfilter(self, schema):
# TODO: REPLICATE THIS WHOLE expression.py SO IT IS CLEAR ES5 QUERIES ARE A BIT DIFFERENT
if schema.snowflake.namespace.es_cluster.version.startswith("5."):
# VERSION 5.2.x
# WE REQUIRE EXIT-EARLY SEMANTICS, OTHERWISE EVERY EXPRESSION IS A SCRIPT EXPRESSION
# {"bool":{"should" :[a, b, c]}} RUNS IN PARALLEL
# {"bool":{"must_not":[a, b, c]}} ALSO RUNS IN PARALLEL
# OR(x) == NOT(AND(NOT(xi) for xi in x))
output = es_not(es_and([
NotOp("not", t).partial_eval().to_esfilter(schema)
for t in self.terms
]))
return output
else:
# VERSION 6.2
return es_or([t.partial_eval().to_esfilter(schema) for t in self.terms])
def to_esfilter(self, schema):
if is_op(self.term, MissingOp_) and is_op(self.term.expr, Variable_):
# PREVENT RECURSIVE LOOP
v = self.term.expr.var
cols = schema.values(v, (OBJECT, NESTED))
if len(cols) == 0:
return MATCH_NONE
elif len(cols) == 1:
return {"exists": {"field": first(cols).es_column}}
else:
return es_and([{
"exists": {
"field": c.es_column
}
} for c in cols])
else:
operand = ES52[self.term].to_esfilter(schema)
return es_not(operand)
def to_esfilter(self, schema):
if is_literal(self.prefix) and not self.prefix.value:
return MATCH_ALL
expr = self.expr
if expr is NULL:
return es_not(MATCH_ALL)
elif not expr:
return MATCH_ALL
if is_op(expr, StringOp_):
expr = expr.term
if is_op(expr, Variable_) and is_literal(self.prefix):
var = first(schema.leaves(expr.var)).es_column
return {"prefix": {var: self.prefix.value}}
else:
return PainlessPrefixOp.to_es_script(self,
schema).to_esfilter(schema)
def to_esfilter(self, schema):
if schema.snowflake.namespace.es_cluster.version.startswith("5."):
# VERSION 5.2.x
# WE REQUIRE EXIT-EARLY SEMANTICS, OTHERWISE EVERY EXPRESSION IS A SCRIPT EXPRESSION
# {"bool":{"should" :[a, b, c]}} RUNS IN PARALLEL
# {"bool":{"must_not":[a, b, c]}} ALSO RUNS IN PARALLEL
# OR(x) == NOT(AND(NOT(xi) for xi in x))
output = es_not(
es_and([
NotOp(t).partial_eval().to_esfilter(schema)
for t in self.terms
]))
return output
else:
# VERSION 6.2+
return es_or([
ES52[t].partial_eval().to_esfilter(schema) for t in self.terms
])
def to_esfilter(self, schema):
if not is_op(self.lhs, Variable_) or not is_literal(self.rhs):
return self.to_es_script(schema).to_esfilter(schema)
return es_not({"term": {self.lhs.var: self.rhs.to_esfilter(schema)}})
def to_esfilter(self, schema):
return es_not({"match_all": {}})
def to_es_script(self, schema):
v = StringOp("string", self.value).partial_eval().to_es_script(schema).expr
start = IntegerOp("string", self.start).partial_eval().to_es_script(schema).expr
end = IntegerOp("integer", self.end).partial_eval().to_es_script(schema).expr
return EsScript(
miss=FALSE,
type=STRING,
expr="(" + v + ").substring(" + start + ", " + end + ")",
frum=self
)
MATCH_ALL = wrap({"match_all": {}})
MATCH_NONE = es_not({"match_all": {}})
def simplify_esfilter(esfilter):
try:
output = wrap(_normalize(wrap(esfilter)))
output.isNormal = None
return output
except Exception as e:
from mo_logs import Log
Log.unexpected("programmer error", cause=e)
def _normalize(esfilter):
"""
