return self
def get_schema(self, name):
if self.name != name:
Log.error("This container only has table by name of {{name}}",
name=name)
return self.schema
def get_table(self, name):
if self is name or self.name == name:
return self
Log.error("This container only has table by name of {{name}}",
name=name)
def _exec(code):
try:
temp = None
exec("temp = " + code)
return temp
except Exception as e:
Log.error("Could not execute {{code|quote}}", code=code, cause=e)
DUAL = ListContainer(name="dual",
data=[{}],
schema=Schema(table_name="dual",
columns=UniqueIndex(keys=("name", ))))
export("jx_base.container", ListContainer)
from mo_imports import export
from tests.modname_name_value1.bars import bar
def foo():
bar()
export("tests.modname_name_value1.bars", "alt", foo)
},
"else": True,
},
{
"when": {
"gte": [{
"count": "nested_path"
}, 2]
},
"then": {
"ne": [
{
"first": {
"right": {
"nested_path": 2
}
}
},
{
"literal": "."
},
]
}, # SECOND-LAST ELEMENT
"else": True,
},
]
},
)
export("jx_base.expressions.query_op", Column)
# GET THE MACHINE METADATA
machine_metadata = dict_to_data({
"pid":
os.getpid(),
"python":
text(platform.python_implementation()),
"os":
text(platform.system() + platform.release()).strip(),
"name":
text(platform.node())
})
def raise_from_none(e):
raise e
if PY3:
exec("def raise_from_none(e):\n raise e from None\n", globals(),
locals())
export("mo_logs.startup", Log)
export("mo_logs.log_usingFile", Log)
export("mo_logs.log_usingMulti", Log)
export("mo_logs.log_usingThread", Log)
if not Log.main_log:
Log.main_log = StructuredLogger_usingStream(STDOUT)
if self.is_value:
return str(self.data)
else:
return str(self.data)
def __int__(self):
if self.is_value:
return int(self.value)
else:
return int(self.data)
def __float__(self):
if self.is_value:
v = self.value
if v == None:
return v
else:
return float(v)
else:
return float(self.data)
def __data__(self):
return Data(select=self.select,
edges=self.edges,
data={k: v.cube
for k, v in self.data.items()},
meta=self.meta)
export("jx_base.container", Cube)
self.term = term
def __data__(self):
return {"not": self.term.__data__()}
def __eq__(self, other):
if not is_op(other, NotOp):
return False
return self.term == other.term
def vars(self):
return self.term.vars()
def map(self, map_):
return self.lang[NotOp(self.term.map(map_))]
def missing(self):
return self.lang[self.term].missing()
def invert(self):
return self.lang[self.term].partial_eval()
@simplified
def partial_eval(self):
return self.lang[self.term].invert()
export("jx_base.expressions.and_op", NotOp)
export("jx_base.expressions.exists_op", NotOp)
export("jx_base.expressions.expression", NotOp)
elif when is FALSE:
pass
else:
whens.append(
WhenOp(when, **{"then": w.then.partial_eval(lang)}))
else:
whens.append((self.whens[-1]).partial_eval(lang))
if len(whens) == 1:
return whens[0]
elif len(whens) == 2:
return WhenOp(whens[0].when, **{
"then": whens[0].then,
"else": whens[1]
})
else:
return CaseOp(whens)
@property
def type(self):
types = set(w.then.type if is_op(w, WhenOp) else w.type
for w in self.whens)
if len(types) > 1:
return OBJECT
else:
return first(types)
export("jx_base.expressions.eq_op", CaseOp)
export("jx_base.expressions.first_op", CaseOp)
return FALSE
def __call__(self, row=None, rownum=None, rows=None):
return Null
def __unicode__(self):
return "null"
def __str__(self):
return b"null"
@property
def type(self):
return IS_NULL
def __hash__(self):
return id(None)
def __bool__(self):
Log.error(
"Detecting truthiness of NullOp is too confusing to be allowed")
NULL = NullOp()
TYPE_ORDER[NullOp] = 9
TYPE_ORDER[NULL] = 9
export("jx_base.expressions._utils", NULL)
export("jx_base.expressions.literal", NULL)
export("jx_base.expressions.expression", NULL)
es_and = expect("es_and")
class OrOp(OrOp_):
def to_es(self, schema):
if schema.snowflake.namespace.es_cluster.version.startswith("5."):
# VERSION 5.2.x
# WE REQUIRE EXIT-EARLY SEMANTICS, OTHERWISE EVERY EXPRESSION IS A SCRIPT EXPRESSION
# {"bool":{"should" :[a, b, c]}} RUNS IN PARALLEL
# {"bool":{"must_not":[a, b, c]}} ALSO RUNS IN PARALLEL
# OR(x) == NOT(AND(NOT(xi) for xi in x))
output = es_not(
es_and([
NotOp(t).partial_eval().to_es(schema) for t in self.terms
]))
return output
else:
# VERSION 6.2+
return es_or(
[ES52[t].partial_eval().to_es(schema) for t in self.terms])
def es_or(terms):
return dict_to_data({"bool": {"should": terms}})
export("jx_elasticsearch.es52.expressions.not_op", es_or)
export("jx_elasticsearch.es52.expressions.utils", OrOp)
"""
sum = 0
for v in vals:
yield sum, v
sum += v
def reverse(vals):
# TODO: Test how to do this fastest
if not hasattr(vals, "len"):
vals = list(vals)
l = len(vals)
output = [None] * l
for v in unwrap(vals):
l -= 1
output[l] = v
return to_data(output)
def countdown(vals):
remaining = len(vals) - 1
return [(remaining - i, v) for i, v in enumerate(vals)]
from jx_python.lists.aggs import is_aggs, list_aggs
export("jx_base.container", run)
from jx_base.expressions import OrOp as OrOp_
from jx_bigquery.expressions._utils import BQLang, check
from jx_bigquery.sql import SQL_OR, sql_iso, JoinSQL
from mo_dots import wrap
from mo_imports import export
class OrOp(OrOp_):
@check
def to_bq(self, schema, not_null=False, boolean=False):
return wrap(
[
{
"name": ".",
"sql": {
"b": JoinSQL(
SQL_OR,
[
sql_iso(BQLang[t].to_bq(schema, boolean=True)[0].sql.b)
for t in self.terms
],
)
},
}
]
)
export("jx_bigquery.expressions._utils", OrOp)
def map(self, map_):
return self
def missing(self, lang):
return FALSE
def invert(self, lang):
return TRUE
def __call__(self, row=None, rownum=None, rows=None):
return False
def __unicode__(self):
return "false"
def __str__(self):
return b"false"
def __bool__(self):
return False
def __nonzero__(self):
return False
FALSE = FalseOp()
export("jx_base.expressions._utils", FALSE)
export("jx_base.expressions.expression", FALSE)
export("jx_base.expressions.literal", FALSE)
# encoding: utf-8 # # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions import FalseOp, extend from mo_imports import export @extend(FalseOp) def to_es(self, schema): return MATCH_NONE MATCH_NONE = {"bool": {"must_not": {"match_all": {}}}} export("jx_elasticsearch.es52.expressions.utils", MATCH_NONE)
else:
obj[path0] = old_value = {}
_assign_to_null(old_value, path[1:], value)
except Exception as e:
raise e
def _split_field(field):
"""
SIMPLE SPLIT, NO CHECKS
"""
if field == ".":
return []
else:
return [k.replace("\a", ".") for k in field.replace("\\.", "\a").split(".")]
def _setdefault(obj, key, value):
"""
DO NOT USE __dict__.setdefault(obj, key, value), IT DOES NOT CHECK FOR obj[key] == None
"""
v = obj.get(key)
if v == None:
obj[key] = value
return value
return v
export("mo_dots.lists", Null)
obj[path0] = old_value = {}
_assign_to_null(old_value, path[1:], value)
except Exception as e:
raise e
def _split_field(field):
"""
SIMPLE SPLIT, NO CHECKS
"""
if field == ".":
return []
else:
return [k.replace("\a", ".") for k in field.replace("\\.", "\a".replace("\b", "\\.")).split(".")]
def _setdefault(obj, key, value):
"""
DO NOT USE __dict__.setdefault(obj, key, value), IT DOES NOT CHECK FOR obj[key] == None
"""
v = obj.get(key)
if v == None:
obj[key] = value
return value
return v
export("mo_dots.lists", Null)
export("mo_dots.lists", "EMPTY", Null)
def invert(self, lang):
return FALSE
def __call__(self, row=None, rownum=None, rows=None):
return True
def __unicode__(self):
return "true"
def __str__(self):
return b"true"
def __bool__(self):
return True
def __rcontains__(self, superset):
return self is superset
def __nonzero__(self):
return True
TRUE = TrueOp()
export("jx_base.expressions.literal", TRUE)
export("jx_base.expressions.false_op", TRUE)
export("jx_base.expressions._utils", TRUE)
export("jx_base.expressions.expression", TRUE)
if simple.type != BOOLEAN:
simple = simple.exists()
if simple is TRUE:
return TRUE
elif simple is FALSE:
continue
elif is_op(simple, OrOp):
terms.extend([tt for tt in simple.terms if tt not in terms])
elif is_op(simple, AndOp):
ands.append(simple)
elif simple not in terms:
terms.append(simple)
if ands: # REMOVE TERMS THAT ARE MORE RESTRICTIVE THAN OTHERS
for a in ands:
for tt in a.terms:
if tt in terms:
break
else:
terms.append(a)
if len(terms) == 0:
return FALSE
if len(terms) == 1:
return terms[0]
return self.lang[OrOp(terms)]
export("jx_base.expressions.and_op", OrOp)
from mo_imports import export
from tests.module_value2.bars import bar
from tests.module_value2 import bars
def foo():
bar()
export(bars, foo)
value.path, None,
AndOp([InOp([value.select, superset]),
value.where])).exists().partial_eval(lang)
else:
return (InOp([value, superset]))
def __call__(self, row):
return self.value(row) in self.superset(row)
def missing(self, lang):
return FALSE
def invert(self, lang):
this = self.partial_eval(lang)
if is_op(this, InOp):
inv = NotOp(BasicInOp([this.value, this.superset]))
inv.simplified = True
return OrOp([MissingOp(this.value), inv])
else:
return this.invert(lang)
def __rcontains__(self, superset):
if (is_op(self.value, Variable) and is_op(superset, MissingOp)
and is_op(superset.value, Variable)
and superset.value.var == self.value.var):
return True
return False
export("jx_base.expressions.eq_op", InOp)
return typer(self._value)
def partial_eval(self, lang):
return self
def str(self):
return str(self.value)
ZERO = Literal(0)
ONE = Literal(1)
literal_op_ids = tuple()
def register_literal(op):
global literal_op_ids
literal_op_ids = literal_op_ids + (op.get_id(), )
def is_literal(l):
try:
return l.get_id() in literal_op_ids
except Exception:
return False
export("jx_base.expressions._utils", Literal)
export("jx_base.expressions.expression", Literal)
export("jx_base.expressions.expression", is_literal)
sum = 0
for v in vals:
yield sum, v
sum += v
def reverse(vals):
# TODO: Test how to do this fastest
if not hasattr(vals, "len"):
vals = list(vals)
l = len(vals)
output = [None] * l
for v in unwrap(vals):
l -= 1
output[l] = v
return to_data(output)
def countdown(vals):
remaining = len(vals) - 1
return [(remaining - i, v) for i, v in enumerate(vals)]
from jx_python.lists.aggs import is_aggs, list_aggs
export("jx_base.container", run)
export("jx_python.containers.list", "jx")
def min_length(self):
return 0
def parseImpl(self, string, start, doActions=True):
word_chars = self.parser_config.word_chars
instrlen = len(string)
if instrlen > 0 and start < instrlen:
if string[start] in word_chars or string[start - 1] not in word_chars:
raise ParseException(self, start, string)
return ParseResults(self, start, start, [])
def __regex__(self):
return "+", self.parser_config.regex.pattern
export("mo_parsing.results", Token)
export("mo_parsing.results", Empty)
export("mo_parsing.core", Empty)
export("mo_parsing.core", StringEnd)
export("mo_parsing.core", Literal)
export("mo_parsing.core", Token)
export("mo_parsing.engine", Literal)
export("mo_parsing.engine", Token)
export("mo_parsing.enhancement", Token)
export("mo_parsing.enhancement", NoMatch)
export("mo_parsing.enhancement", Literal)
export("mo_parsing.enhancement", Keyword)
export("mo_parsing.enhancement", Word)
class DateOp(Literal):
date_type = NUMBER
def __init__(self, term):
if hasattr(self, "date"):
return
if is_text(term):
self.date = term
else:
self.date = coalesce(term.get("literal"), term)
v = unicode2Date(self.date)
if isinstance(v, Date):
Literal.__init__(self, v.unix)
else:
Literal.__init__(self, v.seconds)
@classmethod
def define(cls, expr):
return cls.lang[DateOp(expr.get("date"))]
def __data__(self):
return {"date": self.date}
def __call__(self, row=None, rownum=None, rows=None):
return Date(self.date)
export("jx_base.expressions.literal", DateOp)
elif not s.sort and not s.value and all(d in sort_direction
for d in s.values()):
for v, d in s.items():
output.append({
"value": jx_expression(v),
"sort": sort_direction[d]
})
elif not s.sort and not s.value:
Log.error("`sort` clause must have a `value` property")
else:
output.append({
"value": jx_expression(coalesce(s.value, s.field)),
"sort": sort_direction[s.sort]
})
return output
sort_direction = {
"asc": 1,
"ascending": 1,
"desc": -1,
"descending": -1,
"none": 0,
1: 1,
0: 0,
-1: -1,
None: 1
}
export("jx_base.container", QueryOp)
def is_missing(t):
# RETURN True IF EFFECTIVELY NOTHING
class_ = t.__class__
if class_ in (none_type, NullType):
return True
elif class_ in data_types:
return False
elif class_ in finite_types and not t:
return True
else:
return t == None
# EXPORT
export("mo_dots.nones", to_data)
export("mo_dots.datas", list_to_data)
export("mo_dots.datas", dict_to_data)
export("mo_dots.datas", to_data)
export("mo_dots.datas", from_data)
export("mo_dots.datas", coalesce)
export("mo_dots.datas", _getdefault)
export("mo_dots.datas", hash_value)
export("mo_dots.datas", listwrap)
export("mo_dots.datas", literal_field)
export("mo_dots.lists", list_to_data)
export("mo_dots.lists", to_data)
export("mo_dots.lists", coalesce)
export("mo_dots.lists", from_data)
@simplified
def partial_eval(self):
when = self.lang[BooleanOp(self.when)].partial_eval()
if when is TRUE:
return self.lang[self.then].partial_eval()
elif when in [FALSE, NULL]:
return self.lang[self.els_].partial_eval()
elif is_op(when, Literal):
Log.error("Expecting `when` clause to return a Boolean, or `null`")
then = self.lang[self.then].partial_eval()
els_ = self.lang[self.els_].partial_eval()
if then is TRUE:
if els_ is FALSE:
return when
elif els_ is TRUE:
return TRUE
elif then is FALSE:
if els_ is FALSE:
return FALSE
elif els_ is TRUE:
return self.lang[NotOp(when)].partial_eval()
return self.lang[WhenOp(when, **{"then": then, "else": els_})]
export("jx_base.expressions.first_op", WhenOp)
export("jx_base.expressions.eq_op", WhenOp)
def __iter__(self):
return self.terms.__iter__()
def __data__(self):
return {"tuple": [t.__data__() for t in self.terms]}
def vars(self):
output = set()
for t in self.terms:
output |= t.vars()
return output
def map(self, map_):
return (TupleOp([t.map(map_) for t in self.terms]))
def missing(self, lang):
return FALSE
def __call__(self):
return tuple(t() for t in self.terms)
def partial_eval(self, lang):
if all(is_literal(t) for t in self.terms):
return (Literal([t.value for t in self.terms]))
return self
export("jx_base.expressions._utils", TupleOp)
from mo_imports import export
from tests.module_name_value1.bars import bar
from tests.module_name_value1 import bars
def foo():
bar()
export(bars, "alt", foo)
def vars(self):
return self.expr.vars()
def map(self, map_):
return (MissingOp(self.expr.map(map_)))
def missing(self, lang):
return FALSE
def invert(self, lang):
output = self.expr.missing(lang)
if is_op(output, MissingOp):
# break call cycle
return (NotOp(output))
else:
return (output.invert(lang))
def exists(self):
return TRUE
def partial_eval(self, lang):
output = self.expr.partial_eval(lang).missing(lang)
if is_op(output, MissingOp):
return output
else:
return output.partial_eval(lang)
export("jx_base.expressions.expression", MissingOp)
def __add__(self, other):
return text(self) + text(other)
def __radd__(self, other):
return text(other) + text(self)
if PY2:
__unicode__ = __str__
@check
def to_bq(self, schema, not_null=False, boolean=False, many=True):
return self
def missing(self):
return self.miss
def __data__(self):
return {"script": text(self.sql)}
def __eq__(self, other):
if not isinstance(other, SQLScript_):
return False
elif self.expr == other.expr:
return True
else:
return False
export("jx_bigquery.expressions._utils", BQLScript)
