# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from mo_dots.lists import is_sequence from mo_dots.utils import CLASS, OBJ from mo_future import is_binary, text, none_type from mo_imports import expect, export to_data, = expect("to_data") _get = object.__getattribute__ _set = object.__setattr__ _zero_list = [] _null_hash = hash(None) class NullType(object): """ Structural Null provides closure under the dot (.) operator Null[x] == Null Null.x == Null Null INSTANCES WILL TRACK THEIR OWN DEREFERENCE PATH SO ASSIGNMENT CAN BE DONE
# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from mo_dots.lists import is_sequence from mo_dots.utils import CLASS, OBJ, KEY from mo_future import is_binary, text, none_type from mo_imports import expect, export to_data = expect("to_data") _get = object.__getattribute__ _set = object.__setattr__ _zero_list = [] _null_hash = hash(None) class NullType(object): """ Structural Null provides closure under the dot (.) operator Null[x] == Null Null.x == Null Null INSTANCES WILL TRACK THEIR OWN DEREFERENCE PATH SO ASSIGNMENT CAN BE DONE
import inspect
import json
import re
import string
import sys
import warnings
from collections import namedtuple
from math import isnan
from types import FunctionType
from mo_dots import is_null, Null
from mo_future import unichr, text, generator_types, get_function_name
from mo_imports import expect
ParseException = expect("ParseException")
Many = expect("Many")
def append_config(base, *slots):
dups = set(slots) & set(base.Config._fields)
if dups:
Log.error("Duplicate config fields: {{dups}}", dups=dups)
fields = base.Config._fields + slots
return namedtuple("Config", fields)
try:
from mo_logs import Log, Except
except Exception:
from jx_base.dimensions import Dimension
from jx_base.domains import DefaultDomain, Domain, SetDomain
from jx_base.expressions import Expression, FALSE, LeavesOp, QueryOp as QueryOp_, ScriptOp, Variable, jx_expression
from jx_base.language import is_expression, is_op
from jx_base.table import Table
from jx_base.utils import is_variable_name
from mo_dots import Data, FlatList, Null, coalesce, concat_field, is_container, is_data, is_list, listwrap, \
literal_field, relative_field, set_default, unwrap, unwraplist, is_many, dict_to_data, to_data, list_to_data
from mo_future import is_text, text
from mo_imports import expect, export
from mo_json import INTERNAL
from mo_json.typed_encoder import untype_path
from mo_logs import Log
from mo_math import AND, UNION, is_number
Column, = expect("Column")
BAD_SELECT = "Expecting `value` or `aggregate` in select clause not {{select}}"
DEFAULT_LIMIT = 10
MAX_LIMIT = 10000
DEFAULT_SELECT = Data(name="count",
value=jx_expression("."),
aggregate="count",
default=0)
class QueryOp(QueryOp_):
__slots__ = [
"frum", "select", "edges", "groupby", "where", "window", "sort",
"limit", "format", "chunk_size", "destination"
]
)
from mo_imports import expect
(
_getdefault,
coalesce,
hash_value,
listwrap,
literal_field,
from_data,
to_data,
) = expect(
"_getdefault",
"coalesce",
"hash_value",
"listwrap",
"literal_field",
"from_data",
"to_data",
)
_get = object.__getattribute__
_set = object.__setattr__
_new = object.__new__
SLOT = str("_internal_dict")
DEBUG = False
class Data(object):
"""
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions._utils import simplified, TRUE from jx_base.expressions.expression import Expression from jx_base.expressions.false_op import FALSE from mo_imports import expect from mo_json import BOOLEAN NotOp, = expect("NotOp") class ExistsOp(Expression): data_type = BOOLEAN def __init__(self, term): Expression.__init__(self, [term]) self.expr = term def __data__(self): return {"exists": self.expr.__data__()} def vars(self): return self.expr.vars()
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions._utils import value2json from jx_base.expressions.expression import Expression from mo_dots import Null, is_data, is_many from mo_imports import expect, export from mo_json import python_type_to_json_type, merge_json_type DateOp, FALSE, TRUE, NULL = expect("DateOp", "FALSE", "TRUE", "NULL") class Literal(Expression): """ A literal JSON document """ def __new__(cls, term): if term == None: return NULL if term is True: return TRUE if term is False: return FALSE if is_data(term) and term.get("date"): # SPECIAL CASE
# encoding: utf-8 # # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions import BooleanOp as BooleanOp_, Variable as Variable_ from jx_base.language import is_op from jx_elasticsearch.es52.expressions.utils import ES52 from jx_elasticsearch.es52.expressions.exists_op import es_exists from jx_elasticsearch.es52.painless import Painless from mo_imports import expect FindOp = expect("FindOp") class BooleanOp(BooleanOp_): def to_es(self, schema): if is_op(self.term, Variable_): return es_exists(self.term.var) elif is_op(self.term, FindOp): return ES52[self.term].to_es(schema) else: return Painless[self].to_es_script(schema).to_es(schema)
is_text,
items as items_,
text,
utf8_json_encoder,
)
from mo_imports import expect
from mo_json import BOOLEAN, INTEGER, IS_NULL, NUMBER, OBJECT, STRING, scrub
from mo_logs import Except, Log
from mo_math import is_number
from mo_times import Date
ALLOW_SCRIPTING = False
EMPTY_DICT = {}
Literal, TRUE, FALSE, NULL, TupleOp, Variable = expect("Literal", "TRUE",
"FALSE", "NULL",
"TupleOp", "Variable")
def extend(cls):
"""
DECORATOR TO ADD METHODS TO CLASSES
:param cls: THE CLASS TO ADD THE METHOD TO
:return:
"""
def extender(func):
setattr(cls, get_function_name(func), func)
return func
return extender
SQL_NULL,
SQL_TRUE,
sql_iso,
SQL_ZERO,
SQL_ONE,
SQL_PLUS,
SQL_STAR,
SQL_LT,
)
from mo_dots import wrap
from mo_future import decorate
from mo_imports import expect
from mo_json import BOOLEAN, NESTED, OBJECT, STRING, NUMBER, IS_NULL
from mo_logs import Log
NumberOp, OrOp, BQLScript = expect("NumberOp", "OrOp", "BQLScript")
def check(func):
"""
TEMPORARY TYPE CHECKING TO ENSURE to_bq() IS OUTPUTTING THE CORRECT FORMAT
"""
@decorate(func)
def to_bq(self, schema, not_null=False, boolean=False, **kwargs):
if kwargs.get("many") != None:
Log.error("not expecting many")
output = func(self, schema, not_null, boolean)
if not isinstance(output, BQLScript):
Log.error("Expecting BQLScript")
return output
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from copy import copy from mo_dots import Data, is_data, is_many, join_field, set_default, split_field, to_data from mo_future import is_text from mo_imports import expect from mo_logs import Log ListContainer, Cube, run, QueryOp = expect("ListContainer", "Cube", "run", "QueryOp") type2container = Data() config = Data() # config.default IS EXPECTED TO BE SET BEFORE CALLS ARE MADE class Container(object): """ CONTAINERS HOLD MULTIPLE INDICES AND CAN HANDLE GENERAL JSON QUERY EXPRESSIONS ON ITS CONTENTS METADATA FOR A Container IS CALLED A Namespace """ @classmethod def new_instance(type, frum, schema=None): """ Factory!
# Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals import types from copy import deepcopy from mo_dots.utils import CLASS from mo_future import generator_types, first from mo_imports import expect Log = None datawrap, coalesce, to_data, from_data, Null = expect("datawrap", "coalesce", "to_data", "from_data", "Null") _list = str("list") _get = object.__getattribute__ _set = object.__setattr__ _emit_slice_warning = True def _get_list(self): return _get(self, _list) def _late_import(): global Log
from mo_imports import expect
alt = expect("alt")
def bar():
alt()
from mo_imports import expect
foos = expect("foos")
def bar():
foos.foo()
from __future__ import absolute_import, division, unicode_literals
from jx_base.expressions import (
MissingOp as MissingOp_,
NotOp as NotOp_,
Variable as Variable_,
)
from jx_base.language import is_op
from jx_elasticsearch.es52.expressions.false_op import MATCH_NONE
from jx_elasticsearch.es52.expressions.utils import ES52
from mo_dots import dict_to_data
from mo_future import first
from mo_imports import expect
from mo_json import STRUCT
es_or = expect("es_or")
class NotOp(NotOp_):
def to_es(self, schema):
if is_op(self.term, MissingOp_) and is_op(self.term.expr, Variable_):
# PREVENT RECURSIVE LOOP
v = self.term.expr.var
cols = schema.values(v, STRUCT)
if len(cols) == 0:
return MATCH_NONE
elif len(cols) == 1:
return {"exists": {"field": first(cols).es_column}}
else:
return es_or([{
"exists": {
from jx_base.expressions import TupleOp
from jx_base.language import is_op
from jx_base.expressions.query_op import canonical_aggregates
from jx_python.containers.cube import Cube
from mo_collections.matrix import Matrix
from mo_dots import Data, coalesce, is_list, split_field, to_data
from mo_files import mimetype
from mo_future import sort_using_key, next
from mo_imports import expect
from mo_json import value2json
from mo_logs import Log
from mo_logs.strings import quote
aggs_iterator, count_dim = expect("aggs_iterator", "count_dim")
def format_cube(aggs, es_query, query, decoders, all_selects):
new_edges = count_dim(aggs, es_query, decoders)
dims = []
for e in new_edges:
if is_op(e.value, TupleOp):
e.allowNulls = False
extra = 0 if e.allowNulls is False else 1
dims.append(len(e.domain.partitions) + extra)
dims = tuple(dims)
if any(s.default != canonical_aggregates[s.aggregate].default for s in all_selects):
# # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions.literal import Literal from mo_imports import export, expect from mo_json import BOOLEAN TRUE = expect("TRUE") class FalseOp(Literal): data_type = BOOLEAN def __new__(cls, *args, **kwargs): return object.__new__(cls, *args, **kwargs) def __init__(self, op=None, term=None): Literal.__init__(self, False) @classmethod def define(cls, expr): return FALSE
# You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions.expression import Expression from jx_base.expressions.last_op import LastOp from jx_base.expressions.literal import is_literal from jx_base.language import is_op from mo_imports import expect from mo_json import OBJECT from mo_logs import Log CaseOp, WhenOp = expect("CaseOp", "WhenOp") class FirstOp(Expression): def __init__(self, term): Expression.__init__(self, [term]) self.term = term self.data_type = self.term.type def __data__(self): return {"first": self.term.__data__()} def vars(self): return self.term.vars() def map(self, map_):
# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions import OrOp as OrOp_ from jx_elasticsearch.es52.expressions.not_op import es_not, NotOp from jx_elasticsearch.es52.expressions.utils import ES52 from mo_dots import dict_to_data from mo_imports import expect, export es_and = expect("es_and") class OrOp(OrOp_): def to_es(self, schema): if schema.snowflake.namespace.es_cluster.version.startswith("5."): # VERSION 5.2.x # WE REQUIRE EXIT-EARLY SEMANTICS, OTHERWISE EVERY EXPRESSION IS A SCRIPT EXPRESSION # {"bool":{"should" :[a, b, c]}} RUNS IN PARALLEL # {"bool":{"must_not":[a, b, c]}} ALSO RUNS IN PARALLEL # OR(x) == NOT(AND(NOT(xi) for xi in x)) output = es_not( es_and([ NotOp(t).partial_eval().to_es(schema) for t in self.terms
from jx_base.language import is_expression, is_op
from jx_base.meta_columns import get_schema_from_list
from jx_base.namespace import Namespace
from jx_base.schema import Schema
from jx_base.table import Table
from jx_python.convert import list2cube, list2table
from jx_python.expressions import jx_expression_to_function
from jx_python.lists.aggs import is_aggs, list_aggs
from mo_collections import UniqueIndex
from mo_dots import Data, Null, is_data, is_list, listwrap, unwrap, unwraplist, to_data, coalesce, dict_to_data
from mo_future import first, sort_using_key
from mo_imports import export, expect
from mo_logs import Log
from mo_threads import Lock
jx = expect("jx")
class ListContainer(Container, Namespace, Table):
"""
A CONTAINER WITH ONLY ONE TABLE
"""
def __init__(self, name, data, schema=None):
# TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
data = list(unwrap(data))
Container.__init__(self)
if schema == None:
self._schema = get_schema_from_list(name, data)
else:
self._schema = schema
self.name = coalesce(name, ".")
from jx_base.expressions._utils import (
operators,
jx_expression,
_jx_expression,
)
from jx_base.language import BaseExpression, ID, is_expression, is_op
from mo_dots import is_data, is_sequence, is_container
from mo_future import items as items_, text
from mo_imports import expect
from mo_json import BOOLEAN, OBJECT, value2json
from mo_logs import Log
from mo_threads import register_thread
TRUE, FALSE, Literal, is_literal, MissingOp, NotOp, NULL, Variable, AndOp = expect(
"TRUE", "FALSE", "Literal", "is_literal", "MissingOp", "NotOp", "NULL",
"Variable", "AndOp")
class Expression(BaseExpression):
data_type = OBJECT
has_simple_form = False
def __init__(self, args):
self.simplified = False
# SOME BASIC VERIFICATION THAT THESE ARE REASONABLE PARAMETERS
if is_sequence(args):
bad = [t for t in args if t != None and not is_expression(t)]
if bad:
Log.error("Expecting an expression, not {{bad}}", bad=bad)
elif is_data(args):
from jx_base.expressions._utils import (
operators,
jx_expression,
_jx_expression,
simplified,
)
from jx_base.language import BaseExpression, ID, is_expression, is_op
from mo_dots import is_data, is_sequence, is_container
from mo_future import items as items_, text
from mo_imports import expect
from mo_json import BOOLEAN, OBJECT, value2json
from mo_logs import Log
FALSE, Literal, is_literal, MissingOp, NotOp, NULL, Variable = expect(
"FALSE", "Literal", "is_literal", "MissingOp", "NotOp", "NULL", "Variable"
)
class Expression(BaseExpression):
data_type = OBJECT
has_simple_form = False
def __init__(self, args):
self.simplified = False
# SOME BASIC VERIFICATION THAT THESE ARE REASONABLE PARAMETERS
if is_sequence(args):
bad = [t for t in args if t != None and not is_expression(t)]
if bad:
Log.error("Expecting an expression, not {{bad}}", bad=bad)
elif is_data(args):
)
from jx_base.language import Language, is_op
from jx_elasticsearch.es52.painless import Painless
from jx_elasticsearch.es52.painless.es_script import es_script
from mo_dots import Null, to_data, join_field, split_field, coalesce, startswith_field
from mo_future import first
from mo_imports import expect, delay_import
from mo_json import EXISTS
from mo_json.typed_encoder import EXISTS_TYPE, NESTED_TYPE
from mo_logs import Log
from mo_math import MAX
MATCH_NONE, MATCH_ALL, AndOp, OrOp, NestedOp = expect(
"MATCH_NONE",
"MATCH_ALL",
"AndOp",
"OrOp",
"NestedOp",
)
get_decoders_by_path = delay_import(
"jx_elasticsearch.es52.agg_op.get_decoders_by_path")
def _inequality_to_esfilter(self, schema):
if is_op(self.lhs, Variable_) and is_literal(self.rhs):
cols = schema.leaves(self.lhs.var)
if not cols:
lhs = self.lhs.var # HAPPENS DURING DEBUGGING, AND MAYBE IN REAL LIFE TOO
elif len(cols) == 1:
lhs = first(cols).es_column
else:
# # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http:# mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals from jx_base.expressions.literal import Literal from mo_imports import export, expect from mo_json import BOOLEAN TRUE, = expect("TRUE") class FalseOp(Literal): data_type = BOOLEAN def __new__(cls, *args, **kwargs): return object.__new__(cls, *args, **kwargs) def __init__(self, op=None, term=None): Literal.__init__(self, False) @classmethod def define(cls, expr): return FALSE
# Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals import types from copy import deepcopy from mo_future import generator_types, first, is_text from mo_imports import expect, delay_import from mo_dots.utils import CLASS Log = delay_import("mo_logs.Log") datawrap, coalesce, list_to_data, to_data, from_data, Null, EMPTY = expect( "datawrap", "coalesce", "list_to_data", "to_data", "from_data", "Null", "EMPTY" ) LIST = str("list") _get = object.__getattribute__ _set = object.__setattr__ _emit_slice_warning = True class FlatList(object): """ ENCAPSULATES HANDING OF Nulls BY wrapING ALL MEMBERS AS NEEDED ENCAPSULATES FLAT SLICES ([::]) FOR USE IN WINDOW FUNCTIONS https://github.com/klahnakoski/mo-dots/tree/dev/docs#flatlist-is-flat """
# License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals import time from mo_future import allocate_lock from mo_imports import expect from mo_logs.log_usingNothing import StructuredLogger from mo_logs.strings import expand_template Log = expect("Log") class StructuredLogger_usingFile(StructuredLogger): def __init__(self, file): assert file from mo_files import File self.file = File(file) if self.file.exists: self.file.backup() self.file.delete() self.file_lock = allocate_lock() def write(self, template, params):
# You can obtain one at http://mozilla.org/MPL/2.0/. # # Contact: Kyle Lahnakoski ([email protected]) # from __future__ import absolute_import, division, unicode_literals import argparse as _argparse import os import sys import tempfile from mo_dots import coalesce, listwrap, unwrap, to_data from mo_imports import expect Log, = expect("Log") # PARAMETERS MATCH argparse.ArgumentParser.add_argument() # https://docs.python.org/dev/library/argparse.html#the-add-argument-method # # name or flags - Either a name or a list of option strings, e.g. foo or -f, --foo. # action - The basic type of action to be taken when this argument is encountered at the command line. # nargs - The number of command-line arguments that should be consumed. # const - A constant value required by some action and nargs selections. # default - The value produced if the argument is absent from the command line. # type - The type to which the command-line argument should be converted. # choices - A container of the allowable values for the argument. # required - Whether or not the command-line option may be omitted (optionals only). # help - A brief description of what the argument does. # metavar - A name for the argument in usage messages. # dest - The name of the attribute to be added to the object returned by parse_args().
# encoding: utf-8
import re
from collections import namedtuple
from mo_future import is_text
from mo_imports import expect, Expecting
from mo_parsing.core import ParserElement
from mo_parsing.results import ParseResults
from mo_parsing.utils import Log, indent, quote, regex_range, alphanums, regex_iso
Literal, Token, Empty = expect("Literal", "Token", "Empty")
CURRENT = None
PLAIN_ENGINE = None
STANDARD_ENGINE = None
class Engine(ParserElement):
def __init__(self, white=" \n\r\t"):
self.literal = Literal
self.keyword_chars = alphanums + "_$"
self.ignore_list = []
self.debugActions = DebugActions(noop, noop, noop)
self.all_exceptions = {}
self.content = None
self.skips = {}
self.regex = None
self.expr = None
self.set_whitespace(white)
self.previous = None # WE MAINTAIN A STACK OF ENGINES
#
from __future__ import absolute_import, division, unicode_literals
from jx_base.expressions._utils import simplified
from jx_base.expressions.boolean_op import BooleanOp
from jx_base.expressions.expression import Expression, NULL
from jx_base.expressions.false_op import FALSE
from jx_base.expressions.true_op import TRUE
from jx_base.language import is_op
from mo_dots import is_many
from mo_future import zip_longest
from mo_imports import expect
from mo_json import BOOLEAN
NotOp, OrOp = expect("NotOp", "OrOp")
class AndOp(Expression):
data_type = BOOLEAN
zero = TRUE # ADD THIS TO terms FOR NO EEFECT
def __init__(self, terms):
Expression.__init__(self, terms)
if terms == None:
self.terms = []
elif is_many(terms):
self.terms = terms
else:
self.terms = [terms]
from mo_imports import expect
foo = expect("foo")
def bar():
foo()