def __init__(self, name, params, cwd=None, env=None, debug=False):
self.name = name
self.service_stopped = Signal("stopped signal for " + convert.string2quote(name))
self.stdin = Queue("stdin for process " + convert.string2quote(name), silent=True)
self.stdout = Queue("stdout for process " + convert.string2quote(name), silent=True)
self.stderr = Queue("stderr for process " + convert.string2quote(name), silent=True)
try:
self.debug=debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1,
cwd=cwd,
env=env
)
self.stopper = Signal()
self.stopper.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.stopper, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, service.stdout, self.stdout, please_stop=self.stopper, parent_thread=self),
# Thread.run(self.name + " stderr", self._reader, service.stderr, self.stderr, please_stop=self.stopper, parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)
def __init__(self, name, params, cwd=None, env=None, debug=False):
self.name = name
self.service_stopped = Signal("stopped signal for " +
convert.string2quote(name))
self.stdin = Queue("stdin for process " + convert.string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + convert.string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + convert.string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1,
cwd=cwd,
env=env)
self.stopper = Signal()
self.stopper.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.stopper,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
service.stdout,
self.stdout,
please_stop=self.stopper,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
service.stderr,
self.stderr,
please_stop=self.stopper,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)
def _convert(v):
if v is None:
return "null"
if v is True:
return "true"
if v is False:
return "false"
if isinstance(v, basestring):
return convert.string2quote(v)
if isinstance(v, (int, long, float)):
return unicode(v)
if isinstance(v, dict):
return "[" + ", ".join(convert.string2quote(k) + ": " + _convert(vv) for k, vv in v.items()) + "]"
if isinstance(v, list):
return "[" + ", ".join(_convert(vv) for vv in v) + "]"
def to_ruby(self):
if isinstance(self.field, Variable):
return "!doc["+convert.string2quote(self.field.var)+"].isEmpty()"
elif isinstance(self.field, Literal):
return self.field.exists().to_ruby()
else:
return self.field.to_ruby() + " != null"
def value2query(value):
if isinstance(value, datetime):
return convert.datetime2milli(value)
if isinstance(value, Duration):
return value.milli
if Math.is_number(value):
return value
return convert.string2quote(value)
def value2query(value):
if isinstance(value, datetime):
return convert.datetime2milli(value)
if isinstance(value, Duration):
return value.milli
if Math.is_number(value):
return value
return convert.string2quote(value)
class Json2Redshift(object):
@use_settings
def __init__(
self,
host,
user,
password,
table,
meta, # REDSHIFT COPY COMMAND REQUIRES A BUCKET TO HOLD PARAMETERS
database=None,
port=5439,
settings=None):
self.settings = settings
self.db = Redshift(settings)
INDEX_CACHE[settings.table] = wrap(
{"name": settings.table}) # HACK TO GET parse_columns TO WORK
columns = parse_columns(settings.table,
settings.mapping.test_result.properties)
nested = [c.name for c in columns if c.type == "nested"]
self.columns = wrap([
c for c in columns if c.type not in ["object"] and not any(
c.name.startswith(n + ".") for n in nested)
])
try:
self.db.execute("""
CREATE TABLE {{table_name}} (
"_id" character varying UNIQUE,
{{columns}}
)""", {
"table_name":
self.db.quote_column(settings.table),
"columns":
SQL(",\n".join(
self.db.quote_column(c.name) + " " +
self.db.es_type2pg_type(c.type) for c in self.columns))
},
retry=False)
except Exception, e:
if "already exists" in e:
Log.alert("Table {{table}} exists in Redshift",
table=settings.table)
else:
Log.error("Could not make table", e)
# MAKE jsonpaths FOR COPY COMMAND
jsonpaths = {
"jsonpaths": [
"$" + "".join("[" + convert.string2quote(p) + "]"
for p in split_field(c.name))
for c in self.columns
]
}
content = convert.value2json(jsonpaths)
content = content.replace("\\\"", "'")
# PUSH TO S3
s3.Bucket(meta).write(meta.jsonspath, content)
def value2MVEL(value):
"""
FROM PYTHON VALUE TO MVEL EQUIVALENT
"""
if isinstance(value, datetime):
return str(convert.datetime2milli(value)) + " /*" + value.format("yyNNNdd HHmmss") + "*/" # TIME
if isinstance(value, Duration):
return str(convert.timedelta2milli(value)) + " /*" + str(value) + "*/" # DURATION
if Math.is_number(value):
return str(value)
return convert.string2quote(value)
def value2MVEL(value):
"""
FROM PYTHON VALUE TO MVEL EQUIVALENT
"""
if isinstance(value, datetime):
return str(convert.datetime2milli(value)) + " /*" + value.format("yyNNNdd HHmmss") + "*/" # TIME
if isinstance(value, Duration):
return str(convert.timedelta2milli(value)) + " /*" + str(value) + "*/" # DURATION
if Math.is_number(value):
return str(value)
return convert.string2quote(value)
def compileString2Term(edge):
if edge.esscript:
Log.error("edge script not supported yet")
value = edge.value
if isKeyword(value):
value = strings.expand_template("getDocValue({{path}})", {"path": convert.string2quote(value)})
else:
Log.error("not handled")
def fromTerm(value):
return edge.domain.getPartByKey(value)
return Dict(toTerm={"head": "", "body": value}, fromTerm=fromTerm)
def compileString2Term(edge):
if edge.esscript:
Log.error("edge script not supported yet")
value = edge.value
if isKeyword(value):
value = strings.expand_template("getDocValue({{path}})",
{"path": convert.string2quote(value)})
else:
Log.error("not handled")
def fromTerm(value):
return edge.domain.getPartByKey(value)
return Data(toTerm={"head": "", "body": value}, fromTerm=fromTerm)
def Parts2Term(self, domain):
"""
TERMS ARE ALWAYS ESCAPED SO THEY CAN BE COMPOUNDED WITH PIPE (|)
CONVERT AN ARRAY OF PARTS{name, esfilter} TO AN MVEL EXPRESSION
RETURN expression, function PAIR, WHERE
expression - MVEL EXPRESSION
function - TAKES RESULT OF expression AND RETURNS PART
"""
fields = domain.dimension.fields
term = []
if len(split_field(self.fromData.name)) == 1 and fields:
if isinstance(fields, Mapping):
# CONVERT UNORDERED FIELD DEFS
jx_fields, es_fields = zip(*[(k, fields[k]) for k in sorted(fields.keys())])
else:
jx_fields, es_fields = zip(*[(i, e) for i, e in enumerate(fields)])
# NO LOOPS BECAUSE QUERY IS SHALLOW
# DOMAIN IS FROM A DIMENSION, USE IT'S FIELD DEFS TO PULL
if len(es_fields) == 1:
def fromTerm(term):
return domain.getPartByKey(term)
return Data(
head="",
body='getDocValue('+convert.string2quote(domain.dimension.fields[0])+')'
), fromTerm
else:
def fromTerm(term):
terms = [convert.pipe2value(t) for t in convert.pipe2value(term).split("|")]
candidate = dict(zip(jx_fields, terms))
for p in domain.partitions:
for k, t in candidate.items():
if p.value[k] != t:
break
else:
return p
if domain.type in ["uid", "default"]:
part = {"value": candidate}
domain.partitions.append(part)
return part
else:
return Null
for f in es_fields:
term.append('Value2Pipe(getDocValue('+convert.string2quote(f)+'))')
return Data(
head="",
body='Value2Pipe('+('+"|"+'.join(term))+')'
), fromTerm
else:
for v in domain.partitions:
term.append("if (" + _where(v.esfilter, lambda x: self._translate(x)) + ") " + value2MVEL(domain.getKey(v)) + "; else ")
term.append(value2MVEL(domain.getKey(domain.NULL)))
func_name = "_temp"+UID()
return self.register_function("+\"|\"+".join(term))
def to_python(self):
return "((" + convert.string2quote(self.substring) + " in " + self.var.to_python() + ") if " + self.var.to_python() + "!=None else False)"
def quote_table(column):
if _do_not_quote.match(column):
return column
return convert.string2quote(column)
def _quote_column(column):
return convert.string2quote(column.es_column)
def __json__(self):
return convert.string2quote(self.code)
def to_python(self):
return qb_expression_to_python(
self.field) + ".startswith(" + convert.string2quote(
self.prefix) + ")"
def _quote_column(column):
return convert.string2quote(column.es_column)
def DataClass(name, columns):
"""
Each column has {"name", "required", "nulls", "default", "type"} properties
"""
columns = wrap([{
"name": c,
"required": True,
"nulls": False,
"type": object
} if isinstance(c, basestring) else c for c in columns])
slots = columns.name
required = wrap(
filter(lambda c: c.required and not c.nulls and not c.default,
columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template(
"""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
class {{name}}(Mapping):
__slots__ = {{slots}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
setattr(self, s, kwargs.get(s, kwargs.get('default', Null)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
#if not isinstance(value, types_[item]):
# Log.error("{"+"{item|quote}} not of type "+"{"+"{type}}", item=item, type=types_[item])
object.__setattr__(self, item, value)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
temp = {{name}}
""", {
"name":
name,
"slots":
"(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required":
"{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls":
"{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"len_slots":
len(slots),
"dict":
"{" +
(", ".join(convert.value2quote(s) + ": self." + s
for s in slots)) + "}",
"assign":
"; ".join(
"_set(output, " + convert.value2quote(s) + ", self." + s + ")"
for s in slots),
"types":
"{" + (",".join(
convert.string2quote(k) + ": " + v.__name__
for k, v in types.items())) + "}"
})
return _exec(code, name)
def DataClass(name, columns, constraint=True):
"""
Use the DataClass to define a class, but with some extra features:
1. restrict the datatype of property
2. restrict if `required`, or if `nulls` are allowed
3. generic constraints on object properties
It is expected that this class become a real class (or be removed) in the
long term because it is expensive to use and should only be good for
verifying program correctness, not user input.
:param name: Name of the class we are creating
:param columns: Each columns[i] has properties {
"name", - (required) name of the property
"required", - False if it must be defined (even if None)
"nulls", - True if property can be None, or missing
"default", - A default value, if none is provided
"type" - a Python datatype
}
:param constraint: a JSON query Expression for extra constraints
:return: The class that has been created
"""
columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, basestring) else c for c in columns])
slots = columns.name
required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
defaults = {c.name: coalesce(c.default, None) for c in columns}
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template(
"""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
defaults_ = {{defaults}}
class {{class_name}}(Mapping):
__slots__ = {{slots}}
def _constraint(row, rownum, rows):
return {{constraint_expr}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
object.__setattr__(self, s, kwargs.get(s, {{defaults}}.get(s, None)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
object.__setattr__(self, item, value)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{class_name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{class_name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
temp = {{class_name}}
""",
{
"class_name": name,
"slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"defaults": jx_expression({"literal": defaults}).to_python(),
"len_slots": len(slots),
"dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}",
"assign": "; ".join("_set(output, "+convert.value2quote(s)+", self."+s+")" for s in slots),
"types": "{" + (",".join(convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}",
"constraint_expr": jx_expression(constraint).to_python(),
"constraint": convert.value2json(constraint)
}
)
return _exec(code, name)
def DataClass(name, columns):
"""
Each column has {"name", "required", "nulls", "default", "type"} properties
"""
columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, basestring) else c for c in columns])
slots = columns.name
required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template("""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
class {{name}}(Mapping):
__slots__ = {{slots}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
setattr(self, s, kwargs.get(s, kwargs.get('default', Null)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
#if not isinstance(value, types_[item]):
# Log.error("{"+"{item|quote}} not of type "+"{"+"{type}}", item=item, type=types_[item])
object.__setattr__(self, item, value)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
temp = {{name}}
""",
{
"name": name,
"slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"len_slots": len(slots),
"dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}",
"assign": "; ".join("_set(output, "+convert.value2quote(s)+", self."+s+")" for s in slots),
"types": "{" + (",".join(convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}"
}
)
return _exec(code, name)
def quote(value):
if not _convert:
_late_import()
return _convert.string2quote(value)
def __json__(self):
return convert.string2quote(self.code)
def to_python(self):
return qb_expression_to_python(self.field)+".startswith("+convert.string2quote(self.prefix)+")"
def _where(esFilter, _translate):
if not esFilter or esFilter is True:
return "true"
keys = esFilter.keys()
if len(keys) != 1:
Log.error("Expecting only one filter aggregate")
op = keys[0]
if op == "and":
list = esFilter[op]
if not (list):
return "true"
if len(list) == 1:
return _where(list[0], _translate)
output = "(" + " && ".join(_where(l, _translate) for l in list) + ")"
return output
elif op == "or":
list = esFilter[op]
if not list:
return "false"
if len(list) == 1:
return _where(list[0], _translate)
output = "(" + " || ".join(_where(l, _translate) for l in list) + ")"
return output
elif op == "not":
return "!(" + _where(esFilter[op, _translate]) + ")"
elif op == "term":
pair = esFilter[op]
if len(pair.keys()) == 1:
return [_translate(k) + "==" + value2MVEL(v) for k, v in pair.items()][0]
else:
return "(" + " && ".join(_translate(k) + "==" + value2MVEL(v) for k, v in pair.items()) + ")"
elif op == "terms":
output = []
for variableName, valueList in esFilter[op].items():
if not valueList:
Log.error("Expecting something in 'terms' array")
if len(valueList) == 1:
output.append(_translate(variableName) + "==" + value2MVEL(valueList[0]))
else:
output.append("(" + " || ".join(_translate(variableName) + "==" + value2MVEL(v) for v in valueList) + ")")
return " && ".join(output)
elif op == "exists":
# "exists":{"field":"myField"}
pair = esFilter[op]
variableName = pair.field
return "(" + _translate(variableName) + "!=null)"
elif op == "missing":
fieldName = _translate(esFilter[op].field)
testExistence = coalesce(esFilter[op].existence, True)
testNull = coalesce(esFilter[op].null_value, True)
output = []
if testExistence and not testNull:
output.append("(" + fieldName.replace(".?", ".") + " == empty)") # REMOVE THE .? SO WE REFER TO THE FIELD, NOT GET THE VALUE
if testNull:
output.append("(" + fieldName + "==null)")
return " || ".join(output)
elif op == "range":
pair = esFilter[op]
ranges = []
for variableName, r in pair.items():
if r.gte:
ranges.append(value2MVEL(r.gte) + "<=" + _translate(variableName))
elif r.gt:
ranges.append(value2MVEL(r.gt) + "<" + _translate(variableName))
elif r["from"]:
if r.include_lower == None or r.include_lower:
ranges.append(value2MVEL(r["from"]) + "<=" + _translate(variableName))
else:
ranges.append(value2MVEL(r["from"]) + "<" + _translate(variableName))
if r.lte:
ranges.append(value2MVEL(r.lte) + ">=" + _translate(variableName))
elif r.lt:
ranges.append(value2MVEL(r.lt) + ">" + _translate(variableName))
elif r["from"]:
if r.include_lower == None or r.include_lower:
ranges.append(value2MVEL(r["from"]) + ">=" + _translate(variableName))
else:
ranges.append(value2MVEL(r["from"]) + ">" + _translate(variableName))
return "("+" && ".join(ranges)+")"
elif op == "script":
script = esFilter[op].script
return _translate(script)
elif op == "prefix":
pair = esFilter[op]
variableName, value = pair.items()[0]
return _translate(variableName) + ".startsWith(" + convert.string2quote(value) + ")"
elif op == "match_all":
return "true"
else:
Log.error("'" + op + "' is an unknown aggregate")
return ""
def to_ruby(self):
return qb_expression_to_ruby(
self.field) + ".start_with? " + convert.string2quote(self.prefix)
def qb_expression_to_ruby(expr):
if expr == None:
return "nil"
elif Math.is_number(expr):
return unicode(expr)
elif is_keyword(expr):
return "doc[" + convert.string2quote(expr) + "].value"
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, CODE):
return expr.code
elif isinstance(expr, Date):
return unicode(expr.unix)
elif expr is True:
return "true"
elif expr is False:
return "false"
op, term = expr.items()[0]
mop = ruby_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(
["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(
a) + ")" + mop[0] + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
qb_expression_to_ruby(term)
bop = ruby_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(
["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_ruby(a) + ")" +
bop + "(" + qb_expression_to_ruby(b) +
")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(
a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = ruby_unary_operators.get(op)
if uop:
output = expand_template(uop, {"term": qb_expression_to_ruby(term)})
return output
cop = complex_operators.get(op)
if cop:
output = cop(term).to_ruby()
return output
Log.error("`{{op}}` is not a recognized operation", op=op)
def quote_table(column):
return convert.string2quote(column)
def Parts2Term(self, domain):
"""
TERMS ARE ALWAYS ESCAPED SO THEY CAN BE COMPOUNDED WITH PIPE (|)
CONVERT AN ARRAY OF PARTS{name, esfilter} TO AN MVEL EXPRESSION
RETURN expression, function PAIR, WHERE
expression - MVEL EXPRESSION
function - TAKES RESULT OF expression AND RETURNS PART
"""
fields = domain.dimension.fields
term = []
if len(split_field(self.fromData.name)) == 1 and fields:
if isinstance(fields, Mapping):
# CONVERT UNORDERED FIELD DEFS
jx_fields, es_fields = zip(*[(k, fields[k])
for k in sorted(fields.keys())])
else:
jx_fields, es_fields = zip(*[(i, e)
for i, e in enumerate(fields)])
# NO LOOPS BECAUSE QUERY IS SHALLOW
# DOMAIN IS FROM A DIMENSION, USE IT'S FIELD DEFS TO PULL
if len(es_fields) == 1:
def fromTerm(term):
return domain.getPartByKey(term)
return Dict(head="",
body='getDocValue(' +
convert.string2quote(domain.dimension.fields[0]) +
')'), fromTerm
else:
def fromTerm(term):
terms = [
convert.pipe2value(t)
for t in convert.pipe2value(term).split("|")
]
candidate = dict(zip(jx_fields, terms))
for p in domain.partitions:
for k, t in candidate.items():
if p.value[k] != t:
break
else:
return p
if domain.type in ["uid", "default"]:
part = {"value": candidate}
domain.partitions.append(part)
return part
else:
return Null
for f in es_fields:
term.append('Value2Pipe(getDocValue(' +
convert.string2quote(f) + '))')
return Dict(head="",
body='Value2Pipe(' + ('+"|"+'.join(term)) +
')'), fromTerm
else:
for v in domain.partitions:
term.append("if (" +
_where(v.esfilter, lambda x: self._translate(x)) +
") " + value2MVEL(domain.getKey(v)) + "; else ")
term.append(value2MVEL(domain.getKey(domain.NULL)))
func_name = "_temp" + UID()
return self.register_function("+\"|\"+".join(term))
def to_ruby(self):
if self.var == ".":
return "_source"
else:
q = convert.string2quote(self.var)
return "(doc[" + q + "].isEmpty() ? null : doc[" + q + "].value)"
def _where(esFilter, _translate):
if not esFilter or esFilter is True:
return "true"
keys = esFilter.keys()
if len(keys) != 1:
Log.error("Expecting only one filter aggregate")
op = keys[0]
if op == "and":
list = esFilter[op]
if not (list):
return "true"
if len(list) == 1:
return _where(list[0], _translate)
output = "(" + " && ".join(_where(l, _translate) for l in list) + ")"
return output
elif op == "or":
list = esFilter[op]
if not list:
return "false"
if len(list) == 1:
return _where(list[0], _translate)
output = "(" + " || ".join(_where(l, _translate) for l in list) + ")"
return output
elif op == "not":
return "!(" + _where(esFilter[op, _translate]) + ")"
elif op == "term":
pair = esFilter[op]
if len(pair.keys()) == 1:
return [
_translate(k) + "==" + value2MVEL(v) for k, v in pair.items()
][0]
else:
return "(" + " && ".join(
_translate(k) + "==" + value2MVEL(v)
for k, v in pair.items()) + ")"
elif op == "terms":
output = []
for variableName, valueList in esFilter[op].items():
if not valueList:
Log.error("Expecting something in 'terms' array")
if len(valueList) == 1:
output.append(
_translate(variableName) + "==" + value2MVEL(valueList[0]))
else:
output.append("(" + " || ".join(
_translate(variableName) + "==" + value2MVEL(v)
for v in valueList) + ")")
return " && ".join(output)
elif op == "exists":
# "exists":{"field":"myField"}
pair = esFilter[op]
variableName = pair.field
return "(" + _translate(variableName) + "!=null)"
elif op == "missing":
fieldName = _translate(esFilter[op].field)
testExistence = coalesce(esFilter[op].existence, True)
testNull = coalesce(esFilter[op].null_value, True)
output = []
if testExistence and not testNull:
output.append(
"(" + fieldName.replace(".?", ".") + " == empty)"
) # REMOVE THE .? SO WE REFER TO THE FIELD, NOT GET THE VALUE
if testNull:
output.append("(" + fieldName + "==null)")
return " || ".join(output)
elif op == "range":
pair = esFilter[op]
ranges = []
for variableName, r in pair.items():
if r.gte:
ranges.append(
value2MVEL(r.gte) + "<=" + _translate(variableName))
elif r.gt:
ranges.append(
value2MVEL(r.gt) + "<" + _translate(variableName))
elif r["from"]:
if r.include_lower == None or r.include_lower:
ranges.append(
value2MVEL(r["from"]) + "<=" +
_translate(variableName))
else:
ranges.append(
value2MVEL(r["from"]) + "<" + _translate(variableName))
if r.lte:
ranges.append(
value2MVEL(r.lte) + ">=" + _translate(variableName))
elif r.lt:
ranges.append(
value2MVEL(r.lt) + ">" + _translate(variableName))
elif r["from"]:
if r.include_lower == None or r.include_lower:
ranges.append(
value2MVEL(r["from"]) + ">=" +
_translate(variableName))
else:
ranges.append(
value2MVEL(r["from"]) + ">" + _translate(variableName))
return "(" + " && ".join(ranges) + ")"
elif op == "script":
script = esFilter[op].script
return _translate(script)
elif op == "prefix":
pair = esFilter[op]
variableName, value = pair.items()[0]
return _translate(
variableName) + ".startsWith(" + convert.string2quote(value) + ")"
elif op == "match_all":
return "true"
else:
Log.error("'" + op + "' is an unknown aggregate")
return ""
def quote_table(column):
return convert.string2quote(column)
def to_ruby(self):
return qb_expression_to_ruby(self.field)+".start_with? "+convert.string2quote(self.prefix)
def DataClass(name, columns, constraint=True):
"""
Use the DataClass to define a class, but with some extra features:
1. restrict the datatype of property
2. restrict if `required`, or if `nulls` are allowed
3. generic constraints on object properties
It is expected that this class become a real class (or be removed) in the
long term because it is expensive to use and should only be good for
verifying program correctness, not user input.
:param name: Name of the class we are creating
:param columns: Each columns[i] has properties {
"name", - (required) name of the property
"required", - False if it must be defined (even if None)
"nulls", - True if property can be None, or missing
"default", - A default value, if none is provided
"type" - a Python datatype
}
:param constraint: a JSON query Expression for extra constraints
:return: The class that has been created
"""
columns = wrap([{
"name": c,
"required": True,
"nulls": False,
"type": object
} if isinstance(c, text_type) else c for c in columns])
slots = columns.name
required = wrap(
filter(lambda c: c.required and not c.nulls and not c.default,
columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
defaults = {c.name: coalesce(c.default, None) for c in columns}
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template(
"""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
defaults_ = {{defaults}}
class {{class_name}}(Mapping):
__slots__ = {{slots}}
def _constraint(row, rownum, rows):
return {{constraint_expr}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
object.__setattr__(self, s, kwargs.get(s, {{defaults}}.get(s, None)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
object.__setattr__(self, item, value)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{class_name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{class_name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
""", {
"class_name":
name,
"slots":
"(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required":
"{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls":
"{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"defaults":
jx_expression({
"literal": defaults
}).to_python(),
"len_slots":
len(slots),
"dict":
"{" +
(", ".join(convert.value2quote(s) + ": self." + s
for s in slots)) + "}",
"assign":
"; ".join(
"_set(output, " + convert.value2quote(s) + ", self." + s + ")"
for s in slots),
"types":
"{" + (",".join(
convert.string2quote(k) + ": " + v.__name__
for k, v in types.items())) + "}",
"constraint_expr":
jx_expression(constraint).to_python(),
"constraint":
value2json(constraint)
})
return _exec(code, name)
def qb_expression_to_ruby(expr):
if expr == None:
return "nil"
elif Math.is_number(expr):
return unicode(expr)
elif is_keyword(expr):
return "doc[" + convert.string2quote(expr) + "].value"
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, CODE):
return expr.code
elif isinstance(expr, Date):
return unicode(expr.unix)
elif expr is True:
return "true"
elif expr is False:
return "false"
op, term = expr.items()[0]
mop = ruby_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(a) + ")" + mop[0] + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
qb_expression_to_ruby(term)
bop = ruby_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_ruby(a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = ruby_unary_operators.get(op)
if uop:
output = expand_template(uop, {"term": qb_expression_to_ruby(term)})
return output
cop = complex_operators.get(op)
if cop:
output = cop(term).to_ruby()
return output
Log.error("`{{op}}` is not a recognized operation", op= op)
def quote(value):
if not _convert:
_late_import()
return _convert.string2quote(value)
