def to_python(self, not_null=False, boolean=False, many=False):
path = split_field(self.var)
agg = "row"
if not path:
return agg
elif path[0] in ["row", "rownum"]:
# MAGIC VARIABLES
agg = path[0]
path = path[1:]
if len(path) == 0:
return agg
elif path[0] == "rows":
if len(path) == 1:
return "rows"
elif path[1] in ["first", "last"]:
agg = "rows." + path[1] + "()"
path = path[2:]
else:
Log.error("do not know what {{var}} of `rows` is", var=path[1])
for p in path[:-1]:
if not_null:
agg = agg + ".get(" + convert.value2quote(p) + ")"
else:
agg = agg + ".get(" + convert.value2quote(p) + ", EMPTY_DICT)"
output = agg + ".get(" + convert.value2quote(path[-1]) + ")"
if many:
output = "listwrap(" + output + ")"
return output
def to_python(self, not_null=False, boolean=False, many=False):
path = split_field(self.var)
agg = "row"
if not path:
return agg
elif path[0] in ["row", "rownum"]:
# MAGIC VARIABLES
agg = path[0]
path = path[1:]
if len(path) == 0:
return agg
elif path[0] == "rows":
if len(path) == 1:
return "rows"
elif path[1] in ["first", "last"]:
agg = "rows." + path[1] + "()"
path = path[2:]
else:
Log.error("do not know what {{var}} of `rows` is", var=path[1])
for p in path[:-1]:
if not_null:
agg = agg + ".get(" + convert.value2quote(p) + ")"
else:
agg = agg + ".get(" + convert.value2quote(p) + ", EMPTY_DICT)"
output = agg + ".get(" + convert.value2quote(path[-1]) + ")"
if many:
output = "listwrap(" + output + ")"
return output
def to_python(self, not_null=False, boolean=False, many=False):
agg = "rows[rownum+" + self.offset.to_python() + "]"
path = split_field(json2value(self.var.json))
if not path:
return agg
for p in path[:-1]:
agg = agg + ".get(" + convert.value2quote(p) + ", EMPTY_DICT)"
return agg + ".get(" + convert.value2quote(path[-1]) + ")"
def to_python(self, not_null=False, boolean=False, many=False):
agg = "rows[rownum+" + self.offset.to_python() + "]"
path = split_field(json2value(self.var.json))
if not path:
return agg
for p in path[:-1]:
agg = agg + ".get(" + convert.value2quote(p) + ", EMPTY_DICT)"
return agg + ".get(" + convert.value2quote(path[-1]) + ")"
def compile_expression(source):
"""
THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE
:param source: PYTHON SOURCE CODE
:return: PYTHON FUNCTION
"""
# FORCE MODULES TO BE IN NAMESPACE
_ = coalesce
_ = Date
_ = convert
_ = Log
_ = Data
_ = EMPTY_DICT
_ = re
output = None
exec """
def output(row, rownum=None, rows=None):
try:
return """ + source + """
except Exception as e:
Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote(
source) + """, cause=e)
"""
return output
def compile_expression(source):
"""
THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE
:param source: PYTHON SOURCE CODE
:return: PYTHON FUNCTION
"""
# FORCE MODULES TO BE IN NAMESPACE
_ = coalesce
_ = Date
_ = convert
_ = Log
_ = Data
_ = EMPTY_DICT
_ = re
output = None
exec """
def output(row, rownum=None, rows=None):
try:
return """ + source + """
except Exception, e:
Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote(source) + """, cause=e)
"""
return output
def compile_expression(source):
# FORCE MODULES TO BE IN NAMESPACE
_ = coalesce
_ = Date
_ = convert
output = None
exec """
def output(row, rownum=None, rows=None):
try:
return """ + source + """
except Exception, e:
Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote(source) + """, cause=e)
"""
return output
def compile_expression(source):
# FORCE MODULES TO BE IN NAMESPACE
_ = coalesce
_ = Date
_ = convert
output = None
exec """
def output(row, rownum=None, rows=None):
try:
return """ + source + """
except Exception, e:
Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote(
source) + """, cause=e)
"""
return output
def compile_expression(source):
"""
THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE
:param source: PYTHON SOURCE CODE
:return: PYTHON FUNCTION
"""
# FORCE MODULES TO BE IN NAMESPACE
_ = coalesce
_ = listwrap
_ = Date
_ = convert
_ = Log
_ = Data
_ = EMPTY_DICT
_ = re
_ = wrap_leaves
_ = is_data
fake_locals = {}
try:
exec(
"""
def output(row, rownum=None, rows=None):
_source = """ + convert.value2quote(source) + """
try:
return """ + source + """
except Exception as e:
Log.error("Problem with dynamic function {{func|quote}}", func=_source, cause=e)
""",
globals(),
fake_locals
)
except Exception as e:
Log.error("Bad source: {{source}}", source=source, cause=e)
return fake_locals['output']
def compile_expression(source):
"""
THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE
:param source: PYTHON SOURCE CODE
:return: PYTHON FUNCTION
"""
# FORCE MODULES TO BE IN NAMESPACE
_ = coalesce
_ = listwrap
_ = Date
_ = convert
_ = Log
_ = Data
_ = EMPTY_DICT
_ = re
_ = wrap_leaves
fake_locals = {}
try:
exec(
"""
def output(row, rownum=None, rows=None):
_source = """ + convert.value2quote(source) + """
try:
return """ + source + """
except Exception as e:
Log.error("Problem with dynamic function {{func|quote}}", func=_source, cause=e)
""",
globals(),
fake_locals
)
except Exception as e:
Log.error("Bad source: {{source}}", source=source, cause=e)
return fake_locals['output']
def qb_expression_to_python(expr):
if expr == None:
return "None"
elif Math.is_number(expr):
return unicode(expr)
elif isinstance(expr, Date):
return unicode(expr.unix)
elif isinstance(expr, unicode):
if expr == ".":
return "row"
elif is_keyword(expr):
return "row[" + convert.value2quote(expr) + "]"
else:
Log.error("Expecting a json path")
elif isinstance(expr, CODE):
return expr.code
elif expr is True:
return "True"
elif expr is False:
return "False"
op, term = expr.items()[0]
mop = python_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(
["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_python(
a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")"
return output
else:
qb_expression_to_python(term)
bop = python_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(
["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_python(a) + ")" +
bop + "(" + qb_expression_to_python(b) +
")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_python(
a) + ")" + bop + "(" + qb_expression_to_python(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = python_unary_operators.get(op)
if uop:
output = uop + "(" + qb_expression_to_python(term) + ")"
return output
Log.error("`{{op}}` is not a recognized operation", op=op)
def quote_column(self, column_name, table=None):
if table != None:
return SQL(convert.value2quote(table) + "." + convert.value2quote(column_name))
else:
return SQL(convert.value2quote(column_name))
def DataClass(name, columns, constraint=True):
"""
Use the DataClass to define a class, but with some extra features:
1. restrict the datatype of property
2. restrict if `required`, or if `nulls` are allowed
3. generic constraints on object properties
It is expected that this class become a real class (or be removed) in the
long term because it is expensive to use and should only be good for
verifying program correctness, not user input.
:param name: Name of the class we are creating
:param columns: Each columns[i] has properties {
"name", - (required) name of the property
"required", - False if it must be defined (even if None)
"nulls", - True if property can be None, or missing
"default", - A default value, if none is provided
"type" - a Python datatype
}
:param constraint: a JSON query Expression for extra constraints
:return: The class that has been created
"""
columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, basestring) else c for c in columns])
slots = columns.name
required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
defaults = {c.name: coalesce(c.default, None) for c in columns}
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template(
"""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
defaults_ = {{defaults}}
class {{class_name}}(Mapping):
__slots__ = {{slots}}
def _constraint(row, rownum, rows):
return {{constraint_expr}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
object.__setattr__(self, s, kwargs.get(s, {{defaults}}.get(s, None)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
object.__setattr__(self, item, value)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{class_name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{class_name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
temp = {{class_name}}
""",
{
"class_name": name,
"slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"defaults": jx_expression({"literal": defaults}).to_python(),
"len_slots": len(slots),
"dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}",
"assign": "; ".join("_set(output, "+convert.value2quote(s)+", self."+s+")" for s in slots),
"types": "{" + (",".join(convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}",
"constraint_expr": jx_expression(constraint).to_python(),
"constraint": convert.value2json(constraint)
}
)
return _exec(code, name)
def DataClass(name, columns):
"""
Each column has {"name", "required", "nulls", "default", "type"} properties
"""
columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, basestring) else c for c in columns])
slots = columns.name
required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template("""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
class {{name}}(Mapping):
__slots__ = {{slots}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
setattr(self, s, kwargs.get(s, kwargs.get('default', Null)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
#if not isinstance(value, types_[item]):
# Log.error("{"+"{item|quote}} not of type "+"{"+"{type}}", item=item, type=types_[item])
object.__setattr__(self, item, value)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
temp = {{name}}
""",
{
"name": name,
"slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"len_slots": len(slots),
"dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}",
"assign": "; ".join("_set(output, "+convert.value2quote(s)+", self."+s+")" for s in slots),
"types": "{" + (",".join(convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}"
}
)
return _exec(code, name)
def qb_expression_to_python(expr):
if expr == None:
return "None"
elif Math.is_number(expr):
return unicode(expr)
elif isinstance(expr, Date):
return unicode(expr.unix)
elif isinstance(expr, unicode):
if expr == ".":
return "row"
elif is_keyword(expr):
return "row[" + convert.value2quote(expr) + "]"
else:
Log.error("Expecting a json path")
elif isinstance(expr, CODE):
return expr.code
elif expr is True:
return "True"
elif expr is False:
return "False"
op, term = expr.items()[0]
mop = python_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_python(a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")"
return output
else:
qb_expression_to_python(term)
bop = python_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = python_unary_operators.get(op)
if uop:
output = uop + "(" + qb_expression_to_python(term) + ")"
return output
Log.error("`{{op}}` is not a recognized operation", op= op)
def DataClass(name, columns):
"""
Each column has {"name", "required", "nulls", "default", "type"} properties
"""
columns = wrap([{
"name": c,
"required": True,
"nulls": False,
"type": object
} if isinstance(c, basestring) else c for c in columns])
slots = columns.name
required = wrap(
filter(lambda c: c.required and not c.nulls and not c.default,
columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template(
"""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
class {{name}}(Mapping):
__slots__ = {{slots}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
setattr(self, s, kwargs.get(s, kwargs.get('default', Null)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
#if not isinstance(value, types_[item]):
# Log.error("{"+"{item|quote}} not of type "+"{"+"{type}}", item=item, type=types_[item])
object.__setattr__(self, item, value)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
temp = {{name}}
""", {
"name":
name,
"slots":
"(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required":
"{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls":
"{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"len_slots":
len(slots),
"dict":
"{" +
(", ".join(convert.value2quote(s) + ": self." + s
for s in slots)) + "}",
"assign":
"; ".join(
"_set(output, " + convert.value2quote(s) + ", self." + s + ")"
for s in slots),
"types":
"{" + (",".join(
convert.string2quote(k) + ": " + v.__name__
for k, v in types.items())) + "}"
})
return _exec(code, name)
def to_python(self):
if self.var == ".":
return "row"
else:
return "row[" + convert.value2quote(self.var) + "]"
def quote_column(self, column_name, table=None):
if table != None:
return SQL(convert.value2quote(table) + "." + convert.value2quote(column_name))
else:
return SQL(convert.value2quote(column_name))
def DataClass(name, columns, constraint=True):
"""
Use the DataClass to define a class, but with some extra features:
1. restrict the datatype of property
2. restrict if `required`, or if `nulls` are allowed
3. generic constraints on object properties
It is expected that this class become a real class (or be removed) in the
long term because it is expensive to use and should only be good for
verifying program correctness, not user input.
:param name: Name of the class we are creating
:param columns: Each columns[i] has properties {
"name", - (required) name of the property
"required", - False if it must be defined (even if None)
"nulls", - True if property can be None, or missing
"default", - A default value, if none is provided
"type" - a Python datatype
}
:param constraint: a JSON query Expression for extra constraints
:return: The class that has been created
"""
columns = wrap([{
"name": c,
"required": True,
"nulls": False,
"type": object
} if isinstance(c, text_type) else c for c in columns])
slots = columns.name
required = wrap(
filter(lambda c: c.required and not c.nulls and not c.default,
columns)).name
nulls = wrap(filter(lambda c: c.nulls, columns)).name
defaults = {c.name: coalesce(c.default, None) for c in columns}
types = {c.name: coalesce(c.type, object) for c in columns}
code = expand_template(
"""
from __future__ import unicode_literals
from collections import Mapping
meta = None
types_ = {{types}}
defaults_ = {{defaults}}
class {{class_name}}(Mapping):
__slots__ = {{slots}}
def _constraint(row, rownum, rows):
return {{constraint_expr}}
def __init__(self, **kwargs):
if not kwargs:
return
for s in {{slots}}:
object.__setattr__(self, s, kwargs.get(s, {{defaults}}.get(s, None)))
missed = {{required}}-set(kwargs.keys())
if missed:
Log.error("Expecting properties {"+"{missed}}", missed=missed)
illegal = set(kwargs.keys())-set({{slots}})
if illegal:
Log.error("{"+"{names}} are not a valid properties", names=illegal)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getitem__(self, item):
return getattr(self, item)
def __setitem__(self, item, value):
setattr(self, item, value)
return self
def __setattr__(self, item, value):
if item not in {{slots}}:
Log.error("{"+"{item|quote}} not valid attribute", item=item)
object.__setattr__(self, item, value)
if not self._constraint(0, [self]):
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
def __getattr__(self, item):
Log.error("{"+"{item|quote}} not valid attribute", item=item)
def __hash__(self):
return object.__hash__(self)
def __eq__(self, other):
if isinstance(other, {{class_name}}) and dict(self)==dict(other) and self is not other:
Log.error("expecting to be same object")
return self is other
def __dict__(self):
return {k: getattr(self, k) for k in {{slots}}}
def items(self):
return ((k, getattr(self, k)) for k in {{slots}})
def __copy__(self):
_set = object.__setattr__
output = object.__new__({{class_name}})
{{assign}}
return output
def __iter__(self):
return {{slots}}.__iter__()
def __len__(self):
return {{len_slots}}
def __str__(self):
return str({{dict}})
""", {
"class_name":
name,
"slots":
"(" + (", ".join(convert.value2quote(s) for s in slots)) + ")",
"required":
"{" + (", ".join(convert.value2quote(s) for s in required)) + "}",
"nulls":
"{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}",
"defaults":
jx_expression({
"literal": defaults
}).to_python(),
"len_slots":
len(slots),
"dict":
"{" +
(", ".join(convert.value2quote(s) + ": self." + s
for s in slots)) + "}",
"assign":
"; ".join(
"_set(output, " + convert.value2quote(s) + ", self." + s + ")"
for s in slots),
"types":
"{" + (",".join(
convert.string2quote(k) + ": " + v.__name__
for k, v in types.items())) + "}",
"constraint_expr":
jx_expression(constraint).to_python(),
"constraint":
value2json(constraint)
})
return _exec(code, name)
