def to_python(self, not_null=False, boolean=False, many=False): path = split_field(self.var) agg = "row" if not path: return agg elif path[0] in ["row", "rownum"]: # MAGIC VARIABLES agg = path[0] path = path[1:] if len(path) == 0: return agg elif path[0] == "rows": if len(path) == 1: return "rows" elif path[1] in ["first", "last"]: agg = "rows." + path[1] + "()" path = path[2:] else: Log.error("do not know what {{var}} of `rows` is", var=path[1]) for p in path[:-1]: if not_null: agg = agg + ".get(" + convert.value2quote(p) + ")" else: agg = agg + ".get(" + convert.value2quote(p) + ", EMPTY_DICT)" output = agg + ".get(" + convert.value2quote(path[-1]) + ")" if many: output = "listwrap(" + output + ")" return output
def to_python(self, not_null=False, boolean=False, many=False): agg = "rows[rownum+" + self.offset.to_python() + "]" path = split_field(json2value(self.var.json)) if not path: return agg for p in path[:-1]: agg = agg + ".get(" + convert.value2quote(p) + ", EMPTY_DICT)" return agg + ".get(" + convert.value2quote(path[-1]) + ")"
def compile_expression(source): """ THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE :param source: PYTHON SOURCE CODE :return: PYTHON FUNCTION """ # FORCE MODULES TO BE IN NAMESPACE _ = coalesce _ = Date _ = convert _ = Log _ = Data _ = EMPTY_DICT _ = re output = None exec """ def output(row, rownum=None, rows=None): try: return """ + source + """ except Exception as e: Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote( source) + """, cause=e) """ return output
def compile_expression(source): """ THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE :param source: PYTHON SOURCE CODE :return: PYTHON FUNCTION """ # FORCE MODULES TO BE IN NAMESPACE _ = coalesce _ = Date _ = convert _ = Log _ = Data _ = EMPTY_DICT _ = re output = None exec """ def output(row, rownum=None, rows=None): try: return """ + source + """ except Exception, e: Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote(source) + """, cause=e) """ return output
def compile_expression(source): # FORCE MODULES TO BE IN NAMESPACE _ = coalesce _ = Date _ = convert output = None exec """ def output(row, rownum=None, rows=None): try: return """ + source + """ except Exception, e: Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote(source) + """, cause=e) """ return output
def compile_expression(source): # FORCE MODULES TO BE IN NAMESPACE _ = coalesce _ = Date _ = convert output = None exec """ def output(row, rownum=None, rows=None): try: return """ + source + """ except Exception, e: Log.error("Problem with dynamic function {{func|quote}}", func= """ + convert.value2quote( source) + """, cause=e) """ return output
def compile_expression(source): """ THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE :param source: PYTHON SOURCE CODE :return: PYTHON FUNCTION """ # FORCE MODULES TO BE IN NAMESPACE _ = coalesce _ = listwrap _ = Date _ = convert _ = Log _ = Data _ = EMPTY_DICT _ = re _ = wrap_leaves _ = is_data fake_locals = {} try: exec( """ def output(row, rownum=None, rows=None): _source = """ + convert.value2quote(source) + """ try: return """ + source + """ except Exception as e: Log.error("Problem with dynamic function {{func|quote}}", func=_source, cause=e) """, globals(), fake_locals ) except Exception as e: Log.error("Bad source: {{source}}", source=source, cause=e) return fake_locals['output']
def compile_expression(source): """ THIS FUNCTION IS ON ITS OWN FOR MINIMAL GLOBAL NAMESPACE :param source: PYTHON SOURCE CODE :return: PYTHON FUNCTION """ # FORCE MODULES TO BE IN NAMESPACE _ = coalesce _ = listwrap _ = Date _ = convert _ = Log _ = Data _ = EMPTY_DICT _ = re _ = wrap_leaves fake_locals = {} try: exec( """ def output(row, rownum=None, rows=None): _source = """ + convert.value2quote(source) + """ try: return """ + source + """ except Exception as e: Log.error("Problem with dynamic function {{func|quote}}", func=_source, cause=e) """, globals(), fake_locals ) except Exception as e: Log.error("Bad source: {{source}}", source=source, cause=e) return fake_locals['output']
def qb_expression_to_python(expr): if expr == None: return "None" elif Math.is_number(expr): return unicode(expr) elif isinstance(expr, Date): return unicode(expr.unix) elif isinstance(expr, unicode): if expr == ".": return "row" elif is_keyword(expr): return "row[" + convert.value2quote(expr) + "]" else: Log.error("Expecting a json path") elif isinstance(expr, CODE): return expr.code elif expr is True: return "True" elif expr is False: return "False" op, term = expr.items()[0] mop = python_multi_operators.get(op) if mop: if isinstance(term, list): if not term: return mop[1] # RETURN DEFAULT else: output = mop[0].join( ["(" + qb_expression_to_python(t) + ")" for t in term]) return output elif isinstance(term, Mapping): a, b = term.items()[0] output = "(" + qb_expression_to_python( a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")" return output else: qb_expression_to_python(term) bop = python_binary_operators.get(op) if bop: if isinstance(term, list): output = bop.join( ["(" + qb_expression_to_python(t) + ")" for t in term]) return output elif isinstance(term, Mapping): if op == "eq": # eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE output = " and ".join("(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" for a, b in term.items()) return output else: a, b = term.items()[0] output = "(" + qb_expression_to_python( a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" return output else: Log.error("Expecting binary term") uop = python_unary_operators.get(op) if uop: output = uop + "(" + qb_expression_to_python(term) + ")" return output Log.error("`{{op}}` is not a recognized operation", op=op)
def quote_column(self, column_name, table=None): if table != None: return SQL(convert.value2quote(table) + "." + convert.value2quote(column_name)) else: return SQL(convert.value2quote(column_name))
def DataClass(name, columns, constraint=True): """ Use the DataClass to define a class, but with some extra features: 1. restrict the datatype of property 2. restrict if `required`, or if `nulls` are allowed 3. generic constraints on object properties It is expected that this class become a real class (or be removed) in the long term because it is expensive to use and should only be good for verifying program correctness, not user input. :param name: Name of the class we are creating :param columns: Each columns[i] has properties { "name", - (required) name of the property "required", - False if it must be defined (even if None) "nulls", - True if property can be None, or missing "default", - A default value, if none is provided "type" - a Python datatype } :param constraint: a JSON query Expression for extra constraints :return: The class that has been created """ columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, basestring) else c for c in columns]) slots = columns.name required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name nulls = wrap(filter(lambda c: c.nulls, columns)).name defaults = {c.name: coalesce(c.default, None) for c in columns} types = {c.name: coalesce(c.type, object) for c in columns} code = expand_template( """ from __future__ import unicode_literals from collections import Mapping meta = None types_ = {{types}} defaults_ = {{defaults}} class {{class_name}}(Mapping): __slots__ = {{slots}} def _constraint(row, rownum, rows): return {{constraint_expr}} def __init__(self, **kwargs): if not kwargs: return for s in {{slots}}: object.__setattr__(self, s, kwargs.get(s, {{defaults}}.get(s, None))) missed = {{required}}-set(kwargs.keys()) if missed: Log.error("Expecting properties {"+"{missed}}", missed=missed) illegal = set(kwargs.keys())-set({{slots}}) if illegal: Log.error("{"+"{names}} are not a valid properties", names=illegal) if not self._constraint(0, [self]): Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self) def __getitem__(self, item): return getattr(self, item) def __setitem__(self, item, value): setattr(self, item, value) return self def __setattr__(self, item, value): if item not in {{slots}}: Log.error("{"+"{item|quote}} not valid attribute", item=item) object.__setattr__(self, item, value) if not self._constraint(0, [self]): Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self) def __getattr__(self, item): Log.error("{"+"{item|quote}} not valid attribute", item=item) def __hash__(self): return object.__hash__(self) def __eq__(self, other): if isinstance(other, {{class_name}}) and dict(self)==dict(other) and self is not other: Log.error("expecting to be same object") return self is other def __dict__(self): return {k: getattr(self, k) for k in {{slots}}} def items(self): return ((k, getattr(self, k)) for k in {{slots}}) def __copy__(self): _set = object.__setattr__ output = object.__new__({{class_name}}) {{assign}} return output def __iter__(self): return {{slots}}.__iter__() def __len__(self): return {{len_slots}} def __str__(self): return str({{dict}}) temp = {{class_name}} """, { "class_name": name, "slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")", "required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}", "nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}", "defaults": jx_expression({"literal": defaults}).to_python(), "len_slots": len(slots), "dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}", "assign": "; ".join("_set(output, "+convert.value2quote(s)+", self."+s+")" for s in slots), "types": "{" + (",".join(convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}", "constraint_expr": jx_expression(constraint).to_python(), "constraint": convert.value2json(constraint) } ) return _exec(code, name)
def DataClass(name, columns): """ Each column has {"name", "required", "nulls", "default", "type"} properties """ columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, basestring) else c for c in columns]) slots = columns.name required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name nulls = wrap(filter(lambda c: c.nulls, columns)).name types = {c.name: coalesce(c.type, object) for c in columns} code = expand_template(""" from __future__ import unicode_literals from collections import Mapping meta = None types_ = {{types}} class {{name}}(Mapping): __slots__ = {{slots}} def __init__(self, **kwargs): if not kwargs: return for s in {{slots}}: setattr(self, s, kwargs.get(s, kwargs.get('default', Null))) missed = {{required}}-set(kwargs.keys()) if missed: Log.error("Expecting properties {"+"{missed}}", missed=missed) illegal = set(kwargs.keys())-set({{slots}}) if illegal: Log.error("{"+"{names}} are not a valid properties", names=illegal) def __getitem__(self, item): return getattr(self, item) def __setitem__(self, item, value): setattr(self, item, value) return self def __setattr__(self, item, value): if item not in {{slots}}: Log.error("{"+"{item|quote}} not valid attribute", item=item) #if not isinstance(value, types_[item]): # Log.error("{"+"{item|quote}} not of type "+"{"+"{type}}", item=item, type=types_[item]) object.__setattr__(self, item, value) def __getattr__(self, item): Log.error("{"+"{item|quote}} not valid attribute", item=item) def __hash__(self): return object.__hash__(self) def __eq__(self, other): if isinstance(other, {{name}}) and dict(self)==dict(other) and self is not other: Log.error("expecting to be same object") return self is other def __dict__(self): return {k: getattr(self, k) for k in {{slots}}} def items(self): return ((k, getattr(self, k)) for k in {{slots}}) def __copy__(self): _set = object.__setattr__ output = object.__new__({{name}}) {{assign}} return output def __iter__(self): return {{slots}}.__iter__() def __len__(self): return {{len_slots}} def __str__(self): return str({{dict}}) temp = {{name}} """, { "name": name, "slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")", "required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}", "nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}", "len_slots": len(slots), "dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}", "assign": "; ".join("_set(output, "+convert.value2quote(s)+", self."+s+")" for s in slots), "types": "{" + (",".join(convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}" } ) return _exec(code, name)
def qb_expression_to_python(expr): if expr == None: return "None" elif Math.is_number(expr): return unicode(expr) elif isinstance(expr, Date): return unicode(expr.unix) elif isinstance(expr, unicode): if expr == ".": return "row" elif is_keyword(expr): return "row[" + convert.value2quote(expr) + "]" else: Log.error("Expecting a json path") elif isinstance(expr, CODE): return expr.code elif expr is True: return "True" elif expr is False: return "False" op, term = expr.items()[0] mop = python_multi_operators.get(op) if mop: if isinstance(term, list): if not term: return mop[1] # RETURN DEFAULT else: output = mop[0].join(["(" + qb_expression_to_python(t) + ")" for t in term]) return output elif isinstance(term, Mapping): a, b = term.items()[0] output = "(" + qb_expression_to_python(a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")" return output else: qb_expression_to_python(term) bop = python_binary_operators.get(op) if bop: if isinstance(term, list): output = bop.join(["(" + qb_expression_to_python(t) + ")" for t in term]) return output elif isinstance(term, Mapping): if op == "eq": # eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE output = " and ".join("(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" for a, b in term.items()) return output else: a, b = term.items()[0] output = "(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" return output else: Log.error("Expecting binary term") uop = python_unary_operators.get(op) if uop: output = uop + "(" + qb_expression_to_python(term) + ")" return output Log.error("`{{op}}` is not a recognized operation", op= op)
def DataClass(name, columns): """ Each column has {"name", "required", "nulls", "default", "type"} properties """ columns = wrap([{ "name": c, "required": True, "nulls": False, "type": object } if isinstance(c, basestring) else c for c in columns]) slots = columns.name required = wrap( filter(lambda c: c.required and not c.nulls and not c.default, columns)).name nulls = wrap(filter(lambda c: c.nulls, columns)).name types = {c.name: coalesce(c.type, object) for c in columns} code = expand_template( """ from __future__ import unicode_literals from collections import Mapping meta = None types_ = {{types}} class {{name}}(Mapping): __slots__ = {{slots}} def __init__(self, **kwargs): if not kwargs: return for s in {{slots}}: setattr(self, s, kwargs.get(s, kwargs.get('default', Null))) missed = {{required}}-set(kwargs.keys()) if missed: Log.error("Expecting properties {"+"{missed}}", missed=missed) illegal = set(kwargs.keys())-set({{slots}}) if illegal: Log.error("{"+"{names}} are not a valid properties", names=illegal) def __getitem__(self, item): return getattr(self, item) def __setitem__(self, item, value): setattr(self, item, value) return self def __setattr__(self, item, value): if item not in {{slots}}: Log.error("{"+"{item|quote}} not valid attribute", item=item) #if not isinstance(value, types_[item]): # Log.error("{"+"{item|quote}} not of type "+"{"+"{type}}", item=item, type=types_[item]) object.__setattr__(self, item, value) def __getattr__(self, item): Log.error("{"+"{item|quote}} not valid attribute", item=item) def __hash__(self): return object.__hash__(self) def __eq__(self, other): if isinstance(other, {{name}}) and dict(self)==dict(other) and self is not other: Log.error("expecting to be same object") return self is other def __dict__(self): return {k: getattr(self, k) for k in {{slots}}} def items(self): return ((k, getattr(self, k)) for k in {{slots}}) def __copy__(self): _set = object.__setattr__ output = object.__new__({{name}}) {{assign}} return output def __iter__(self): return {{slots}}.__iter__() def __len__(self): return {{len_slots}} def __str__(self): return str({{dict}}) temp = {{name}} """, { "name": name, "slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")", "required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}", "nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}", "len_slots": len(slots), "dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}", "assign": "; ".join( "_set(output, " + convert.value2quote(s) + ", self." + s + ")" for s in slots), "types": "{" + (",".join( convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}" }) return _exec(code, name)
def to_python(self): if self.var == ".": return "row" else: return "row[" + convert.value2quote(self.var) + "]"
def DataClass(name, columns, constraint=True): """ Use the DataClass to define a class, but with some extra features: 1. restrict the datatype of property 2. restrict if `required`, or if `nulls` are allowed 3. generic constraints on object properties It is expected that this class become a real class (or be removed) in the long term because it is expensive to use and should only be good for verifying program correctness, not user input. :param name: Name of the class we are creating :param columns: Each columns[i] has properties { "name", - (required) name of the property "required", - False if it must be defined (even if None) "nulls", - True if property can be None, or missing "default", - A default value, if none is provided "type" - a Python datatype } :param constraint: a JSON query Expression for extra constraints :return: The class that has been created """ columns = wrap([{ "name": c, "required": True, "nulls": False, "type": object } if isinstance(c, text_type) else c for c in columns]) slots = columns.name required = wrap( filter(lambda c: c.required and not c.nulls and not c.default, columns)).name nulls = wrap(filter(lambda c: c.nulls, columns)).name defaults = {c.name: coalesce(c.default, None) for c in columns} types = {c.name: coalesce(c.type, object) for c in columns} code = expand_template( """ from __future__ import unicode_literals from collections import Mapping meta = None types_ = {{types}} defaults_ = {{defaults}} class {{class_name}}(Mapping): __slots__ = {{slots}} def _constraint(row, rownum, rows): return {{constraint_expr}} def __init__(self, **kwargs): if not kwargs: return for s in {{slots}}: object.__setattr__(self, s, kwargs.get(s, {{defaults}}.get(s, None))) missed = {{required}}-set(kwargs.keys()) if missed: Log.error("Expecting properties {"+"{missed}}", missed=missed) illegal = set(kwargs.keys())-set({{slots}}) if illegal: Log.error("{"+"{names}} are not a valid properties", names=illegal) if not self._constraint(0, [self]): Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self) def __getitem__(self, item): return getattr(self, item) def __setitem__(self, item, value): setattr(self, item, value) return self def __setattr__(self, item, value): if item not in {{slots}}: Log.error("{"+"{item|quote}} not valid attribute", item=item) object.__setattr__(self, item, value) if not self._constraint(0, [self]): Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self) def __getattr__(self, item): Log.error("{"+"{item|quote}} not valid attribute", item=item) def __hash__(self): return object.__hash__(self) def __eq__(self, other): if isinstance(other, {{class_name}}) and dict(self)==dict(other) and self is not other: Log.error("expecting to be same object") return self is other def __dict__(self): return {k: getattr(self, k) for k in {{slots}}} def items(self): return ((k, getattr(self, k)) for k in {{slots}}) def __copy__(self): _set = object.__setattr__ output = object.__new__({{class_name}}) {{assign}} return output def __iter__(self): return {{slots}}.__iter__() def __len__(self): return {{len_slots}} def __str__(self): return str({{dict}}) """, { "class_name": name, "slots": "(" + (", ".join(convert.value2quote(s) for s in slots)) + ")", "required": "{" + (", ".join(convert.value2quote(s) for s in required)) + "}", "nulls": "{" + (", ".join(convert.value2quote(s) for s in nulls)) + "}", "defaults": jx_expression({ "literal": defaults }).to_python(), "len_slots": len(slots), "dict": "{" + (", ".join(convert.value2quote(s) + ": self." + s for s in slots)) + "}", "assign": "; ".join( "_set(output, " + convert.value2quote(s) + ", self." + s + ")" for s in slots), "types": "{" + (",".join( convert.string2quote(k) + ": " + v.__name__ for k, v in types.items())) + "}", "constraint_expr": jx_expression(constraint).to_python(), "constraint": value2json(constraint) }) return _exec(code, name)