def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from mo_logs import Log
Log.error("Do not know how to handle")
r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)])
output = floor(m / amount.month) + r
return output
elif is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def quote_value(value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return SQL_NULL
elif isinstance(value, SQL):
return quote_sql(value.template, value.param)
elif is_text(value):
return SQL("'" + "".join(ESCAPE_DCT.get(c, c) for c in value) + "'")
elif is_data(value):
return quote_value(json_encode(value))
elif is_number(value):
return SQL(text_type(value))
elif isinstance(value, datetime):
return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')")
elif isinstance(value, Date):
return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')")
elif hasattr(value, '__iter__'):
return quote_value(json_encode(value))
else:
return quote_value(text_type(value))
except Exception as e:
Log.error("problem quoting SQL {{value}}", value=repr(value), cause=e)
def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, delta=None):
"""
Snagged from unittest/case.py, then modified (Aug2014)
"""
if expected is NULL:
if test == None: # pandas dataframes reject any comparision with an exception!
return
else:
raise AssertionError(expand_template("{{test}} != {{expected}}", locals()))
if expected == None: # None has no expectations
return
if test == expected:
# shortcut
return
if not is_number(expected):
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
if is_list(expected) and len(expected) == 0 and test == None:
return
if is_data(expected) and not expected.keys() and test == None:
return
if test != expected:
raise AssertionError(expand_template("{{test}} != {{expected}}", locals()))
return
num_param = 0
if digits != None:
num_param += 1
if places != None:
num_param += 1
if delta != None:
num_param += 1
if num_param>1:
raise TypeError("specify only one of digits, places or delta")
if digits is not None:
with suppress_exception:
diff = log10(abs(test-expected))
if diff < digits:
return
standardMsg = expand_template("{{test}} != {{expected}} within {{digits}} decimal places", locals())
elif delta is not None:
if abs(test - expected) <= delta:
return
standardMsg = expand_template("{{test}} != {{expected}} within {{delta}} delta", locals())
else:
if places is None:
places = 15
with suppress_exception:
diff = mo_math.log10(abs(test-expected))
if diff < mo_math.ceiling(mo_math.log10(abs(test)))-places:
return
standardMsg = expand_template("{{test|json}} != {{expected|json}} within {{places}} places", locals())
raise AssertionError(coalesce(msg, "") + ": (" + standardMsg + ")")
def convert(self, expr):
"""
EXPAND INSTANCES OF name TO value
"""
if expr is True or expr == None or expr is False:
return expr
elif is_number(expr):
return expr
elif expr == ".":
return "."
elif is_variable_name(expr):
return coalesce(self.dimensions[expr], expr)
elif is_text(expr):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, Date):
return expr
elif is_op(expr, QueryOp):
return self._convert_query(expr)
elif is_data(expr):
if expr["from"]:
return self._convert_query(expr)
elif len(expr) >= 2:
#ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
return dict_to_data({name: self.convert(value) for name, value in expr.leaves()})
else:
# ASSUME SINGLE-CLAUSE EXPRESSION
k, v = expr.items()[0]
return converter_map.get(k, self._convert_bop)(self, k, v)
elif is_many(expr):
return list_to_data([self.convert(value) for value in expr])
else:
return expr
def __new__(cls, value=None, **kwargs):
output = object.__new__(cls)
if value == None:
if kwargs:
output.milli = datetime.timedelta(
**kwargs).total_seconds() * 1000
output.month = 0
return output
else:
return None
if is_number(value):
output._milli = float(value) * 1000
output.month = 0
return output
elif is_text(value):
return parse(value)
elif isinstance(value, Duration):
output.milli = value.milli
output.month = value.month
return output
elif isinstance(value, float) and is_nan(value):
return None
else:
from mo_logs import Log
Log.error("Do not know type of object (" +
get_module("mo_json").value2json(value) +
")of to make a Duration")
def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from mo_logs import Log
Log.error("Do not know how to handle")
r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)])
output = floor(m / amount.month) + r
return output
elif is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def quote_value(value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return SQL_NULL
elif isinstance(value, SQL):
return value
elif is_text(value):
return SQL("'" + "".join(ESCAPE_DCT.get(c, c)
for c in value) + "'")
elif is_data(value):
return quote_value(json_encode(value))
elif isinstance(value, datetime):
return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") +
"', '%Y%m%d%H%i%s.%f')")
elif isinstance(value, Date):
return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") +
"', '%Y%m%d%H%i%s.%f')")
elif is_number(value):
return SQL(text(value))
elif hasattr(value, '__iter__'):
return quote_value(json_encode(value))
else:
return quote_value(text(value))
except Exception as e:
Log.error("problem quoting SQL {{value}}", value=repr(value), cause=e)
def convert(self, expr):
"""
EXPAND INSTANCES OF name TO value
"""
if expr is True or expr == None or expr is False:
return expr
elif is_number(expr):
return expr
elif expr == ".":
return "."
elif is_variable_name(expr):
return coalesce(self.dimensions[expr], expr)
elif is_text(expr):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, Date):
return expr
elif is_op(expr, QueryOp):
return self._convert_query(expr)
elif is_data(expr):
if expr["from"]:
return self._convert_query(expr)
elif len(expr) >= 2:
#ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
return wrap({name: self.convert(value) for name, value in expr.leaves()})
else:
# ASSUME SINGLE-CLAUSE EXPRESSION
k, v = expr.items()[0]
return converter_map.get(k, self._convert_bop)(self, k, v)
elif is_many(expr):
return wrap([self.convert(value) for value in expr])
else:
return expr
def __new__(cls, value=None, **kwargs):
output = object.__new__(cls)
if value == None:
if kwargs:
output.milli = datetime.timedelta(**kwargs).total_seconds() * 1000
output.month = 0
return output
else:
return None
if is_number(value):
output._milli = float(value) * 1000
output.month = 0
return output
elif is_text(value):
return parse(value)
elif isinstance(value, Duration):
output.milli = value.milli
output.month = value.month
return output
elif isinstance(value, float) and is_nan(value):
return None
else:
from mo_logs import Log
Log.error("Do not know type of object (" + get_module("mo_json").value2json(value) + ")of to make a Duration")
def _normalize_select_no_context(select, schema=None):
"""
SAME NORMALIZE, BUT NO SOURCE OF COLUMNS
"""
if not _Column:
_late_import()
if is_text(select):
select = Data(value=select)
else:
select = wrap(select)
output = select.copy()
if not select.value:
output.name = coalesce(select.name, select.aggregate)
if output.name:
output.value = jx_expression(".", schema=schema)
elif len(select):
Log.error(BAD_SELECT, select=select)
else:
return Null
elif is_text(select.value):
if select.value.endswith(".*"):
name = select.value[:-2].lstrip(".")
output.name = coalesce(select.name, name)
output.value = LeavesOp(Variable(name),
prefix=coalesce(select.prefix, name))
else:
if select.value == ".":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = jx_expression(select.value, schema=schema)
elif select.value == "*":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = LeavesOp(Variable("."))
else:
output.name = coalesce(select.name, select.value.lstrip("."),
select.aggregate)
output.value = jx_expression(select.value, schema=schema)
elif is_number(output.value):
if not output.name:
output.name = text(output.value)
output.value = jx_expression(select.value, schema=schema)
else:
output.value = jx_expression(select.value, schema=schema)
if not output.name:
Log.error("expecting select to have a name: {{select}}", select=select)
if output.name.endswith(".*"):
Log.error("{{name|quote}} is invalid select", name=output.name)
output.aggregate = coalesce(canonical_aggregates[select.aggregate].name,
select.aggregate, "none")
output.default = coalesce(select.default,
canonical_aggregates[output.aggregate].default)
return output
def to_bq(self, schema, not_null=False, boolean=False):
value = self.value
if value == None:
return wrap([{"name": "."}])
elif isinstance(value, text):
return wrap([{"name": ".", "sql": {"s": quote_value(value)}}])
elif is_number(value):
return wrap([{"name": ".", "sql": {"n": quote_value(value)}}])
elif value in [True, False]:
return wrap([{"name": ".", "sql": {"b": quote_value(value)}}])
else:
return wrap([{"name": ".", "sql": {"j": quote_value(self.json)}}])
def scrub_args(args):
output = {}
for k, v in list(args.items()):
vs = []
for v in listwrap(v):
if is_integer(v):
vs.append(int(v))
elif is_number(v):
vs.append(float(v))
else:
vs.append(v)
output[k] = unwraplist(vs)
return wrap(output)
def to_sql(self, schema, not_null=False, boolean=False):
value = self.value
v = quote_value(value)
if v == None:
return wrap([{"name": "."}])
elif is_text(value):
return wrap([{"name": ".", "sql": {"s": quote_value(value)}}])
elif is_number(v):
return wrap([{"name": ".", "sql": {"n": quote_value(value)}}])
elif v in [True, False]:
return wrap([{"name": ".", "sql": {"b": quote_value(value)}}])
else:
return wrap([{"name": ".", "sql": {"j": quote_value(self.json)}}])
def _normalize_select_no_context(select, schema=None):
"""
SAME NORMALIZE, BUT NO SOURCE OF COLUMNS
"""
if is_text(select):
select = Data(value=select)
else:
select = to_data(select)
output = select.copy()
if not select.value:
output.name = coalesce(select.name, select.aggregate)
if output.name:
output.value = jx_expression(".", schema=schema)
elif len(select):
Log.error(BAD_SELECT, select=select)
else:
return Null
elif is_text(select.value):
if select.value.endswith("*"):
path = split_field(select.value)
var = join_field(path[:-1])
name = var.strip(".")
if not name:
name = "."
output.name = coalesce(select.name, select.aggregate, name)
output.value = LeavesOp(Variable(var), prefix=select.prefix)
elif select.value == ".":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = jx_expression(select.value, schema=schema)
else:
output.name = coalesce(select.name, select.value.lstrip("."),
select.aggregate)
output.value = jx_expression(select.value, schema=schema)
elif is_number(output.value):
if not output.name:
output.name = text(output.value)
output.value = jx_expression(select.value, schema=schema)
else:
output.value = jx_expression(select.value, schema=schema)
if not output.name:
Log.error("expecting select to have a name: {{select}}", select=select)
if output.name.endswith(".*"):
Log.error("{{name|quote}} is invalid select", name=output.name)
output.aggregate = coalesce(canonical_aggregates[select.aggregate].name,
select.aggregate, "none")
output.default = coalesce(select.default,
canonical_aggregates[output.aggregate].default)
return output
def _jx_expression(expr, lang):
"""
WRAP A JSON EXPRESSION WITH OBJECT REPRESENTATION
"""
if is_expression(expr):
# CONVERT TO lang
new_op = lang[expr]
if not new_op:
# CAN NOT BE FOUND, TRY SOME PARTIAL EVAL
return language[expr.get_id()].partial_eval()
return expr
# return new_op(expr.args) # THIS CAN BE DONE, BUT IT NEEDS MORE CODING, AND I WOULD EXPECT IT TO BE SLOW
if expr is None:
return TRUE
elif is_text(expr):
return Variable(expr)
elif expr in (True, False, None) or expr == None or is_number(expr):
return Literal(expr)
elif expr.__class__ is Date:
return Literal(expr.unix)
elif is_sequence(expr):
return lang[TupleOp([_jx_expression(e, lang) for e in expr])]
# expr = to_data(expr)
try:
items = items_(expr)
for op, term in items:
# ONE OF THESE IS THE OPERATOR
full_op = operators.get(op)
if full_op:
class_ = lang.ops[full_op.get_id()]
if class_:
return class_.define(expr)
# THIS LANGUAGE DOES NOT SUPPORT THIS OPERATOR, GOTO BASE LANGUAGE AND GET THE MACRO
class_ = language[op.get_id()]
output = class_.define(expr).partial_eval()
return _jx_expression(output, lang)
else:
if not items:
return NULL
raise Log.error("{{instruction|json}} is not known",
instruction=expr)
except Exception as e:
Log.error("programmer error expr = {{value|quote}}",
value=expr,
cause=e)
def _normalize_select_no_context(select, schema=None):
"""
SAME NORMALIZE, BUT NO SOURCE OF COLUMNS
"""
if not _Column:
_late_import()
if is_text(select):
select = Data(value=select)
else:
select = wrap(select)
output = select.copy()
if not select.value:
output.name = coalesce(select.name, select.aggregate)
if output.name:
output.value = jx_expression(".", schema=schema)
else:
return Null
elif is_text(select.value):
if select.value.endswith(".*"):
name = select.value[:-2].lstrip(".")
output.name = coalesce(select.name, name)
output.value = LeavesOp(Variable(name), prefix=coalesce(select.prefix, name))
else:
if select.value == ".":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = jx_expression(select.value, schema=schema)
elif select.value == "*":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = LeavesOp(Variable("."))
else:
output.name = coalesce(select.name, select.value.lstrip("."), select.aggregate)
output.value = jx_expression(select.value, schema=schema)
elif is_number(output.value):
if not output.name:
output.name = text_type(output.value)
output.value = jx_expression(select.value, schema=schema)
else:
output.value = jx_expression(select.value, schema=schema)
if not output.name:
Log.error("expecting select to have a name: {{select}}", select= select)
if output.name.endswith(".*"):
Log.error("{{name|quote}} is invalid select", name=output.name)
output.aggregate = coalesce(canonical_aggregates[select.aggregate].name, select.aggregate, "none")
output.default = coalesce(select.default, canonical_aggregates[output.aggregate].default)
return output
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)
) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + text(rem) + "milli" + output
rest = floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + text(rem) + "second" + output
rest = floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + text(rem) + "minute" + output
rest = floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + text(rem) + "hour" + output
rest = floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = floor(rest / 7)
if rem != 0:
output = "+" + text(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + text(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = abs(self.month)
if month <= 18 and month != 12:
output = sign + text(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + text(m) + "month" + output
y = floor(month / 12)
output = sign + text(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not is_number(output[1]):
output = output[1::]
return output
def test_isnumber(self):
assert mo_math.is_number(9999999999000)
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + text_type(rem) + "milli" + output
rest = floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "second" + output
rest = floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "minute" + output
rest = floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + text_type(rem) + "hour" + output
rest = floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = floor(rest / 7)
if rem != 0:
output = "+" + text_type(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + text_type(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = abs(self.month)
if month <= 18 and month != 12:
output = sign + text_type(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + text_type(m) + "month" + output
y = floor(month / 12)
output = sign + text_type(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not is_number(output[1]):
output = output[1::]
return output