def quote_value(self, value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return "NULL"
elif isinstance(value, SQL):
if not value.param:
# value.template CAN BE MORE THAN A TEMPLATE STRING
return self.quote_sql(value.template)
param = {k: self.quote_sql(v) for k, v in value.param.items()}
return expand_template(value.template, param)
elif isinstance(value, basestring):
return self.db.literal(value)
elif isinstance(value, datetime):
return "str_to_date('" + value.strftime("%Y%m%d%H%M%S") + "', '%Y%m%d%H%i%s')"
elif hasattr(value, '__iter__'):
return self.db.literal(json_encode(value))
elif isinstance(value, Mapping):
return self.db.literal(json_encode(value))
elif Math.is_number(value):
return unicode(value)
else:
return self.db.literal(value)
except Exception, e:
Log.error("problem quoting SQL", e)
def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = Math.floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from pyLibrary.debugs.logs import Log
Log.error("Do not know how to handle")
r = MIN(29 / 30, (r + tod) / (MILLI_VALUES.day * 30))
output = Math.floor(m / amount.month) + r
return output
elif Math.is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def __new__(cls, value=None, **kwargs):
output = object.__new__(cls)
if value == None:
if kwargs:
output.milli = datetime.timedelta(**kwargs).total_seconds() * 1000
output.month = 0
return output
else:
return None
if Math.is_number(value):
output._milli = float(value) * 1000
output.month = 0
return output
elif isinstance(value, basestring):
return parse(value)
elif isinstance(value, Duration):
output.milli = value.milli
output.month = value.month
return output
elif isinstance(value, float) and Math.is_nan(value):
return None
else:
from pyLibrary import convert
from pyLibrary.debugs.logs import Log
Log.error("Do not know type of object (" + convert.value2json(value) + ")of to make a Duration")
def __new__(cls, value=None, **kwargs):
output = object.__new__(cls)
if value == None:
if kwargs:
output.milli = datetime.timedelta(
**kwargs).total_seconds() * 1000
output.month = 0
return output
else:
return None
if Math.is_number(value):
output._milli = float(value) * 1000
output.month = 0
return output
elif isinstance(value, basestring):
return parse(value)
elif isinstance(value, Duration):
output.milli = value.milli
output.month = value.month
return output
elif isinstance(value, float) and Math.is_nan(value):
return None
else:
from pyLibrary import convert
from pyLibrary.debugs.logs import Log
Log.error("Do not know type of object (" +
convert.value2json(value) + ")of to make a Duration")
def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = Math.floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from pyLibrary.debugs.logs import Log
Log.error("Do not know how to handle")
r = MIN(29 / 30, (r + tod) / (MILLI_VALUES.day * 30))
output = Math.floor(m / amount.month) + r
return output
elif Math.is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def convert(self, expr):
"""
ADD THE ".$value" SUFFIX TO ALL VARIABLES
"""
if expr is True or expr == None or expr is False:
return expr
elif Math.is_number(expr):
return expr
elif expr == ".":
return "."
elif is_keyword(expr):
#TODO: LOOKUP SCHEMA AND ADD ALL COLUMNS WITH THIS PREFIX
return expr + ".$value"
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, Date):
return expr
elif isinstance(expr, Query):
return self._convert_query(expr)
elif isinstance(expr, Mapping):
if expr["from"]:
return self._convert_query(expr)
elif len(expr) >= 2:
#ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
return wrap({name: self.convert(value) for name, value in expr.items()})
else:
# ASSUME SINGLE-CLAUSE EXPRESSION
k, v = expr.items()[0]
return self.converter_map.get(k, self._convert_bop)(k, v)
elif isinstance(expr, (list, set, tuple)):
return wrap([self.convert(value) for value in expr])
def convert(self, expr):
"""
EXPAND INSTANCES OF name TO value
"""
if expr is True or expr == None or expr is False:
return expr
elif Math.is_number(expr):
return expr
elif expr == ".":
return "."
elif is_keyword(expr):
return coalesce(self.dimensions[expr], expr)
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, Date):
return expr
elif isinstance(expr, Query):
return self._convert_query(expr)
elif isinstance(expr, Mapping):
if expr["from"]:
return self._convert_query(expr)
elif len(expr) >= 2:
#ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
return wrap({name: self.convert(value) for name, value in expr.leaves()})
else:
# ASSUME SINGLE-CLAUSE EXPRESSION
k, v = expr.items()[0]
return converter_map.get(k, self._convert_bop)(self, k, v)
elif isinstance(expr, (list, set, tuple)):
return wrap([self.convert(value) for value in expr])
else:
return expr
def convert(self, expr):
"""
EXPAND INSTANCES OF name TO value
"""
if expr is True or expr == None or expr is False:
return expr
elif Math.is_number(expr):
return expr
elif expr == ".":
return "."
elif is_keyword(expr):
return coalesce(self.dimensions[expr], expr)
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, Date):
return expr
elif isinstance(expr, QueryOp):
return self._convert_query(expr)
elif isinstance(expr, Mapping):
if expr["from"]:
return self._convert_query(expr)
elif len(expr) >= 2:
#ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
return wrap({
name: self.convert(value)
for name, value in expr.leaves()
})
else:
# ASSUME SINGLE-CLAUSE EXPRESSION
k, v = expr.items()[0]
return converter_map.get(k, self._convert_bop)(self, k, v)
elif isinstance(expr, (list, set, tuple)):
return wrap([self.convert(value) for value in expr])
else:
return expr
def quote_value(self, value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return "NULL"
elif isinstance(value, SQL):
if not value.param:
# value.template CAN BE MORE THAN A TEMPLATE STRING
return self.quote_sql(value.template)
param = {k: self.quote_sql(v) for k, v in value.param.items()}
return expand_template(value.template, param)
elif isinstance(value, basestring):
return self.db.literal(value)
elif isinstance(value, datetime):
return "str_to_date('" + value.strftime(
"%Y%m%d%H%M%S") + "', '%Y%m%d%H%i%s')"
elif hasattr(value, '__iter__'):
return self.db.literal(json_encode(value))
elif isinstance(value, Mapping):
return self.db.literal(json_encode(value))
elif Math.is_number(value):
return unicode(value)
else:
return self.db.literal(value)
except Exception, e:
Log.error("problem quoting SQL", e)
def get_all_vars(expr):
if expr == None:
return set()
elif isinstance(expr, unicode):
if expr == "." or is_keyword(expr):
return set([expr])
else:
Log.error("Expecting a json path")
elif expr is True:
return set()
elif expr is False:
return set()
elif Math.is_number(expr):
return set()
op, term = expr.items()[0]
mop = ruby_multi_operators.get(op)
if mop:
if isinstance(term, list):
output = set()
for t in term:
output |= get_all_vars(t)
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
return get_all_vars(a) | get_all_vars(b)
else:
get_all_vars(term)
bop = ruby_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = set()
for t in term:
output |= get_all_vars(t)
return output
elif isinstance(term, Mapping):
if op == "eq":
output = set()
for a, b in term.items():
output |= get_all_vars(
a) # {k:v} k IS VARIABLE, v IS A VALUE
return output
else:
a, b = term.items()[0]
return get_all_vars(a)
else:
Log.error("Expecting binary term")
uop = ruby_unary_operators.get(op)
if uop:
return get_all_vars(term)
cop = complex_operators.get(op)
if cop:
return cop(op, term).vars()
Log.error("`{{op}}` is not a recognized operation", op=op)
def get_all_vars(expr):
if expr == None:
return set()
elif isinstance(expr, unicode):
if expr == "." or is_keyword(expr):
return set([expr])
else:
Log.error("Expecting a json path")
elif expr is True:
return set()
elif expr is False:
return set()
elif Math.is_number(expr):
return set()
op, term = expr.items()[0]
mop = ruby_multi_operators.get(op)
if mop:
if isinstance(term, list):
output = set()
for t in term:
output |= get_all_vars(t)
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
return get_all_vars(a) | get_all_vars(b)
else:
get_all_vars(term)
bop = ruby_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = set()
for t in term:
output |= get_all_vars(t)
return output
elif isinstance(term, Mapping):
if op == "eq":
output = set()
for a, b in term.items():
output |= get_all_vars(a) # {k:v} k IS VARIABLE, v IS A VALUE
return output
else:
a, b = term.items()[0]
return get_all_vars(a)
else:
Log.error("Expecting binary term")
uop = ruby_unary_operators.get(op)
if uop:
return get_all_vars(term)
cop = complex_operators.get(op)
if cop:
return cop(op, term).vars()
Log.error("`{{op}}` is not a recognized operation", op= op)
def value2query(value):
if isinstance(value, datetime):
return convert.datetime2milli(value)
if isinstance(value, Duration):
return value.milli
if Math.is_number(value):
return value
return convert.string2quote(value)
def value2query(value):
if isinstance(value, datetime):
return convert.datetime2milli(value)
if isinstance(value, Duration):
return value.milli
if Math.is_number(value):
return value
return convert.string2quote(value)
def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, delta=None):
"""
Snagged from unittest/case.py, then modified (Aug2014)
"""
if expected == None: # None has no expectations
return
if test == expected:
# shortcut
return
if not Math.is_number(expected):
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
if isinstance(expected, list) and len(expected)==0 and test == None:
return
if isinstance(expected, Mapping) and not expected.keys() and test == None:
return
if test != expected:
raise AssertionError(expand_template("{{test}} != {{expected}}", locals()))
return
num_param = 0
if digits != None:
num_param += 1
if places != None:
num_param += 1
if delta != None:
num_param += 1
if num_param>1:
raise TypeError("specify only one of digits, places or delta")
if digits is not None:
with suppress_exception:
diff = Math.log10(abs(test-expected))
if diff < digits:
return
standardMsg = expand_template("{{test}} != {{expected}} within {{digits}} decimal places", locals())
elif delta is not None:
if abs(test - expected) <= delta:
return
standardMsg = expand_template("{{test}} != {{expected}} within {{delta}} delta", locals())
else:
if places is None:
places = 15
with suppress_exception:
diff = Math.log10(abs(test-expected))
if diff < Math.ceiling(Math.log10(abs(test)))-places:
return
standardMsg = expand_template("{{test|json}} != {{expected|json}} within {{places}} places", locals())
raise AssertionError(coalesce(msg, "") + ": (" + standardMsg + ")")
def value2MVEL(value):
"""
FROM PYTHON VALUE TO MVEL EQUIVALENT
"""
if isinstance(value, datetime):
return str(convert.datetime2milli(value)) + " /*" + value.format("yyNNNdd HHmmss") + "*/" # TIME
if isinstance(value, Duration):
return str(convert.timedelta2milli(value)) + " /*" + str(value) + "*/" # DURATION
if Math.is_number(value):
return str(value)
return convert.string2quote(value)
def _partition(min_, max_):
try:
source_count = source.search({
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"range": {config.primary_field: {"gte": min_, "lt": max_}}}
}},
"size": 0
})
if num_mismatches[0] < 10:
# SOMETIMES THE TWO ARE TOO DIFFERENT TO BE OPTIMISTIC
dest_count = destination.search({
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"range": {config.primary_field: {"gte": min_, "lt": max_}}}
}},
"size": 0
})
if source_count.hits.total == dest_count.hits.total:
return
else:
num_mismatches[0] += 1
if source_count.hits.total < 200000:
_copy(min_, max_)
elif Math.is_number(min_) and Math.is_number(max_):
mid_ = int(round((float(min_) + float(max_)) / 2, 0))
# WORK BACKWARDS
_partition(mid_, max_)
_partition(min_, mid_)
else:
Log.error("can not split alphabetical in half")
except Exception, e:
Log.warning("Scanning had a problem", cause=e)
def value2MVEL(value):
"""
FROM PYTHON VALUE TO MVEL EQUIVALENT
"""
if isinstance(value, datetime):
return str(convert.datetime2milli(value)) + " /*" + value.format(
"yyNNNdd HHmmss") + "*/" # TIME
if isinstance(value, Duration):
return str(convert.timedelta2milli(value)) + " /*" + str(
value) + "*/" # DURATION
if Math.is_number(value):
return str(value)
return convert.string2quote(value)
def assertAlmostEqualValue(test,
expected,
digits=None,
places=None,
msg=None,
delta=None):
"""
Snagged from unittest/case.py, then modified (Aug2014)
"""
if test == expected:
# shortcut
return
if not Math.is_number(expected):
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
if isinstance(expected, list) and len(expected) == 0 and test == None:
return
if isinstance(expected,
Mapping) and not expected.keys() and test == None:
return
if test != expected:
raise AssertionError(
expand_template("{{test}} != {{expected}}", locals()))
return
num_param = 0
if digits != None:
num_param += 1
if places != None:
num_param += 1
if delta != None:
num_param += 1
if num_param > 1:
raise TypeError("specify only one of digits, places or delta")
if digits is not None:
try:
diff = Math.log10(abs(test - expected))
if diff < digits:
return
except Exception, e:
pass
standardMsg = expand_template(
"{{test}} != {{expected}} within {{digits}} decimal places",
locals())
def convert(self, expr):
"""
ADD THE ".$value" SUFFIX TO ALL VARIABLES
"""
if isinstance(expr, Expression):
vars_ = expr.vars()
rename = {
v: join_field(split_field(v) + ["$value"])
for v in vars_
}
return expr.map(rename)
if expr is True or expr == None or expr is False:
return expr
elif Math.is_number(expr):
return expr
elif expr == ".":
return "."
elif is_keyword(expr):
#TODO: LOOKUP SCHEMA AND ADD ALL COLUMNS WITH THIS PREFIX
return expr + ".$value"
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, Date):
return expr
elif isinstance(expr, QueryOp):
return self._convert_query(expr)
elif isinstance(expr, Mapping):
if expr["from"]:
return self._convert_query(expr)
elif len(expr) >= 2:
#ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
return wrap({
name: self.convert(value)
for name, value in expr.items()
})
else:
# ASSUME SINGLE-CLAUSE EXPRESSION
k, v = expr.items()[0]
return self.converter_map.get(k, self._convert_bop)(k, v)
elif isinstance(expr, (list, set, tuple)):
return wrap([self.convert(value) for value in expr])
def _scrub(r):
try:
if r == None:
return None
elif isinstance(r, basestring):
if r == "":
return None
return r
elif Math.is_number(r):
return convert.value2number(r)
elif isinstance(r, Mapping):
if isinstance(r, Dict):
r = object.__getattribute__(r, "_dict")
output = {}
for k, v in r.items():
v = _scrub(v)
if v != None:
output[k.lower()] = v
if len(output) == 0:
return None
return output
elif hasattr(r, '__iter__'):
if isinstance(r, DictList):
r = r.list
output = []
for v in r:
v = _scrub(v)
if v != None:
output.append(v)
if not output:
return None
if len(output) == 1:
return output[0]
try:
return sort(output)
except Exception:
return output
else:
return r
except Exception, e:
Log.warning("Can not scrub: {{json}}", json= r)
def _scrub(r):
try:
if r == None:
return None
elif isinstance(r, basestring):
if r == "":
return None
return r
elif Math.is_number(r):
return convert.value2number(r)
elif isinstance(r, Mapping):
if isinstance(r, Dict):
r = object.__getattribute__(r, "_dict")
output = {}
for k, v in r.items():
v = _scrub(v)
if v != None:
output[k.lower()] = v
if len(output) == 0:
return None
return output
elif hasattr(r, '__iter__'):
if isinstance(r, DictList):
r = r.list
output = []
for v in r:
v = _scrub(v)
if v != None:
output.append(v)
if not output:
return None
if len(output) == 1:
return output[0]
try:
return sort(output)
except Exception:
return output
else:
return r
except Exception, e:
Log.warning("Can not scrub: {{json}}", json= r)
def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, delta=None):
"""
Snagged from unittest/case.py, then modified (Aug2014)
"""
if test == expected:
# shortcut
return
if not Math.is_number(expected):
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
if isinstance(expected, list) and len(expected) == 0 and test == None:
return
if isinstance(expected, Mapping) and not expected.keys() and test == None:
return
if test != expected:
raise AssertionError(expand_template("{{test}} != {{expected}}", locals()))
return
num_param = 0
if digits != None:
num_param += 1
if places != None:
num_param += 1
if delta != None:
num_param += 1
if num_param > 1:
raise TypeError("specify only one of digits, places or delta")
if digits is not None:
try:
diff = Math.log10(abs(test - expected))
if diff < digits:
return
except Exception, e:
pass
standardMsg = expand_template("{{test}} != {{expected}} within {{digits}} decimal places", locals())
def stats(values):
"""
RETURN LOTS OF AGGREGATES
"""
if values == None:
return None
values = values.map(float, includeNone=False)
z = ZeroMoment.new_instance(values)
s = Dict()
for k, v in z.dict.items():
s[k] = v
for k, v in ZeroMoment2Stats(z).items():
s[k] = v
s.max = MAX(values)
s.min = MIN(values)
s.median = pyLibrary.maths.stats.median(values, simple=False)
s.last = values.last()
s.first = values[0]
if Math.is_number(s.variance) and not Math.is_nan(s.variance):
s.std = sqrt(s.variance)
return s
def stats(values):
"""
RETURN LOTS OF AGGREGATES
"""
if values == None:
return None
values = values.map(float, includeNone=False)
z = ZeroMoment.new_instance(values)
s = Struct()
for k, v in z.dict.items():
s[k] = v
for k, v in ZeroMoment2Stats(z).items():
s[k] = v
s.max = MAX(values)
s.min = MIN(values)
s.median = pyLibrary.stats.median(values, simple=False)
s.last = values.last()
s.first = values[0]
if Math.is_number(s.variance) and not Math.is_nan(s.variance):
s.std = sqrt(s.variance)
return s
def qb_expression_to_ruby(expr):
if expr == None:
return "nil"
elif Math.is_number(expr):
return unicode(expr)
elif is_keyword(expr):
return "doc[" + convert.string2quote(expr) + "].value"
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, CODE):
return expr.code
elif isinstance(expr, Date):
return unicode(expr.unix)
elif expr is True:
return "true"
elif expr is False:
return "false"
op, term = expr.items()[0]
mop = ruby_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(
["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(
a) + ")" + mop[0] + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
qb_expression_to_ruby(term)
bop = ruby_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(
["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_ruby(a) + ")" +
bop + "(" + qb_expression_to_ruby(b) +
")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(
a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = ruby_unary_operators.get(op)
if uop:
output = expand_template(uop, {"term": qb_expression_to_ruby(term)})
return output
cop = complex_operators.get(op)
if cop:
output = cop(term).to_ruby()
return output
Log.error("`{{op}}` is not a recognized operation", op=op)
def get_pending(source, since, pending_bugs, please_stop):
try:
while not please_stop:
if since == None:
Log.note("Get all records")
result = source.search({
# "query": {"match_all": {}},
"query": {
"filtered": {
"filter": {"exists": {"field": config.primary_field}},
"query": {"match_all": {}}
}},
"fields": ["_id", config.primary_field],
"from": 0,
"size": BATCH_SIZE,
"sort": [config.primary_field]
})
else:
Log.note(
"Get records with {{primary_field}} >= {{max_time|datetime}}",
primary_field=config.primary_field,
max_time=since
)
result = source.search({
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"range": {config.primary_field: {"gte": since}}},
}},
"fields": ["_id", config.primary_field],
"from": 0,
"size": BATCH_SIZE,
"sort": [config.primary_field]
})
new_max_value = MAX([unwraplist(h.fields[literal_field(config.primary_field)]) for h in result.hits.hits])
if since == new_max_value:
# GET ALL WITH THIS TIMESTAMP
result = source.search({
"query": {"filtered": {
"query": {"match_all": {}},
"filter": {"term": {config.primary_field: since}},
}},
"fields": ["_id", config.primary_field],
"from": 0,
"size": 100000
})
if Math.is_integer(new_max_value):
since = int(new_max_value) + 1
elif Math.is_number(new_max_value):
since = float(new_max_value) + 0.5
else:
since = unicode(new_max_value) + "a"
else:
since = new_max_value
ids = result.hits.hits._id
Log.note("Adding {{num}} to pending queue", num=len(ids))
pending_bugs.extend(ids)
if len(result.hits.hits) < BATCH_SIZE:
break
Log.note("No more ids")
except Exception, e:
please_stop.go()
Log.error("Problem while copying records", cause=e)
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)
) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = Math.abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + unicode(rem) + "milli" + output
rest = Math.floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + unicode(rem) + "second" + output
rest = Math.floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + unicode(rem) + "minute" + output
rest = Math.floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + unicode(rem) + "hour" + output
rest = Math.floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = Math.floor(rest / 7)
if rem != 0:
output = "+" + unicode(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + unicode(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = Math.abs(self.month)
if month <= 18 and month != 12:
output = sign + unicode(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + unicode(m) + "month" + output
y = Math.floor(month / 12)
output = sign + unicode(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not Math.is_number(output[1]):
output = output[1::]
return output
all = set()
with open(settings.output_file, "r") as input_file:
with open("good_talos.tab", "w") as output_file:
for line in input_file:
try:
if len(line.strip()) == 0:
continue
col = line.split("\t")
id = int(col[0])
if id < MINIMUM_ID:
continue
json = col[1]
if Math.is_number(json):
json = col[2]
data = CNV.JSON2object(json).json_blob
date = CNV.unix2datetime(data.testrun.date)
if id % 1000 == 0:
Log.println("loading id " + str(id) + " date: " + CNV.datetime2string(date, "%Y-%m-%d %H:%M:%S"))
if date < MINIMUM_DATE:
continue
if id in all:
continue
all.add(id)
arrays_add(id, "[" + data.test_build.branch + "][" + data.testrun.suite + "]", data)
def test_isnumber(self):
assert Math.is_number(9999999999000)
def assertAlmostEqualValue(test,
expected,
digits=None,
places=None,
msg=None,
delta=None):
"""
Snagged from unittest/case.py, then modified (Aug2014)
"""
if expected == None: # None has no expectations
return
if test == expected:
# shortcut
return
if not Math.is_number(expected):
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
if isinstance(expected, list) and len(expected) == 0 and test == None:
return
if isinstance(expected,
Mapping) and not expected.keys() and test == None:
return
if test != expected:
raise AssertionError(
expand_template("{{test}} != {{expected}}", locals()))
return
num_param = 0
if digits != None:
num_param += 1
if places != None:
num_param += 1
if delta != None:
num_param += 1
if num_param > 1:
raise TypeError("specify only one of digits, places or delta")
if digits is not None:
with suppress_exception:
diff = Math.log10(abs(test - expected))
if diff < digits:
return
standardMsg = expand_template(
"{{test}} != {{expected}} within {{digits}} decimal places",
locals())
elif delta is not None:
if abs(test - expected) <= delta:
return
standardMsg = expand_template(
"{{test}} != {{expected}} within {{delta}} delta", locals())
else:
if places is None:
places = 15
with suppress_exception:
diff = Math.log10(abs(test - expected))
if diff < Math.ceiling(Math.log10(abs(test))) - places:
return
standardMsg = expand_template(
"{{test|json}} != {{expected|json}} within {{places}} places",
locals())
raise AssertionError(coalesce(msg, "") + ": (" + standardMsg + ")")
def qb_expression_to_python(expr):
if expr == None:
return "None"
elif Math.is_number(expr):
return unicode(expr)
elif isinstance(expr, Date):
return unicode(expr.unix)
elif isinstance(expr, unicode):
if expr == ".":
return "row"
elif is_keyword(expr):
return "row[" + convert.value2quote(expr) + "]"
else:
Log.error("Expecting a json path")
elif isinstance(expr, CODE):
return expr.code
elif expr is True:
return "True"
elif expr is False:
return "False"
op, term = expr.items()[0]
mop = python_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_python(a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")"
return output
else:
qb_expression_to_python(term)
bop = python_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = python_unary_operators.get(op)
if uop:
output = uop + "(" + qb_expression_to_python(term) + ")"
return output
Log.error("`{{op}}` is not a recognized operation", op= op)
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = Math.abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + unicode(rem) + "milli" + output
rest = Math.floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + unicode(rem) + "second" + output
rest = Math.floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + unicode(rem) + "minute" + output
rest = Math.floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + unicode(rem) + "hour" + output
rest = Math.floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = Math.floor(rest / 7)
if rem != 0:
output = "+" + unicode(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + unicode(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = Math.abs(self.month)
if month <= 18 and month != 12:
output = sign + unicode(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + unicode(m) + "month" + output
y = Math.floor(month / 12)
output = sign + unicode(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not Math.is_number(output[1]):
output = output[1::]
return output
def qb_expression_to_ruby(expr):
if expr == None:
return "nil"
elif Math.is_number(expr):
return unicode(expr)
elif is_keyword(expr):
return "doc[" + convert.string2quote(expr) + "].value"
elif isinstance(expr, basestring):
Log.error("{{name|quote}} is not a valid variable name", name=expr)
elif isinstance(expr, CODE):
return expr.code
elif isinstance(expr, Date):
return unicode(expr.unix)
elif expr is True:
return "true"
elif expr is False:
return "false"
op, term = expr.items()[0]
mop = ruby_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(a) + ")" + mop[0] + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
qb_expression_to_ruby(term)
bop = ruby_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(["(" + qb_expression_to_ruby(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_ruby(a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_ruby(a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = ruby_unary_operators.get(op)
if uop:
output = expand_template(uop, {"term": qb_expression_to_ruby(term)})
return output
cop = complex_operators.get(op)
if cop:
output = cop(term).to_ruby()
return output
Log.error("`{{op}}` is not a recognized operation", op= op)
def qb_expression_to_python(expr):
if expr == None:
return "None"
elif Math.is_number(expr):
return unicode(expr)
elif isinstance(expr, Date):
return unicode(expr.unix)
elif isinstance(expr, unicode):
if expr == ".":
return "row"
elif is_keyword(expr):
return "row[" + convert.value2quote(expr) + "]"
else:
Log.error("Expecting a json path")
elif isinstance(expr, CODE):
return expr.code
elif expr is True:
return "True"
elif expr is False:
return "False"
op, term = expr.items()[0]
mop = python_multi_operators.get(op)
if mop:
if isinstance(term, list):
if not term:
return mop[1] # RETURN DEFAULT
else:
output = mop[0].join(
["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
a, b = term.items()[0]
output = "(" + qb_expression_to_python(
a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")"
return output
else:
qb_expression_to_python(term)
bop = python_binary_operators.get(op)
if bop:
if isinstance(term, list):
output = bop.join(
["(" + qb_expression_to_python(t) + ")" for t in term])
return output
elif isinstance(term, Mapping):
if op == "eq":
# eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
output = " and ".join("(" + qb_expression_to_python(a) + ")" +
bop + "(" + qb_expression_to_python(b) +
")" for a, b in term.items())
return output
else:
a, b = term.items()[0]
output = "(" + qb_expression_to_python(
a) + ")" + bop + "(" + qb_expression_to_python(b) + ")"
return output
else:
Log.error("Expecting binary term")
uop = python_unary_operators.get(op)
if uop:
output = uop + "(" + qb_expression_to_python(term) + ")"
return output
Log.error("`{{op}}` is not a recognized operation", op=op)
