Beispiel #1
0
 def quote_value(self, value):
     """
     convert values to mysql code for the same
     mostly delegate directly to the mysql lib, but some exceptions exist
     """
     try:
         if value == None:
             return "NULL"
         elif isinstance(value, SQL):
             if not value.param:
                 # value.template CAN BE MORE THAN A TEMPLATE STRING
                 return self.quote_sql(value.template)
             param = {k: self.quote_sql(v) for k, v in value.param.items()}
             return expand_template(value.template, param)
         elif isinstance(value, basestring):
             return self.db.literal(value)
         elif isinstance(value, datetime):
             return "str_to_date('" + value.strftime("%Y%m%d%H%M%S") + "', '%Y%m%d%H%i%s')"
         elif hasattr(value, '__iter__'):
             return self.db.literal(json_encode(value))
         elif isinstance(value, Mapping):
             return self.db.literal(json_encode(value))
         elif Math.is_number(value):
             return unicode(value)
         else:
             return self.db.literal(value)
     except Exception, e:
         Log.error("problem quoting SQL", e)
    def __div__(self, amount):
        if isinstance(amount, Duration) and amount.month:
            m = self.month
            r = self.milli

            # DO NOT CONSIDER TIME OF DAY
            tod = r % MILLI_VALUES.day
            r = r - tod

            if m == 0 and r > (MILLI_VALUES.year / 3):
                m = Math.floor(12 * self.milli / MILLI_VALUES.year)
                r -= (m / 12) * MILLI_VALUES.year
            else:
                r = r - (self.month * MILLI_VALUES.month)
                if r >= MILLI_VALUES.day * 31:
                    from pyLibrary.debugs.logs import Log
                    Log.error("Do not know how to handle")
            r = MIN(29 / 30, (r + tod) / (MILLI_VALUES.day * 30))

            output = Math.floor(m / amount.month) + r
            return output
        elif Math.is_number(amount):
            output = Duration(0)
            output.milli = self.milli / amount
            output.month = self.month / amount
            return output
        else:
            return self.milli / amount.milli
    def __new__(cls, value=None, **kwargs):
        output = object.__new__(cls)
        if value == None:
            if kwargs:
                output.milli = datetime.timedelta(**kwargs).total_seconds() * 1000
                output.month = 0
                return output
            else:
                return None

        if Math.is_number(value):
            output._milli = float(value) * 1000
            output.month = 0
            return output
        elif isinstance(value, basestring):
            return parse(value)
        elif isinstance(value, Duration):
            output.milli = value.milli
            output.month = value.month
            return output
        elif isinstance(value, float) and Math.is_nan(value):
            return None
        else:
            from pyLibrary import convert
            from pyLibrary.debugs.logs import Log
            Log.error("Do not know type of object (" + convert.value2json(value) + ")of to make a Duration")
Beispiel #4
0
    def __new__(cls, value=None, **kwargs):
        output = object.__new__(cls)
        if value == None:
            if kwargs:
                output.milli = datetime.timedelta(
                    **kwargs).total_seconds() * 1000
                output.month = 0
                return output
            else:
                return None

        if Math.is_number(value):
            output._milli = float(value) * 1000
            output.month = 0
            return output
        elif isinstance(value, basestring):
            return parse(value)
        elif isinstance(value, Duration):
            output.milli = value.milli
            output.month = value.month
            return output
        elif isinstance(value, float) and Math.is_nan(value):
            return None
        else:
            from pyLibrary import convert
            from pyLibrary.debugs.logs import Log
            Log.error("Do not know type of object (" +
                      convert.value2json(value) + ")of to make a Duration")
Beispiel #5
0
    def __div__(self, amount):
        if isinstance(amount, Duration) and amount.month:
            m = self.month
            r = self.milli

            # DO NOT CONSIDER TIME OF DAY
            tod = r % MILLI_VALUES.day
            r = r - tod

            if m == 0 and r > (MILLI_VALUES.year / 3):
                m = Math.floor(12 * self.milli / MILLI_VALUES.year)
                r -= (m / 12) * MILLI_VALUES.year
            else:
                r = r - (self.month * MILLI_VALUES.month)
                if r >= MILLI_VALUES.day * 31:
                    from pyLibrary.debugs.logs import Log
                    Log.error("Do not know how to handle")
            r = MIN(29 / 30, (r + tod) / (MILLI_VALUES.day * 30))

            output = Math.floor(m / amount.month) + r
            return output
        elif Math.is_number(amount):
            output = Duration(0)
            output.milli = self.milli / amount
            output.month = self.month / amount
            return output
        else:
            return self.milli / amount.milli
Beispiel #6
0
 def convert(self, expr):
     """
     ADD THE ".$value" SUFFIX TO ALL VARIABLES
     """
     if expr is True or expr == None or expr is False:
         return expr
     elif Math.is_number(expr):
         return expr
     elif expr == ".":
         return "."
     elif is_keyword(expr):
         #TODO: LOOKUP SCHEMA AND ADD ALL COLUMNS WITH THIS PREFIX
         return expr + ".$value"
     elif isinstance(expr, basestring):
         Log.error("{{name|quote}} is not a valid variable name", name=expr)
     elif isinstance(expr, Date):
         return expr
     elif isinstance(expr, Query):
         return self._convert_query(expr)
     elif isinstance(expr, Mapping):
         if expr["from"]:
             return self._convert_query(expr)
         elif len(expr) >= 2:
             #ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
             return wrap({name: self.convert(value) for name, value in expr.items()})
         else:
             # ASSUME SINGLE-CLAUSE EXPRESSION
             k, v = expr.items()[0]
             return self.converter_map.get(k, self._convert_bop)(k, v)
     elif isinstance(expr, (list, set, tuple)):
         return wrap([self.convert(value) for value in expr])
Beispiel #7
0
 def convert(self, expr):
     """
     EXPAND INSTANCES OF name TO value
     """
     if expr is True or expr == None or expr is False:
         return expr
     elif Math.is_number(expr):
         return expr
     elif expr == ".":
         return "."
     elif is_keyword(expr):
         return coalesce(self.dimensions[expr], expr)
     elif isinstance(expr, basestring):
         Log.error("{{name|quote}} is not a valid variable name", name=expr)
     elif isinstance(expr, Date):
         return expr
     elif isinstance(expr, Query):
         return self._convert_query(expr)
     elif isinstance(expr, Mapping):
         if expr["from"]:
             return self._convert_query(expr)
         elif len(expr) >= 2:
             #ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
             return wrap({name: self.convert(value) for name, value in expr.leaves()})
         else:
             # ASSUME SINGLE-CLAUSE EXPRESSION
             k, v = expr.items()[0]
             return converter_map.get(k, self._convert_bop)(self, k, v)
     elif isinstance(expr, (list, set, tuple)):
         return wrap([self.convert(value) for value in expr])
     else:
         return expr
Beispiel #8
0
 def convert(self, expr):
     """
     EXPAND INSTANCES OF name TO value
     """
     if expr is True or expr == None or expr is False:
         return expr
     elif Math.is_number(expr):
         return expr
     elif expr == ".":
         return "."
     elif is_keyword(expr):
         return coalesce(self.dimensions[expr], expr)
     elif isinstance(expr, basestring):
         Log.error("{{name|quote}} is not a valid variable name", name=expr)
     elif isinstance(expr, Date):
         return expr
     elif isinstance(expr, QueryOp):
         return self._convert_query(expr)
     elif isinstance(expr, Mapping):
         if expr["from"]:
             return self._convert_query(expr)
         elif len(expr) >= 2:
             #ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
             return wrap({
                 name: self.convert(value)
                 for name, value in expr.leaves()
             })
         else:
             # ASSUME SINGLE-CLAUSE EXPRESSION
             k, v = expr.items()[0]
             return converter_map.get(k, self._convert_bop)(self, k, v)
     elif isinstance(expr, (list, set, tuple)):
         return wrap([self.convert(value) for value in expr])
     else:
         return expr
Beispiel #9
0
 def quote_value(self, value):
     """
     convert values to mysql code for the same
     mostly delegate directly to the mysql lib, but some exceptions exist
     """
     try:
         if value == None:
             return "NULL"
         elif isinstance(value, SQL):
             if not value.param:
                 # value.template CAN BE MORE THAN A TEMPLATE STRING
                 return self.quote_sql(value.template)
             param = {k: self.quote_sql(v) for k, v in value.param.items()}
             return expand_template(value.template, param)
         elif isinstance(value, basestring):
             return self.db.literal(value)
         elif isinstance(value, datetime):
             return "str_to_date('" + value.strftime(
                 "%Y%m%d%H%M%S") + "', '%Y%m%d%H%i%s')"
         elif hasattr(value, '__iter__'):
             return self.db.literal(json_encode(value))
         elif isinstance(value, Mapping):
             return self.db.literal(json_encode(value))
         elif Math.is_number(value):
             return unicode(value)
         else:
             return self.db.literal(value)
     except Exception, e:
         Log.error("problem quoting SQL", e)
Beispiel #10
0
def get_all_vars(expr):
    if expr == None:
        return set()
    elif isinstance(expr, unicode):
        if expr == "." or is_keyword(expr):
            return set([expr])
        else:
            Log.error("Expecting a json path")
    elif expr is True:
        return set()
    elif expr is False:
        return set()
    elif Math.is_number(expr):
        return set()

    op, term = expr.items()[0]

    mop = ruby_multi_operators.get(op)
    if mop:
        if isinstance(term, list):
            output = set()
            for t in term:
                output |= get_all_vars(t)
            return output
        elif isinstance(term, Mapping):
            a, b = term.items()[0]
            return get_all_vars(a) | get_all_vars(b)
        else:
            get_all_vars(term)

    bop = ruby_binary_operators.get(op)
    if bop:
        if isinstance(term, list):
            output = set()
            for t in term:
                output |= get_all_vars(t)
            return output
        elif isinstance(term, Mapping):
            if op == "eq":
                output = set()
                for a, b in term.items():
                    output |= get_all_vars(
                        a)  # {k:v} k IS VARIABLE, v IS A VALUE
                return output
            else:
                a, b = term.items()[0]
                return get_all_vars(a)
        else:
            Log.error("Expecting binary term")

    uop = ruby_unary_operators.get(op)
    if uop:
        return get_all_vars(term)

    cop = complex_operators.get(op)
    if cop:
        return cop(op, term).vars()

    Log.error("`{{op}}` is not a recognized operation", op=op)
def get_all_vars(expr):
    if expr == None:
        return set()
    elif isinstance(expr, unicode):
        if expr == "." or is_keyword(expr):
            return set([expr])
        else:
            Log.error("Expecting a json path")
    elif expr is True:
        return set()
    elif expr is False:
        return set()
    elif Math.is_number(expr):
        return set()

    op, term = expr.items()[0]

    mop = ruby_multi_operators.get(op)
    if mop:
        if isinstance(term, list):
            output = set()
            for t in term:
                output |= get_all_vars(t)
            return output
        elif isinstance(term, Mapping):
            a, b = term.items()[0]
            return get_all_vars(a) | get_all_vars(b)
        else:
            get_all_vars(term)

    bop = ruby_binary_operators.get(op)
    if bop:
        if isinstance(term, list):
            output = set()
            for t in term:
                output |= get_all_vars(t)
            return output
        elif isinstance(term, Mapping):
            if op == "eq":
                output = set()
                for a, b in term.items():
                    output |= get_all_vars(a)  # {k:v} k IS VARIABLE, v IS A VALUE
                return output
            else:
                a, b = term.items()[0]
                return get_all_vars(a)
        else:
            Log.error("Expecting binary term")

    uop = ruby_unary_operators.get(op)
    if uop:
        return get_all_vars(term)

    cop = complex_operators.get(op)
    if cop:
        return cop(op, term).vars()

    Log.error("`{{op}}` is not a recognized operation",  op= op)
Beispiel #12
0
def value2query(value):
    if isinstance(value, datetime):
        return convert.datetime2milli(value)
    if isinstance(value, Duration):
        return value.milli

    if Math.is_number(value):
        return value
    return convert.string2quote(value)
Beispiel #13
0
def value2query(value):
    if isinstance(value, datetime):
        return convert.datetime2milli(value)
    if isinstance(value, Duration):
        return value.milli

    if Math.is_number(value):
        return value
    return convert.string2quote(value)
def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, delta=None):
    """
    Snagged from unittest/case.py, then modified (Aug2014)
    """
    if expected == None:  # None has no expectations
        return
    if test == expected:
        # shortcut
        return

    if not Math.is_number(expected):
        # SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
        if isinstance(expected, list) and len(expected)==0 and test == None:
            return
        if isinstance(expected, Mapping) and not expected.keys() and test == None:
            return
        if test != expected:
            raise AssertionError(expand_template("{{test}} != {{expected}}", locals()))
        return

    num_param = 0
    if digits != None:
        num_param += 1
    if places != None:
        num_param += 1
    if delta != None:
        num_param += 1
    if num_param>1:
        raise TypeError("specify only one of digits, places or delta")

    if digits is not None:
        with suppress_exception:
            diff = Math.log10(abs(test-expected))
            if diff < digits:
                return

        standardMsg = expand_template("{{test}} != {{expected}} within {{digits}} decimal places", locals())
    elif delta is not None:
        if abs(test - expected) <= delta:
            return

        standardMsg = expand_template("{{test}} != {{expected}} within {{delta}} delta", locals())
    else:
        if places is None:
            places = 15

        with suppress_exception:
            diff = Math.log10(abs(test-expected))
            if diff < Math.ceiling(Math.log10(abs(test)))-places:
                return


        standardMsg = expand_template("{{test|json}} != {{expected|json}} within {{places}} places", locals())

    raise AssertionError(coalesce(msg, "") + ": (" + standardMsg + ")")
Beispiel #15
0
def value2MVEL(value):
    """
    FROM PYTHON VALUE TO MVEL EQUIVALENT
    """
    if isinstance(value, datetime):
        return str(convert.datetime2milli(value)) + " /*" + value.format("yyNNNdd HHmmss") + "*/"        # TIME
    if isinstance(value, Duration):
        return str(convert.timedelta2milli(value)) + " /*" + str(value) + "*/"    # DURATION

    if Math.is_number(value):
        return str(value)
    return convert.string2quote(value)
Beispiel #16
0
    def _partition(min_, max_):
        try:
            source_count = source.search({
                "query": {"filtered": {
                    "query": {"match_all": {}},
                    "filter": {"range": {config.primary_field: {"gte": min_, "lt": max_}}}
                }},
                "size": 0
            })

            if num_mismatches[0] < 10:
                # SOMETIMES THE TWO ARE TOO DIFFERENT TO BE OPTIMISTIC
                dest_count = destination.search({
                    "query": {"filtered": {
                        "query": {"match_all": {}},
                        "filter": {"range": {config.primary_field: {"gte": min_, "lt": max_}}}
                    }},
                    "size": 0
                })

                if source_count.hits.total == dest_count.hits.total:
                    return
                else:
                    num_mismatches[0] += 1

            if source_count.hits.total < 200000:
                _copy(min_, max_)
            elif Math.is_number(min_) and Math.is_number(max_):
                mid_ = int(round((float(min_) + float(max_)) / 2, 0))
                # WORK BACKWARDS
                _partition(mid_, max_)
                _partition(min_, mid_)
            else:
                Log.error("can not split alphabetical in half")


        except Exception, e:
            Log.warning("Scanning had a problem", cause=e)
Beispiel #17
0
def value2MVEL(value):
    """
    FROM PYTHON VALUE TO MVEL EQUIVALENT
    """
    if isinstance(value, datetime):
        return str(convert.datetime2milli(value)) + " /*" + value.format(
            "yyNNNdd HHmmss") + "*/"  # TIME
    if isinstance(value, Duration):
        return str(convert.timedelta2milli(value)) + " /*" + str(
            value) + "*/"  # DURATION

    if Math.is_number(value):
        return str(value)
    return convert.string2quote(value)
Beispiel #18
0
def assertAlmostEqualValue(test,
                           expected,
                           digits=None,
                           places=None,
                           msg=None,
                           delta=None):
    """
    Snagged from unittest/case.py, then modified (Aug2014)
    """
    if test == expected:
        # shortcut
        return

    if not Math.is_number(expected):
        # SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
        if isinstance(expected, list) and len(expected) == 0 and test == None:
            return
        if isinstance(expected,
                      Mapping) and not expected.keys() and test == None:
            return
        if test != expected:
            raise AssertionError(
                expand_template("{{test}} != {{expected}}", locals()))
        return

    num_param = 0
    if digits != None:
        num_param += 1
    if places != None:
        num_param += 1
    if delta != None:
        num_param += 1
    if num_param > 1:
        raise TypeError("specify only one of digits, places or delta")

    if digits is not None:
        try:
            diff = Math.log10(abs(test - expected))
            if diff < digits:
                return
        except Exception, e:
            pass

        standardMsg = expand_template(
            "{{test}} != {{expected}} within {{digits}} decimal places",
            locals())
Beispiel #19
0
    def convert(self, expr):
        """
        ADD THE ".$value" SUFFIX TO ALL VARIABLES
        """
        if isinstance(expr, Expression):
            vars_ = expr.vars()
            rename = {
                v: join_field(split_field(v) + ["$value"])
                for v in vars_
            }
            return expr.map(rename)

        if expr is True or expr == None or expr is False:
            return expr
        elif Math.is_number(expr):
            return expr
        elif expr == ".":
            return "."
        elif is_keyword(expr):
            #TODO: LOOKUP SCHEMA AND ADD ALL COLUMNS WITH THIS PREFIX
            return expr + ".$value"
        elif isinstance(expr, basestring):
            Log.error("{{name|quote}} is not a valid variable name", name=expr)
        elif isinstance(expr, Date):
            return expr
        elif isinstance(expr, QueryOp):
            return self._convert_query(expr)
        elif isinstance(expr, Mapping):
            if expr["from"]:
                return self._convert_query(expr)
            elif len(expr) >= 2:
                #ASSUME WE HAVE A NAMED STRUCTURE, NOT AN EXPRESSION
                return wrap({
                    name: self.convert(value)
                    for name, value in expr.items()
                })
            else:
                # ASSUME SINGLE-CLAUSE EXPRESSION
                k, v = expr.items()[0]
                return self.converter_map.get(k, self._convert_bop)(k, v)
        elif isinstance(expr, (list, set, tuple)):
            return wrap([self.convert(value) for value in expr])
def _scrub(r):
    try:
        if r == None:
            return None
        elif isinstance(r, basestring):
            if r == "":
                return None
            return r
        elif Math.is_number(r):
            return convert.value2number(r)
        elif isinstance(r, Mapping):
            if isinstance(r, Dict):
                r = object.__getattribute__(r, "_dict")
            output = {}
            for k, v in r.items():
                v = _scrub(v)
                if v != None:
                    output[k.lower()] = v
            if len(output) == 0:
                return None
            return output
        elif hasattr(r, '__iter__'):
            if isinstance(r, DictList):
                r = r.list
            output = []
            for v in r:
                v = _scrub(v)
                if v != None:
                    output.append(v)
            if not output:
                return None
            if len(output) == 1:
                return output[0]
            try:
                return sort(output)
            except Exception:
                return output
        else:
            return r
    except Exception, e:
        Log.warning("Can not scrub: {{json}}",  json= r)
Beispiel #21
0
def _scrub(r):
    try:
        if r == None:
            return None
        elif isinstance(r, basestring):
            if r == "":
                return None
            return r
        elif Math.is_number(r):
            return convert.value2number(r)
        elif isinstance(r, Mapping):
            if isinstance(r, Dict):
                r = object.__getattribute__(r, "_dict")
            output = {}
            for k, v in r.items():
                v = _scrub(v)
                if v != None:
                    output[k.lower()] = v
            if len(output) == 0:
                return None
            return output
        elif hasattr(r, '__iter__'):
            if isinstance(r, DictList):
                r = r.list
            output = []
            for v in r:
                v = _scrub(v)
                if v != None:
                    output.append(v)
            if not output:
                return None
            if len(output) == 1:
                return output[0]
            try:
                return sort(output)
            except Exception:
                return output
        else:
            return r
    except Exception, e:
        Log.warning("Can not scrub: {{json}}",  json= r)
Beispiel #22
0
def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, delta=None):
    """
    Snagged from unittest/case.py, then modified (Aug2014)
    """
    if test == expected:
        # shortcut
        return

    if not Math.is_number(expected):
        # SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
        if isinstance(expected, list) and len(expected) == 0 and test == None:
            return
        if isinstance(expected, Mapping) and not expected.keys() and test == None:
            return
        if test != expected:
            raise AssertionError(expand_template("{{test}} != {{expected}}", locals()))
        return

    num_param = 0
    if digits != None:
        num_param += 1
    if places != None:
        num_param += 1
    if delta != None:
        num_param += 1
    if num_param > 1:
        raise TypeError("specify only one of digits, places or delta")

    if digits is not None:
        try:
            diff = Math.log10(abs(test - expected))
            if diff < digits:
                return
        except Exception, e:
            pass

        standardMsg = expand_template("{{test}} != {{expected}} within {{digits}} decimal places", locals())
def stats(values):
    """
    RETURN LOTS OF AGGREGATES
    """
    if values == None:
        return None

    values = values.map(float, includeNone=False)

    z = ZeroMoment.new_instance(values)
    s = Dict()
    for k, v in z.dict.items():
        s[k] = v
    for k, v in ZeroMoment2Stats(z).items():
        s[k] = v
    s.max = MAX(values)
    s.min = MIN(values)
    s.median = pyLibrary.maths.stats.median(values, simple=False)
    s.last = values.last()
    s.first = values[0]
    if Math.is_number(s.variance) and not Math.is_nan(s.variance):
        s.std = sqrt(s.variance)

    return s
def stats(values):
    """
    RETURN LOTS OF AGGREGATES
    """
    if values == None:
        return None

    values = values.map(float, includeNone=False)

    z = ZeroMoment.new_instance(values)
    s = Struct()
    for k, v in z.dict.items():
        s[k] = v
    for k, v in ZeroMoment2Stats(z).items():
        s[k] = v
    s.max = MAX(values)
    s.min = MIN(values)
    s.median = pyLibrary.stats.median(values, simple=False)
    s.last = values.last()
    s.first = values[0]
    if Math.is_number(s.variance) and not Math.is_nan(s.variance):
        s.std = sqrt(s.variance)

    return s
Beispiel #25
0
def qb_expression_to_ruby(expr):
    if expr == None:
        return "nil"
    elif Math.is_number(expr):
        return unicode(expr)
    elif is_keyword(expr):
        return "doc[" + convert.string2quote(expr) + "].value"
    elif isinstance(expr, basestring):
        Log.error("{{name|quote}} is not a valid variable name", name=expr)
    elif isinstance(expr, CODE):
        return expr.code
    elif isinstance(expr, Date):
        return unicode(expr.unix)
    elif expr is True:
        return "true"
    elif expr is False:
        return "false"

    op, term = expr.items()[0]

    mop = ruby_multi_operators.get(op)
    if mop:
        if isinstance(term, list):
            if not term:
                return mop[1]  # RETURN DEFAULT
            else:
                output = mop[0].join(
                    ["(" + qb_expression_to_ruby(t) + ")" for t in term])
                return output
        elif isinstance(term, Mapping):
            a, b = term.items()[0]
            output = "(" + qb_expression_to_ruby(
                a) + ")" + mop[0] + "(" + qb_expression_to_ruby(b) + ")"
            return output
        else:
            qb_expression_to_ruby(term)

    bop = ruby_binary_operators.get(op)
    if bop:
        if isinstance(term, list):
            output = bop.join(
                ["(" + qb_expression_to_ruby(t) + ")" for t in term])
            return output
        elif isinstance(term, Mapping):
            if op == "eq":
                # eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
                output = " and ".join("(" + qb_expression_to_ruby(a) + ")" +
                                      bop + "(" + qb_expression_to_ruby(b) +
                                      ")" for a, b in term.items())
                return output
            else:
                a, b = term.items()[0]
                output = "(" + qb_expression_to_ruby(
                    a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")"
                return output
        else:
            Log.error("Expecting binary term")

    uop = ruby_unary_operators.get(op)
    if uop:
        output = expand_template(uop, {"term": qb_expression_to_ruby(term)})
        return output

    cop = complex_operators.get(op)
    if cop:
        output = cop(term).to_ruby()
        return output

    Log.error("`{{op}}` is not a recognized operation", op=op)
Beispiel #26
0
def get_pending(source, since, pending_bugs, please_stop):
    try:
        while not please_stop:
            if since == None:
                Log.note("Get all records")
                result = source.search({
                    # "query": {"match_all": {}},
                    "query": {
                        "filtered": {
                            "filter": {"exists": {"field": config.primary_field}},
                            "query": {"match_all": {}}
                        }},
                    "fields": ["_id", config.primary_field],
                    "from": 0,
                    "size": BATCH_SIZE,
                    "sort": [config.primary_field]
                })
            else:
                Log.note(
                    "Get records with {{primary_field}} >= {{max_time|datetime}}",
                    primary_field=config.primary_field,
                    max_time=since
                )
                result = source.search({
                    "query": {"filtered": {
                        "query": {"match_all": {}},
                        "filter": {"range": {config.primary_field: {"gte": since}}},
                    }},
                    "fields": ["_id", config.primary_field],
                    "from": 0,
                    "size": BATCH_SIZE,
                    "sort": [config.primary_field]
                })

            new_max_value = MAX([unwraplist(h.fields[literal_field(config.primary_field)]) for h in result.hits.hits])

            if since == new_max_value:
                # GET ALL WITH THIS TIMESTAMP
                result = source.search({
                    "query": {"filtered": {
                        "query": {"match_all": {}},
                        "filter": {"term": {config.primary_field: since}},
                    }},
                    "fields": ["_id", config.primary_field],
                    "from": 0,
                    "size": 100000
                })
                if Math.is_integer(new_max_value):
                    since = int(new_max_value) + 1
                elif Math.is_number(new_max_value):
                    since = float(new_max_value) + 0.5
                else:
                    since = unicode(new_max_value) + "a"
            else:
                since = new_max_value

            ids = result.hits.hits._id
            Log.note("Adding {{num}} to pending queue", num=len(ids))
            pending_bugs.extend(ids)

            if len(result.hits.hits) < BATCH_SIZE:
                break

        Log.note("No more ids")
    except Exception, e:
        please_stop.go()
        Log.error("Problem while copying records", cause=e)
Beispiel #27
0
    def __unicode__(self):
        if not self.milli:
            return "zero"

        output = ""
        rest = (self.milli - (MILLI_VALUES.month * self.month)
                )  # DO NOT INCLUDE THE MONTH'S MILLIS
        isNegative = (rest < 0)
        rest = Math.abs(rest)

        # MILLI
        rem = rest % 1000
        if rem != 0:
            output = "+" + unicode(rem) + "milli" + output
        rest = Math.floor(rest / 1000)

        # SECOND
        rem = rest % 60
        if rem != 0:
            output = "+" + unicode(rem) + "second" + output
        rest = Math.floor(rest / 60)

        # MINUTE
        rem = rest % 60
        if rem != 0:
            output = "+" + unicode(rem) + "minute" + output
        rest = Math.floor(rest / 60)

        # HOUR
        rem = rest % 24
        if rem != 0:
            output = "+" + unicode(rem) + "hour" + output
        rest = Math.floor(rest / 24)

        # DAY
        if (rest < 11 and rest != 7) or rest % 10 == 0:
            rem = rest
            rest = 0
        else:
            rem = rest % 7
            rest = Math.floor(rest / 7)

        if rem != 0:
            output = "+" + unicode(rem) + "day" + output

        # WEEK
        if rest != 0:
            output = "+" + unicode(rest) + "week" + output

        if isNegative:
            output = output.replace("+", "-")

        # MONTH AND YEAR
        if self.month:
            sign = "-" if self.month < 0 else "+"
            month = Math.abs(self.month)

            if month <= 18 and month != 12:
                output = sign + unicode(month) + "month" + output
            else:
                m = month % 12
                if m != 0:
                    output = sign + unicode(m) + "month" + output
                y = Math.floor(month / 12)
                output = sign + unicode(y) + "year" + output

        if output[0] == "+":
            output = output[1::]
        if output[0] == '1' and not Math.is_number(output[1]):
            output = output[1::]
        return output
all = set()

with open(settings.output_file, "r") as input_file:
    with open("good_talos.tab", "w") as output_file:
        for line in input_file:
            try:
                if len(line.strip()) == 0:
                    continue

                col = line.split("\t")
                id = int(col[0])
                if id < MINIMUM_ID:
                    continue

                json = col[1]
                if Math.is_number(json):
                    json = col[2]
                data = CNV.JSON2object(json).json_blob
                date = CNV.unix2datetime(data.testrun.date)

                if id % 1000 == 0:
                    Log.println("loading id " + str(id) + " date: " + CNV.datetime2string(date, "%Y-%m-%d %H:%M:%S"))

                if date < MINIMUM_DATE:
                    continue

                if id in all:
                    continue
                all.add(id)

                arrays_add(id, "[" + data.test_build.branch + "][" + data.testrun.suite + "]", data)
Beispiel #29
0
 def test_isnumber(self):
     assert Math.is_number(9999999999000)
def assertAlmostEqualValue(test,
                           expected,
                           digits=None,
                           places=None,
                           msg=None,
                           delta=None):
    """
    Snagged from unittest/case.py, then modified (Aug2014)
    """
    if expected == None:  # None has no expectations
        return
    if test == expected:
        # shortcut
        return

    if not Math.is_number(expected):
        # SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
        if isinstance(expected, list) and len(expected) == 0 and test == None:
            return
        if isinstance(expected,
                      Mapping) and not expected.keys() and test == None:
            return
        if test != expected:
            raise AssertionError(
                expand_template("{{test}} != {{expected}}", locals()))
        return

    num_param = 0
    if digits != None:
        num_param += 1
    if places != None:
        num_param += 1
    if delta != None:
        num_param += 1
    if num_param > 1:
        raise TypeError("specify only one of digits, places or delta")

    if digits is not None:
        with suppress_exception:
            diff = Math.log10(abs(test - expected))
            if diff < digits:
                return

        standardMsg = expand_template(
            "{{test}} != {{expected}} within {{digits}} decimal places",
            locals())
    elif delta is not None:
        if abs(test - expected) <= delta:
            return

        standardMsg = expand_template(
            "{{test}} != {{expected}} within {{delta}} delta", locals())
    else:
        if places is None:
            places = 15

        with suppress_exception:
            diff = Math.log10(abs(test - expected))
            if diff < Math.ceiling(Math.log10(abs(test))) - places:
                return

        standardMsg = expand_template(
            "{{test|json}} != {{expected|json}} within {{places}} places",
            locals())

    raise AssertionError(coalesce(msg, "") + ": (" + standardMsg + ")")
def qb_expression_to_python(expr):
    if expr == None:
        return "None"
    elif Math.is_number(expr):
        return unicode(expr)
    elif isinstance(expr, Date):
        return unicode(expr.unix)
    elif isinstance(expr, unicode):
        if expr == ".":
            return "row"
        elif is_keyword(expr):
            return "row[" + convert.value2quote(expr) + "]"
        else:
            Log.error("Expecting a json path")
    elif isinstance(expr, CODE):
        return expr.code
    elif expr is True:
        return "True"
    elif expr is False:
        return "False"

    op, term = expr.items()[0]

    mop = python_multi_operators.get(op)
    if mop:
        if isinstance(term, list):
            if not term:
                return mop[1]  # RETURN DEFAULT
            else:
                output = mop[0].join(["(" + qb_expression_to_python(t) + ")" for t in term])
                return output
        elif isinstance(term, Mapping):
            a, b = term.items()[0]
            output = "(" + qb_expression_to_python(a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")"
            return output
        else:
            qb_expression_to_python(term)

    bop = python_binary_operators.get(op)
    if bop:
        if isinstance(term, list):
            output = bop.join(["(" + qb_expression_to_python(t) + ")" for t in term])
            return output
        elif isinstance(term, Mapping):
            if op == "eq":
                # eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
                output = " and ".join("(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")" for a, b in term.items())
                return output
            else:
                a, b = term.items()[0]
                output = "(" + qb_expression_to_python(a) + ")" + bop + "(" + qb_expression_to_python(b) + ")"
                return output
        else:
            Log.error("Expecting binary term")

    uop = python_unary_operators.get(op)
    if uop:
        output = uop + "(" + qb_expression_to_python(term) + ")"
        return output

    Log.error("`{{op}}` is not a recognized operation",  op= op)
    def __unicode__(self):
        if not self.milli:
            return "zero"

        output = ""
        rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS
        isNegative = (rest < 0)
        rest = Math.abs(rest)

        # MILLI
        rem = rest % 1000
        if rem != 0:
            output = "+" + unicode(rem) + "milli" + output
        rest = Math.floor(rest / 1000)

        # SECOND
        rem = rest % 60
        if rem != 0:
            output = "+" + unicode(rem) + "second" + output
        rest = Math.floor(rest / 60)

        # MINUTE
        rem = rest % 60
        if rem != 0:
            output = "+" + unicode(rem) + "minute" + output
        rest = Math.floor(rest / 60)

        # HOUR
        rem = rest % 24
        if rem != 0:
            output = "+" + unicode(rem) + "hour" + output
        rest = Math.floor(rest / 24)

        # DAY
        if (rest < 11 and rest != 7) or rest % 10 == 0:
            rem = rest
            rest = 0
        else:
            rem = rest % 7
            rest = Math.floor(rest / 7)

        if rem != 0:
            output = "+" + unicode(rem) + "day" + output

        # WEEK
        if rest != 0:
            output = "+" + unicode(rest) + "week" + output

        if isNegative:
            output = output.replace("+", "-")

        # MONTH AND YEAR
        if self.month:
            sign = "-" if self.month < 0 else "+"
            month = Math.abs(self.month)

            if month <= 18 and month != 12:
                output = sign + unicode(month) + "month" + output
            else:
                m = month % 12
                if m != 0:
                    output = sign + unicode(m) + "month" + output
                y = Math.floor(month / 12)
                output = sign + unicode(y) + "year" + output

        if output[0] == "+":
            output = output[1::]
        if output[0] == '1' and not Math.is_number(output[1]):
            output = output[1::]
        return output
def qb_expression_to_ruby(expr):
    if expr == None:
        return "nil"
    elif Math.is_number(expr):
        return unicode(expr)
    elif is_keyword(expr):
        return "doc[" + convert.string2quote(expr) + "].value"
    elif isinstance(expr, basestring):
        Log.error("{{name|quote}} is not a valid variable name", name=expr)
    elif isinstance(expr, CODE):
        return expr.code
    elif isinstance(expr, Date):
        return unicode(expr.unix)
    elif expr is True:
        return "true"
    elif expr is False:
        return "false"

    op, term = expr.items()[0]

    mop = ruby_multi_operators.get(op)
    if mop:
        if isinstance(term, list):
            if not term:
                return mop[1]  # RETURN DEFAULT
            else:
                output = mop[0].join(["(" + qb_expression_to_ruby(t) + ")" for t in term])
                return output
        elif isinstance(term, Mapping):
            a, b = term.items()[0]
            output = "(" + qb_expression_to_ruby(a) + ")" + mop[0] + "(" + qb_expression_to_ruby(b) + ")"
            return output
        else:
            qb_expression_to_ruby(term)


    bop = ruby_binary_operators.get(op)
    if bop:
        if isinstance(term, list):
            output = bop.join(["(" + qb_expression_to_ruby(t) + ")" for t in term])
            return output
        elif isinstance(term, Mapping):
            if op == "eq":
                # eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
                output = " and ".join("(" + qb_expression_to_ruby(a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")" for a, b in term.items())
                return output
            else:
                a, b = term.items()[0]
                output = "(" + qb_expression_to_ruby(a) + ")" + bop + "(" + qb_expression_to_ruby(b) + ")"
                return output
        else:
            Log.error("Expecting binary term")

    uop = ruby_unary_operators.get(op)
    if uop:
        output = expand_template(uop, {"term": qb_expression_to_ruby(term)})
        return output

    cop = complex_operators.get(op)
    if cop:
        output = cop(term).to_ruby()
        return output

    Log.error("`{{op}}` is not a recognized operation",  op= op)
Beispiel #34
0
def qb_expression_to_python(expr):
    if expr == None:
        return "None"
    elif Math.is_number(expr):
        return unicode(expr)
    elif isinstance(expr, Date):
        return unicode(expr.unix)
    elif isinstance(expr, unicode):
        if expr == ".":
            return "row"
        elif is_keyword(expr):
            return "row[" + convert.value2quote(expr) + "]"
        else:
            Log.error("Expecting a json path")
    elif isinstance(expr, CODE):
        return expr.code
    elif expr is True:
        return "True"
    elif expr is False:
        return "False"

    op, term = expr.items()[0]

    mop = python_multi_operators.get(op)
    if mop:
        if isinstance(term, list):
            if not term:
                return mop[1]  # RETURN DEFAULT
            else:
                output = mop[0].join(
                    ["(" + qb_expression_to_python(t) + ")" for t in term])
                return output
        elif isinstance(term, Mapping):
            a, b = term.items()[0]
            output = "(" + qb_expression_to_python(
                a) + ")" + mop[0] + "(" + qb_expression_to_python(b) + ")"
            return output
        else:
            qb_expression_to_python(term)

    bop = python_binary_operators.get(op)
    if bop:
        if isinstance(term, list):
            output = bop.join(
                ["(" + qb_expression_to_python(t) + ")" for t in term])
            return output
        elif isinstance(term, Mapping):
            if op == "eq":
                # eq CAN ACCEPT A WHOLE OBJECT OF key:value PAIRS TO COMPARE
                output = " and ".join("(" + qb_expression_to_python(a) + ")" +
                                      bop + "(" + qb_expression_to_python(b) +
                                      ")" for a, b in term.items())
                return output
            else:
                a, b = term.items()[0]
                output = "(" + qb_expression_to_python(
                    a) + ")" + bop + "(" + qb_expression_to_python(b) + ")"
                return output
        else:
            Log.error("Expecting binary term")

    uop = python_unary_operators.get(op)
    if uop:
        output = uop + "(" + qb_expression_to_python(term) + ")"
        return output

    Log.error("`{{op}}` is not a recognized operation", op=op)