Example #1
0
        def exe(node):
            """
				node[0] - operator name
				node[1:] - params
			"""
            types = [
                str, timeutils.datetime.time, timeutils.datetime.date,
                timeutils.datetime.datetime
            ]
            try:
                types += [unicode]
            except:
                pass
            if D:
                self.start("executing node %s",
                           color.bold(self.cleanOutput(node)))
            type_node = type(node)
            if node is None or type_node in TYPES:
                return node
            elif type_node in types:
                return node
            elif type_node is list:
                return (exe(n) for n in node)
            elif type_node is dict:
                ret = {}
                for i in node.items():
                    ret[exe(i[0])] = exe(i[1])
                return ret
            op = node[0]
            if op == "or":
                if D: self.debug("%s or %s", node[1], node[2])
                return exe(node[1]) or exe(node[2])
            elif op == "and":
                if D: self.debug("%s and %s", node[1], node[2])
                return exe(node[1]) and exe(node[2])
            elif op == "+":
                if len(node) > 2:
                    fst = exe(node[1])
                    snd = exe(node[2])
                    if None in (fst, snd):
                        return fst or snd
                    typefst = type(fst)
                    typesnd = type(snd)
                    if typefst is dict:
                        try:
                            fst.update(snd)
                        except Exception:
                            if type(snd) is not dict:
                                raise ProgrammingError(
                                    "Can't add value of type %s to %s" %
                                    (color.bold(
                                        PY_TYPES_MAP.get(
                                            type(snd).__name__,
                                            type(snd).__name__)),
                                     color.bold("object")))
                        return fst
                    if typefst is list and typesnd is list:
                        if D:
                            self.debug("both sides are lists, returning '%s'",
                                       fst + snd)
                        return fst + snd
                    if typefst in ITER_TYPES or typesnd in ITER_TYPES:
                        if typefst not in ITER_TYPES:
                            fst = [fst]
                        elif typesnd not in ITER_TYPES:
                            snd = [snd]
                        if D:
                            self.debug(
                                "at least one side is a generator and the other is an iterable, returning chain"
                            )
                        return chain(fst, snd)
                    if typefst in NUM_TYPES:
                        try:
                            return fst + snd
                        except Exception:
                            return fst + float(snd)
                    if typefst in STR_TYPES or typesnd in STR_TYPES:
                        if D:
                            self.info("doing string comparison '%s' is '%s'",
                                      fst, snd)
                        if sys.version_info[0] < 3:
                            if typefst is unicode:
                                fst = fst.encode("utf-8")
                            if typesnd is unicode:
                                snd = snd.encode("utf-8")
                        return str(fst) + str(snd)
                    try:
                        timeType = timeutils.datetime.time
                        if typefst is timeType and typesnd is timeType:
                            return timeutils.addTimes(fst, snd)
                    except Exception:
                        pass
                    if D:
                        self.debug("standard addition, returning '%s'",
                                   fst + snd)
                    return fst + snd
                else:
                    return exe(node[1])
            elif op == "-":
                if len(node) > 2:
                    fst = exe(node[1])
                    snd = exe(node[2])
                    try:
                        return fst - snd
                    except Exception:
                        typefst = type(fst)
                        typesnd = type(snd)
                        timeType = timeutils.datetime.time
                        if typefst is timeType and typesnd is timeType:
                            return timeutils.subTimes(fst, snd)
                else:
                    return -exe(node[1])
            elif op == "*":
                return exe(node[1]) * exe(node[2])
            elif op == "%":
                return exe(node[1]) % exe(node[2])
            elif op == "/":
                return exe(node[1]) / float(exe(node[2]))
            elif op == ">":
                if D:
                    self.debug("%s > %s, %s", node[1], node[2],
                               node[1] > node[2])
                return exe(node[1]) > exe(node[2])
            elif op == "<":
                return exe(node[1]) < exe(node[2])
            elif op == ">=":
                return exe(node[1]) >= exe(node[2])
            elif op == "<=":
                return exe(node[1]) <= exe(node[2])
            # TODO this algorithm produces 3 for 1<2<3 and should be true
            # elif op in "<=>=":
            # 	fst=exe(node[1])
            # 	snd=exe(node[2])
            # 	if op==">":
            # 		return fst > snd and snd or False
            # 	elif op=="<":
            # 		return fst < snd and snd or False
            # 	elif op==">=":
            # 		return fst >= snd and snd or False
            # 	elif op=="<=":
            # 		return fst <= snd and snd or False
            elif op == "not":
                fst = exe(node[1])
                if D: self.debug("doing not '%s'", fst)
                return not fst
            elif op == "in":
                fst = exe(node[1])
                snd = exe(node[2])
                if D: self.debug("doing '%s' in '%s'", node[1], node[2])
                if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
                    return any(x in max(fst, snd, key=len)
                               for x in min(fst, snd, key=len))
                return exe(node[1]) in exe(node[2])
            elif op == "not in":
                fst = exe(node[1])
                snd = exe(node[2])
                if D: self.debug("doing '%s' not in '%s'", node[1], node[2])
                if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
                    return not any(x in max(fst, snd, key=len)
                                   for x in min(fst, snd, key=len))
                return exe(node[1]) not in exe(node[2])
            elif op in ("is", "is not"):
                if D: self.debug("found operator '%s'", op)
                # try:
                fst = exe(node[1])
                # except Exception as e:
                # 	if D: self.debug("NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",node[1],str(e))
                # 	fst=node[1]
                # try:
                snd = exe(node[2])
                # except Exception as e:
                # 	if D: self.debug("NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",node[2],str(e))
                # 	snd=node[2]
                if op == "is" and fst == snd:
                    return True

                # this doesn't work for 3 is not '3'
                # if op == "is not" and fst != snd:
                # 	return True
                typefst = type(fst)
                typesnd = type(snd)
                if D:
                    self.debug("type fst: '%s', type snd: '%s'", typefst,
                               typesnd)
                if typefst in STR_TYPES:
                    if D:
                        self.info("doing string comparison '\"%s\" is \"%s\"'",
                                  fst, snd)
                    ret = str(fst) == str(snd)
                elif typefst is float or typesnd is float:
                    if D:
                        self.info("doing float comparison '%s is %s'", fst,
                                  snd)
                    try:
                        ret = abs(float(fst) - float(snd)) < EPSILON
                    except:
                        ret = False
                elif typefst is int or typesnd is int:
                    if D:
                        self.info("doing integer comparison '%s is %s'", fst,
                                  snd)
                    try:
                        ret = int(fst) == int(snd)
                    except:
                        ret = False
                elif typefst is list and typesnd is list:
                    if D:
                        self.info("doing array comparison '%s' is '%s'", fst,
                                  snd)
                    ret = fst == snd
                elif typefst is dict and typesnd is dict:
                    if D:
                        self.info("doing object comparison '%s' is '%s'", fst,
                                  snd)
                    ret = fst == snd
                elif fst is None or snd is None:
                    if fst is None and snd is None:
                        # this executes only for "is not"
                        ret = True
                    else:
                        ret = (fst or snd) is None
                        if D:
                            self.info("doing None comparison %s is %s = %s",
                                      color.bold(fst), color.bold(snd),
                                      color.bold(not not (fst or snd)))
                else:
                    if D:
                        self.info("can't compare %s and %s. Returning False",
                                  self.cleanOutput(fst), self.cleanOutput(snd))
                    ret = False
                # else:
                # 	try:
                # 		global ObjectId
                # 		if not ObjectId:
                # 			from bson.objectid import ObjectId
                # 		if typefst is ObjectId or typesnd is ObjectId:
                # 			if D: self.info("doing MongoDB objectID comparison '%s' is '%s'",fst,snd)
                # 			ret=str(fst)==str(snd)
                # 		else:
                # 			if D: self.info("doing standard comparison '%s' is '%s'",fst,snd)
                # 			ret=fst is snd
                # 	except Exception:
                # 		pass
                if op == "is not":
                    if D: self.info("'is not' found. Returning %s", not ret)
                    return not ret
                else:
                    if D:
                        self.info("returning %s is %s => %s",
                                  color.bold(self.cleanOutput(fst)),
                                  color.bold(self.cleanOutput(snd)),
                                  color.bold(ret))
                    return ret
            elif op == "re":
                return re.compile(exe(node[1]))
            elif op == "matches":
                fst = exe(node[1])
                snd = exe(node[2])
                if type(fst) not in STR_TYPES + [RE_TYPE]:
                    raise Exception(
                        "operator " + color.bold("matches") +
                        " expects regexp on the left. Example: 'a.*d' matches 'abcd'"
                    )
                if type(snd) in ITER_TYPES:
                    for i in snd:
                        if not not re.match(fst, i):
                            return True
                    return False
                else:
                    # regex matches string
                    return not not re.match(fst, snd)
            # elif op=="(literal)":
            # 	fstLetter=node[1][0]
            # 	if fstLetter is "'":
            # 		return node[1][1:-1]
            # 	elif fstLetter.isdigit:
            # 		return int(node[1])
            elif op == "(root)":  # this is $
                return self.data
            # elif op=="(node)":# this is !
            # 	if D: self.debug("returning node %s",self.node)
            # 	return self.node
            elif op == "(current)":  # this is @
                if D:
                    self.debug("returning current node: \n  %s",
                               color.bold(self.current))
                return self.current
            elif op == "name":
                return node[1]
            elif op == ".":
                fst = node[1]
                if type(fst) is tuple:
                    fst = exe(fst)
                typefst = type(fst)
                if D:
                    self.debug(
                        color.op(".") + " left is '%s'",
                        color.bold(self.cleanOutput(fst)))
                # try:
                if node[2][0] == "*":
                    if D:
                        self.end(
                            color.op(".") + " returning '%s'",
                            color.bold(typefst in ITER_TYPES and fst or [fst]))
                    return fst  # typefst in ITER_TYPES and fst or [fst]
                # except:
                # 	pass
                snd = exe(node[2])
                if D:
                    self.debug(
                        color.op(".") + " right is '%s'", color.bold(snd))
                if typefst in ITER_TYPES:
                    if D:
                        self.debug(
                            color.op(".") + " filtering %s by %s",
                            color.bold(self.cleanOutput(fst)), color.bold(snd))
                    if type(snd) in ITER_TYPES:
                        return filter_dict(fst, list(snd))
                    else:
                        # if D: self.debug(list(fst))
                        return (e[snd] for e in fst
                                if type(e) is dict and snd in e)
                try:
                    if D:
                        self.end(
                            color.op(".") + " returning '%s'", fst.get(snd))
                    return fst.get(snd)
                except Exception:
                    if isinstance(fst, object):
                        return self.object_getter(fst, snd)
                    if D:
                        self.end(
                            color.op(".") + " returning '%s'", color.bold(fst))
                    return fst
            elif op == "..":
                fst = flatten(exe(node[1]))
                if node[2][0] == "*":
                    if D:
                        self.debug(
                            color.op("..") + " returning '%s'",
                            color.bold(fst))
                    return fst
                # reduce objects to selected attributes
                snd = exe(node[2])
                if D:
                    self.debug(
                        color.op("..") + " finding all %s in %s",
                        color.bold(snd), color.bold(self.cleanOutput(fst)))
                if type(snd) in ITER_TYPES:
                    ret = filter_dict(fst, list(snd))
                    if D:
                        self.debug(
                            color.op("..") + " returning %s", color.bold(ret))
                    return ret
                else:
                    ret = chain(*(type(x) in ITER_TYPES and x or [x]
                                  for x in (e[snd] for e in fst if snd in e)))
                    # print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
                    if D:
                        self.debug(
                            color.op("..") + " returning %s",
                            color.bold(self.cleanOutput(ret)))
                    return ret
            elif op == "[":
                len_node = len(node)
                # TODO move it to tree generation phase
                if len_node is 1:  # empty list
                    if D: self.debug("returning an empty list")
                    return []
                if len_node is 2:  # list - preserved to catch possible event of leaving it as '[' operator
                    if D: self.debug("doing list mapping")
                    return [exe(x) for x in node[1]]
                if len_node is 3:  # selector used []
                    fst = exe(node[1])
                    # check against None
                    if not fst:
                        return fst
                    selector = node[2]
                    if D:
                        self.debug(
                            "\n  found selector '%s'.\n  executing on %s",
                            color.bold(selector), color.bold(fst))
                    selectorIsTuple = type(selector) is tuple

                    if selectorIsTuple and selector[0] is "[":
                        nodeList = []
                        nodeList_append = nodeList.append
                        for i in fst:
                            if D:
                                self.debug("setting self.current to %s",
                                           color.bold(i))
                            self.current = i
                            nodeList_append(
                                exe((selector[0], exe(selector[1]),
                                     exe(selector[2]))))
                        if D:
                            self.debug("returning %s objects: %s",
                                       color.bold(len(nodeList)),
                                       color.bold(nodeList))
                        return nodeList

                    if selectorIsTuple and selector[0] == "(current)":
                        if D:
                            self.warning(
                                color.bold("$.*[@]") + " is eqivalent to " +
                                color.bold("$.*") + "!")
                        return fst

                    if selectorIsTuple and selector[0] in SELECTOR_OPS:
                        if D:
                            self.debug("found %s operator in selector, %s",
                                       color.bold(selector[0]),
                                       color.bold(selector))
                        if type(fst) is dict:
                            fst = [fst]
                        # TODO move it to tree building phase
                        if type(selector[1]
                                ) is tuple and selector[1][0] == "name":
                            selector = (selector[0], selector[1][1],
                                        selector[2])
                        selector0 = selector[0]
                        selector1 = selector[1]
                        selector2 = selector[2]

                        def exeSelector(fst):
                            for i in fst:
                                if D:
                                    self.debug("setting self.current to %s",
                                               color.bold(i))
                                    self.debug(
                                        "  s0: %s\n  s1: %s\n  s2: %s\n  Current: %s",
                                        selector0, selector1, selector2, i)
                                self.current = i
                                if selector0 == "fn":
                                    yield exe(selector)
                                # elif type(selector1) in STR_TYPES and False:
                                # 	if D: self.debug("found string %s", type(i))
                                # 	try:
                                # 		if exe((selector0,i[selector1],selector2)):
                                # 			yield i
                                # 			if D: self.debug("appended")
                                # 		if D: self.debug("discarded")
                                # 	except Exception as e:
                                # 		if D: self.debug("discarded, Exception: %s",color.bold(e))
                                else:
                                    try:
                                        # TODO optimize an event when @ is not used. exe(selector1) can be cached
                                        if exe((selector0, exe(selector1),
                                                exe(selector2))):
                                            yield i
                                            if D: self.debug("appended %s", i)
                                        elif D: self.debug("discarded")
                                    except Exception:
                                        if D: self.debug("discarded")

                        # if D and nodeList: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
                        return exeSelector(fst)
                    self.current = fst
                    snd = exe(node[2])
                    typefst = type(fst)
                    if typefst in [tuple] + ITER_TYPES + STR_TYPES:
                        typesnd = type(snd)
                        # nodes[N]
                        if typesnd in NUM_TYPES or typesnd is str and snd.isdigit(
                        ):
                            n = int(snd)
                            if D:
                                self.info("getting %sth element from '%s'",
                                          color.bold(n), color.bold(fst))
                            if typefst in (generator, chain):
                                if n > 0:
                                    return skip(fst, n)
                                elif n == 0:
                                    return next(fst)
                                else:
                                    fst = list(fst)
                            else:
                                try:
                                    return fst[n]
                                except (IndexError, TypeError):
                                    return None
                        # $.*['string']==$.string
                        if type(snd) in STR_TYPES:
                            return exe((".", fst, snd))
                        else:
                            # $.*[@.string] - bad syntax, but allowed
                            return snd
                    else:
                        try:
                            if D:
                                self.debug("returning %s",
                                           color.bold(fst[snd]))
                            return fst[snd]
                        except KeyError:
                            # CHECK - is it ok to do that or should it be ProgrammingError?
                            if D: self.debug("returning an empty list")
                            return []
                raise ProgrammingError("Wrong usage of " + color.bold("[") +
                                       " operator")
            elif op == "fn":
                # Built-in functions
                fnName = node[1]
                args = None
                try:
                    args = [exe(x) for x in node[2:]]
                except IndexError:
                    if D:
                        self.debug("NOT ERROR: can't map '%s' with '%s'",
                                   node[2:], exe)
                # arithmetic
                if fnName == "sum":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    return sum((x for x in args if type(x) in NUM_TYPES))
                elif fnName == "max":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    return max((x for x in args if type(x) in NUM_TYPES))
                elif fnName == "min":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    return min((x for x in args if type(x) in NUM_TYPES))
                elif fnName == "avg":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    if type(args) not in ITER_TYPES:
                        raise Exception("Argument for avg() is not an array")
                    else:
                        args = list(args)
                    try:
                        return sum(args) / float(len(args))
                    except TypeError:
                        args = [x for x in args if type(x) in NUM_TYPES]
                        self.warning("Some items in array were ommited")
                        return sum(args) / float(len(args))
                elif fnName == "round":
                    return round(*args)
                # casting
                elif fnName == "int":
                    return int(args[0])
                elif fnName == "float":
                    return float(args[0])
                elif fnName == "str":
                    return str(py2JSON(args[0]))
                elif fnName in ("list", "array"):
                    try:
                        a = args[0]
                    except IndexError:
                        return []
                    targs = type(a)
                    if targs is timeutils.datetime.datetime:
                        return timeutils.date2list(a) + timeutils.time2list(a)
                    if targs is timeutils.datetime.date:
                        return timeutils.date2list(a)
                    if targs is timeutils.datetime.time:
                        return timeutils.time2list(a)
                    return list(a)
                # string
                elif fnName == "upper":
                    return args[0].upper()
                elif fnName == "lower":
                    return args[0].lower()
                elif fnName == "capitalize":
                    return args[0].capitalize()
                elif fnName == "title":
                    return args[0].title()
                elif fnName == "split":
                    return args[0].split(*args[1:])
                elif fnName == "slice":
                    if args and type(args[1]) not in ITER_TYPES:
                        raise ExecutionError(
                            "Wrong usage of slice(STRING, ARRAY). Second argument is not an array but %s."
                            % color.bold(type(args[1]).__name__))
                    try:
                        pos = list(args[1])
                        if type(pos[0]) in ITER_TYPES:
                            if D:
                                self.debug("run slice() for a list of slicers")
                            return (args[0][x[0]:x[1]] for x in pos)
                        return args[0][pos[0]:pos[1]]
                    except IndexError:
                        if len(args) != 2:
                            raise ProgrammingError(
                                "Wrong usage of slice(STRING, ARRAY). Provided %s argument, should be exactly 2."
                                % len(args))
                elif fnName == "escape":
                    global escape, escapeDict
                    if not escape:
                        from objectpath.utils import escape, escapeDict
                    return escape(args[0], escapeDict)
                elif fnName == "unescape":
                    global unescape, unescapeDict
                    if not unescape:
                        from objectpath.utils import unescape, unescapeDict
                    return unescape(args[0], unescapeDict)
                elif fnName == "replace":
                    if sys.version_info[0] < 3 and type(args[0]) is unicode:
                        args[0] = args[0].encode("utf8")
                    return str.replace(args[0], args[1], args[2])
                # TODO this should be supported by /regex/
                # elif fnName=="REsub":
                # 	return re.sub(args[1],args[2],args[0])
                elif fnName == "sort":
                    if len(args) > 1:
                        key = args[1]
                        a = {"key": lambda x: x.get(key, 0)}
                    else:
                        a = {}
                    args = args[0]
                    if D: self.debug("doing sort on '%s'", args)
                    try:
                        return sorted(args, **a)
                    except TypeError:
                        return args
                elif fnName == "reverse":
                    args = args[0]
                    try:
                        args.reverse()
                        return args
                    except TypeError:
                        return args
                elif fnName == "unique":
                    try:
                        return list(set(args[0]))
                    except TypeError:
                        return args[0]
                elif fnName == "map":
                    return chain(
                        *map(lambda x: exe(("fn", args[0], x)), args[1]))
                elif fnName in ("count", "len"):
                    args = args[0]
                    if args in (True, False, None):
                        return args
                    if type(args) in ITER_TYPES:
                        return len(list(args))
                    return len(args)
                elif fnName == "join":
                    try:
                        joiner = args[1]
                    except Exception:
                        joiner = ""
                    try:
                        return joiner.join(args[0])
                    except TypeError:
                        try:
                            return joiner.join(map(str, args[0]))
                        except Exception:
                            return args[0]
                # time
                elif fnName in ("now", "age", "time", "date", "dateTime"):
                    if fnName == "now":
                        return timeutils.now()
                    if fnName == "date":
                        return timeutils.date(args)
                    if fnName == "time":
                        return timeutils.time(args)
                    if fnName == "dateTime":
                        return timeutils.dateTime(args)
                    # TODO move lang to localize() entirely!
                    if fnName == "age":
                        a = {}
                        if len(args) > 1:
                            a["reference"] = args[1]
                        if len(args) > 2:
                            a["lang"] = args[2]
                        return list(timeutils.age(args[0], **a))
                elif fnName == "toMillis":
                    args = args[0]
                    if args.utcoffset() is not None:
                        args = args - args.utcoffset()  # pylint: disable=E1103
                    global calendar
                    if not calendar:
                        import calendar
                    return int(
                        calendar.timegm(args.timetuple()) * 1000 +
                        args.microsecond / 1000)
                elif fnName == "localize":
                    if type(args[0]) is timeutils.datetime.datetime:
                        return timeutils.UTC2local(*args)
                # polygons
                elif fnName == "area":

                    def segments(p):
                        p = list(map(lambda x: x[0:2], p))
                        return zip(p, p[1:] + [p[0]])

                    return 0.5 * abs(
                        sum(x0 * y1 - x1 * y0
                            for ((x0, y0), (x1, y1)) in segments(args[0])))
                # misc
                elif fnName == "keys":
                    try:
                        return list(args[0].keys())
                    except AttributeError:
                        raise ExecutionError(
                            "Argument is not " + color.bold("object") +
                            " but %s in keys()" %
                            color.bold(type(args[0]).__name__))
                elif fnName == "values":
                    try:
                        return list(args[0].values())
                    except AttributeError:
                        raise ExecutionError(
                            "Argument is not " + color.bold("object") +
                            " but %s in values()" %
                            color.bold(type(args[0]).__name__))
                elif fnName == "type":
                    ret = type(args[0])
                    if ret in ITER_TYPES:
                        return "array"
                    if ret is dict:
                        return "object"
                    return ret.__name__
                elif fnName in self._REGISTERED_FUNCTIONS:
                    return self._REGISTERED_FUNCTIONS[fnName](*args)
                else:
                    raise ProgrammingError("Function " + color.bold(fnName) +
                                           " does not exist.")
            else:
                return node
Example #2
0
		def exe(node):
			"""
				node[0] - operator name
				node[1:] - params
			"""
			if D: self.start("executing node '%s'", node)
			type_node=type(node)
			if node is None or isinstance(node, tuple(TYPES)):
				return node
			elif isinstance(node, list):
				return (exe(n) for n in node)
			elif isinstance(node, dict):
				ret={}
				for i in node.items():
					ret[exe(i[0])]=exe(i[1])
				return ret
			op=node[0]
			if op=="or":
				if D: self.debug("%s or %s", node[1],node[2])
				return exe(node[1]) or exe(node[2])
			elif op=="and":
				if D: self.debug("%s and %s", node[1],node[2])
				return exe(node[1]) and exe(node[2])
			elif op=="+":
				if len(node)>2:
					fst=exe(node[1])
					snd=exe(node[2])
					if None in (fst,snd):
						return fst or snd
					if isinstance(fst, dict):
						try:
							fst.update(snd)
						except Exception:
							if not isinstance(snd, dict):
								raise ProgrammingError("Can't add value of type %s to %s" % (color.bold(PY_TYPES_MAP.get(type(snd).__name__, type(snd).__name__)), color.bold("object")))
						return fst
					if isinstance(fst, list) and isinstance(snd, list):
						if D: self.debug("both sides are lists, returning '%s'",fst+snd)
						return fst+snd
					if isinstance(fst, tuple(ITER_TYPES)) or isinstance(snd, tuple(ITER_TYPES)):
						if not isinstance(fst, tuple(ITER_TYPES)):
							fst=[fst]
						elif not isinstance(snd, tuple(ITER_TYPES)):
							snd=[snd]
						if D: self.debug("at least one side is generator and other is iterable, returning chain")
						return chain(fst,snd)
					if isinstance(fst, tuple(NUM_TYPES)):
						try:
							return fst+snd
						except Exception:
							return fst+float(snd)
					if isinstance(fst, tuple(STR_TYPES)) or isinstance(snd, tuple(STR_TYPES)):
						if D: self.info("doing string comparison '%s' is '%s'",fst,snd)
						if sys.version_info.major < 3:
							if isinstance(fst, unicode):
								fst=fst.encode("utf-8")
							if isinstance(snd, unicode):
								snd=snd.encode("utf-8")
						return str(fst)+str(snd)
					try:
						timeType=timeutils.datetime.time
						if isinstance(fst, timeType) and isinstance(snd, timeType):
							return timeutils.addTimes(fst,snd)
					except Exception:
						pass
					if D: self.debug("standard addition, returning '%s'",fst+snd)
					return fst + snd
				else:
					return exe(node[1])
			elif op=="-":
				if len(node)>2:
					fst=exe(node[1])
					snd=exe(node[2])
					try:
						return fst-snd
					except Exception:
						timeType=timeutils.datetime.time
						if isinstance(fst, timeType) and isinstance(snd, timeType):
							return timeutils.subTimes(fst,snd)
				else:
					return - exe(node[1])
			elif op=="*":
				return exe(node[1]) * exe(node[2])
			elif op=="%":
				return exe(node[1]) % exe(node[2])
			elif op=="/":
				return exe(node[1]) / float(exe(node[2]))
			elif op==">":
				if D: self.debug("%s > %s", node[1],node[2])
				return exe(node[1]) > exe(node[2])
			elif op=="<":
				return exe(node[1]) < exe(node[2])
			elif op==">=":
				return exe(node[1]) >= exe(node[2])
			elif op=="<=":
				return exe(node[1]) <= exe(node[2])
			# TODO this algorithm produces 3 for 1<2<3 and should be true
			# elif op in "<=>=":
			# 	fst=exe(node[1])
			# 	snd=exe(node[2])
			# 	if op==">":
			# 		return fst > snd and snd or False
			# 	elif op=="<":
			# 		return fst < snd and snd or False
			# 	elif op==">=":
			# 		return fst >= snd and snd or False
			# 	elif op=="<=":
			# 		return fst <= snd and snd or False
			elif op=="not":
				fst=exe(node[1])
				if D: self.debug("doing not '%s'",fst)
				return not fst
			elif op=="in":
				fst=exe(node[1])
				snd=exe(node[2])
				if D: self.debug("doing '%s' in '%s'",node[1],node[2])
				if isinstance(fst, tuple(ITER_TYPES)) and isinstance(snd, tuple(ITER_TYPES)):
					return any(x in max(fst,snd,key=len) for x in min(fst,snd,key=len))
				return exe(node[1]) in exe(node[2])
			elif op=="not in":
				fst=exe(node[1])
				snd=exe(node[2])
				if D: self.debug("doing '%s' not in '%s'",node[1],node[2])
				if isinstance(fst, tuple(ITER_TYPES)) and isinstance(snd, tuple(ITER_TYPES)):
					return not any(x in max(fst,snd,key=len) for x in min(fst,snd,key=len))
				return exe(node[1]) not in exe(node[2])
			elif op in ("is","is not"):
				if D: self.debug("found operator '%s'",op)
				# try:
				fst=exe(node[1])
				# except Exception as e:
				# 	if D: self.debug("NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",node[1],str(e))
				# 	fst=node[1]
				# try:
				snd=exe(node[2])
				# except Exception as e:
				# 	if D: self.debug("NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",node[2],str(e))
				# 	snd=node[2]
				if op == "is" and fst == snd:
					return True
				# this doesn't work for 3 is not '3'
				# if op == "is not" and fst != snd:
				# 	return True
				if D: self.debug("type fst: '%s', type snd: '%s'",typefst,typesnd)
				if isinstance(fst, tuple(STR_TYPES)):
					if D: self.info("doing string comparison '\"%s\" is \"%s\"'",fst,snd)
					ret=fst==str(snd)
				elif isinstance(fst, float):
					if D: self.info("doing float comparison '%s is %s'",fst,snd)
					ret=abs(fst-float(snd))<EPSILON
				elif isinstance(fst, int):
					if D: self.info("doing integer comparison '%s is %s'",fst,snd)
					ret=fst==int(snd)
				elif isinstance(fst, list) and isinstance(snd, list):
					if D: self.info("doing array comparison '%s' is '%s'",fst,snd)
					ret=fst==snd
				elif isinstance(fst, dict) and isinstance(snd, dict):
					if D: self.info("doing object comparison '%s' is '%s'",fst,snd)
					ret=fst==snd
				# else:
				# 	try:
				# 		global ObjectId
				# 		if not ObjectId:
				# 			from bson.objectid import ObjectId
				# 		if typefst is ObjectId or typesnd is ObjectId:
				# 			if D: self.info("doing MongoDB objectID comparison '%s' is '%s'",fst,snd)
				# 			ret=str(fst)==str(snd)
				# 		else:
				# 			if D: self.info("doing standard comparison '%s' is '%s'",fst,snd)
				# 			ret=fst is snd
				# 	except Exception:
				# 		pass
				if op=="is not":
					if D: self.info("'is not' found. Returning %s",not ret)
					return not ret
				else:
					if D: self.info("returning '%s' is '%s'='%s'",fst,snd,ret)
					return ret
			elif op=="re":
				return re.compile(exe(node[1]))
			elif op=="matches":
				return not not re.match(exe(node[1]), exe(node[2]))
			# elif op=="(literal)":
			# 	fstLetter=node[1][0]
			# 	if fstLetter is "'":
			# 		return node[1][1:-1]
			# 	elif fstLetter.isdigit:
			# 		return int(node[1])
			elif op=="(root)": # this is $
				return self.data
			# elif op=="(node)":# this is !
			# 	if D: self.debug("returning node %s",self.node)
			# 	return self.node
			elif op=="(current)": # this is @
				if D: self.debug("returning current node %s", self.current)
				return self.current
			elif op=="name":
				return node[1]
			elif op==".":
				fst=node[1]
				if isinstance(fst, tuple):
					fst=exe(fst)
				if D: self.debug(color.op(".")+" left is '%s'", fst)
				# try:
				if node[2][0] == "*":
					if D: self.end(color.op(".")+" returning '%s'", isinstance(fst, tuple(ITER_TYPES)) and fst or [fst])
					return fst # typefst in ITER_TYPES and fst or [fst]
				# except:
				# 	pass
				snd=exe(node[2])
				if D: self.debug(color.op(".")+" right is '%s'",snd)
				if isinstance(fst, tuple(ITER_TYPES)):
					if D: self.debug(color.op(".")+" filtering %s by %s",color.bold(fst),color.bold(snd))
					if isinstance(snd, tuple(ITER_TYPES)):
						return filter_dict(fst, list(snd))
					else:
						# if D: self.debug(list(fst))
						return (e[snd] for e in fst if isinstance(e, dict) and snd in e)
				try:
					if D: self.end(color.op(".")+" returning '%s'",fst.get(snd))
					return fst.get(snd)
				except Exception:
					if isinstance(fst,object):
						return self.object_getter(fst, snd)
					if D: self.end(color.op(".")+" returning '%s'", color.bold(fst))
					return fst
			elif op=="..":
				fst=flatten(exe(node[1]))
				if node[2][0]=="*":
					if D: self.debug(color.op("..")+" returning '%s'", color.bold(fst))
					return fst
				# reduce objects to selected attributes
				snd=exe(node[2])
				if D: self.debug(color.op("..")+" finding all %s in %s", color.bold(snd), color.bold(fst))
				if isinstance(snd, tuple(ITER_TYPES)):
					ret=filter_dict(fst, list(snd))
					if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
					return ret
				else:
					ret=chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))
					# print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
					if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
					return ret
			elif op=="[":
				len_node=len(node)
				# TODO move it to tree generation phase
				if len_node is 1: # empty list
					if D: self.debug("returning an empty list")
					return []
				if len_node is 2: # list - preserved to catch possible event of leaving it as '[' operator
					if D: self.debug("doing list mapping")
					return [exe(x) for x in node[1]]
				if len_node is 3: # selector used []
					fst=exe(node[1])
					# check against None
					if not fst:
						return fst
					selector=node[2]
					if D: self.debug("found '%s' selector. executing on %s", color.bold(selector),color.bold(fst))
					selectorIsTuple=isinstance(selector, tuple)

					if selectorIsTuple and selector[0] is "[":
						nodeList=[]
						nodeList_append=nodeList.append
						for i in fst:
							if D: self.debug("setting self.current to %s",color.bold(i))
							self.current=i
							nodeList_append(exe((selector[0],exe(selector[1]),exe(selector[2]))))
						if D: self.debug("returning %s objects: %s", color.bold(len(nodeList)),color.bold(nodeList))
						return nodeList

					if selectorIsTuple and selector[0] == "(current)":
						if D: self.warning(color.bold("$.*[@]")+" is eqivalent to "+color.bold("$.*")+"!")
						return fst

					if selectorIsTuple and selector[0] in SELECTOR_OPS:
						if D: self.debug("found %s operator in selector", color.bold(selector[0]))
						if isinstance(fst, dict):
							fst=[fst]
						# TODO move it to tree building phase
						if isinstance(selector[1], tuple) and selector[1][0]=="name":
							selector=(selector[0],selector[1][1],selector[2])
						selector0=selector[0]
						selector1=selector[1]
						selector2=selector[2]

						def exeSelector(fst):
							for i in fst:
								if D: self.debug("setting self.current to %s", color.bold(i))
								self.current=i
								if selector0=="fn":
									yield exe(selector)
								elif isinstance(selector1, tuple(STR_TYPES)):
									try:
										if exe((selector0,i[selector1],selector2)):
											yield i
											if D: self.debug("appended")
										if D: self.debug("discarded")
									except Exception as e:
										if D: self.debug("discarded, Exception: %s",color.bold(e))
								else:
									try:
										# TODO optimize an event when @ is not used. exe(selector1) can be cached
										if exe((selector0,exe(selector1),exe(selector2))):
											yield i
											if D: self.debug("appended")
										if D: self.debug("discarded")
									except Exception:
										if D: self.debug("discarded")

						if D: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
						return exeSelector(fst)
					self.current=fst
					snd=exe(node[2])
					if isinstance(fst, tuple([tuple]+ITER_TYPES+STR_TYPES)):
						# nodes[N]
						if isinstance(snd, tuple(NUM_TYPES)) or isinstance(snd, str) and snd.isdigit():
							n=int(snd)
							if D: self.info("getting %sth element from '%s'", color.bold(n), color.bold(fst))
							if isinstance(fst, (generator,chain)):
								if n>0:
									return skip(fst,n)
								elif n==0:
									return next(fst)
								else:
									fst=list(fst)
							else:
								try:
									return fst[n]
								except (IndexError, TypeError):
									return None
						# $.*['string']==$.string
						if isinstance(snd, tuple(STR_TYPES)):
							return exe((".",fst,snd))
						else:
							# $.*[@.string] - bad syntax, but allowed
							return snd
					else:
						try:
							if D: self.debug("returning %s", color.bold(fst[snd]))
							return fst[snd]
						except KeyError:
							# CHECK - is it ok to do that or should it be ProgrammingError?
							if D: self.debug("returning an empty list")
							return []
				raise ProgrammingError("Wrong usage of "+color.bold("[")+" operator")
			elif op=="fn":
				# Built-in functions
				fnName=node[1]
				args=None
				try:
					args=[exe(x) for x in node[2:]]
				except IndexError:
					if D: self.debug("NOT ERROR: can't map '%s' with '%s'",node[2:],exe)
				# arithmetic
				if fnName=="sum":
					args=args[0]
					if isinstance(args, tuple(NUM_TYPES)):
						return args
					return sum((x for x in args if isinstance(x, tuple(NUM_TYPES))))
				elif fnName=="max":
					args=args[0]
					if isinstance(args, tuple(NUM_TYPES)):
						return args
					return max((x for x in args if isinstance(x, tuple(NUM_TYPES))))
				elif fnName=="min":
					args=args[0]
					if isinstance(args, tuple(NUM_TYPES)):
						return args
					return min((x for x in args if isinstance(x, tuple(NUM_TYPES))))
				elif fnName=="avg":
					args=args[0]
					if isinstance(args, tuple(NUM_TYPES)):
						return args
					if not isinstance(args, tuple(ITER_TYPES)):
						raise Exception("Argument for avg() is not an array")
					else:
						args=list(args)
					try:
						return sum(args)/float(len(args))
					except TypeError:
						args=[x for x in args if isinstance(x, tuple(NUM_TYPES))]
						self.warning("Some items in array were ommited")
						return sum(args)/float(len(args))
				elif fnName=="round":
					return round(*args)
				# casting
				elif fnName=="int":
					return int(args[0])
				elif fnName=="float":
					return float(args[0])
				elif fnName=="str":
					return str(py2JSON(args[0]))
				elif fnName in ("list","array"):
					try:
						a=args[0]
					except IndexError:
						return []
					if isinstance(a, timeutils.datetime.datetime):
						return timeutils.date2list(a)+timeutils.time2list(a)
					if isinstance(a, timeutils.datetime.date):
						return timeutils.date2list(a)
					if isinstance(a, timeutils.datetime.time):
						return timeutils.time2list(a)
					return list(a)
				# string
				elif fnName=="upper":
					return args[0].upper()
				elif fnName=="lower":
					return args[0].lower()
				elif fnName=="capitalize":
					return args[0].capitalize()
				elif fnName=="title":
					return args[0].title()
				elif fnName=="split":
					return args[0].split(*args[1:])
				elif fnName=="slice":
					if args and not isinstance(args[1], tuple(ITER_TYPES)):
						raise ExecutionError("Wrong usage of slice(STRING, ARRAY). Second argument is not an array but %s."%color.bold(type(args[1]).__name__))
					try:
						pos=list(args[1])
						if isinstance(pos[0], tuple(ITER_TYPES)):
							if D: self.debug("run slice() for a list of slicers")
							return (args[0][x[0]:x[1]] for x in pos)
						return args[0][pos[0]:pos[1]]
					except IndexError:
						if len(args)!=2:
							raise ProgrammingError("Wrong usage of slice(STRING, ARRAY). Provided %s argument, should be exactly 2."%len(args))
				elif fnName=="escape":
					global escape,escapeDict
					if not escape:
						from objectpath.utils import escape, escapeDict
					return escape(args[0],escapeDict)
				elif fnName=="unescape":
					global unescape,unescapeDict
					if not unescape:
						from objectpath.utils import unescape, unescapeDict
					return unescape(args[0],unescapeDict)
				elif fnName=="replace":
					if sys.version_info.major < 3 and isinstance(args[0], unicode):
						args[0]=args[0].encode("utf8")
					return str.replace(args[0],args[1],args[2])
				# TODO this should be supported by /regex/
				# elif fnName=="REsub":
				# 	return re.sub(args[1],args[2],args[0])
				elif fnName=="sort":
					if len(args)>1:
						key=args[1]
						a={"key":lambda x: x.get(key, 0)}
					else:
						a={}
					args=args[0]
					if D: self.debug("doing sort on '%s'",args)
					try:
						return sorted(args,**a)
					except TypeError:
						return args
				elif fnName=="reverse":
					args=args[0]
					try:
						args.reverse()
						return args
					except TypeError:
						return args
				elif fnName=="map":
					return map(lambda x: exe(("fn",args[0],x)), args[1])
				elif fnName in ("count","len"):
					args=args[0]
					if args in (True,False,None):
						return args
					if isinstance(args, tuple(ITER_TYPES)):
						return len(list(args))
					return len(args)
				elif fnName=="join":
					try:
						joiner=args[1]
					except Exception:
						joiner=""
					try:
						return joiner.join(args[0])
					except TypeError:
						try:
							return joiner.join(map(str,args[0]))
						except Exception:
							return args[0]
				# time
				elif fnName in ("now","age","time","date","dateTime"):
					if fnName=="now":
						return timeutils.now()
					if fnName=="date":
						return timeutils.date(args)
					if fnName=="time":
						return timeutils.time(args)
					if fnName=="dateTime":
						return timeutils.dateTime(args)
					# TODO move lang to localize() entirely!
					if fnName=="age":
						a={}
						if len(args)>1:
							a["reference"]=args[1]
						if len(args)>2:
							a["lang"]=args[2]
						return list(timeutils.age(args[0],**a))
				elif fnName=="toMillis":
					args=args[0]
					if args.utcoffset() is not None:
						args=args-args.utcoffset() # pylint: disable=E1103
					global calendar
					if not calendar:
						import calendar
					return int(calendar.timegm(args.timetuple()) * 1000 + args.microsecond / 1000)
				elif fnName=="localize":
					if isinstance(args[0], timeutils.datetime.datetime):
						return timeutils.UTC2local(*args)
				# polygons
				elif fnName=="area":
					def segments(p):
						p=list(map(lambda x: x[0:2],p))
						return zip(p, p[1:] + [p[0]])
					return 0.5 * abs(sum(x0*y1 - x1*y0
						for ((x0, y0), (x1, y1)) in segments(args[0])))
				# misc
				elif fnName=="keys":
					try:
						return list(args[0].keys())
					except AttributeError:
						raise ExecutionError("Argument is not "+color.bold("object")+" but %s in keys()"%color.bold(type(args[0]).__name__))
				elif fnName=="type":
					if isinstance(args[0], tuple(ITER_TYPES)):
						return "array"
					if isinstance(args[0], dict):
						return "object"
					return args[0].__class__.__name__
				elif fnName in self._REGISTERED_FUNCTIONS:
					return self._REGISTERED_FUNCTIONS[fnName](*args)
				else:
					raise ProgrammingError("Function "+color.bold(fnName)+" does not exist.")
			else:
				return node
Example #3
0
    def _execute_subscript(self, node):
        D = self.D
        len_node = len(node)
        # TODO move it to tree generation phase
        if len_node is 1:  # empty list
            if D: self.debug("returning an empty list")
            return []
        if len_node is 2:  # list - preserved to catch possible event of leaving it as '[' operator
            if D: self.debug("doing list mapping")
            return [self._execute_node(x) for x in node[1]]
        if len_node is 3:  # selector used []
            fst = self._execute_node(node[1])
            # check against None
            if not fst:
                return fst
            selector = node[2]
            if D:
                self.debug("\n    found selector '%s'.\n    executing on %s",
                           color.bold(selector), color.bold(fst))
            selectorIsTuple = type(selector) is tuple

            if selectorIsTuple and selector[0] is "[":
                nodeList = []
                nodeList_append = nodeList.append
                for i in fst:
                    if D:
                        self.debug("setting self.current to %s", color.bold(i))
                    self.current = i
                    nodeList_append(
                        self._execute_node(
                            (selector[0], self._execute_node(selector[1]),
                             self._execute_node(selector[2]))))
                if D:
                    self.debug("returning %s objects: %s",
                               color.bold(len(nodeList)), color.bold(nodeList))
                return nodeList

            if selectorIsTuple and selector[0] == "(current)":
                if D:
                    self.warning(
                        color.bold("$.*[@]") + " is eqivalent to " +
                        color.bold("$.*") + "!")
                return fst

            if selectorIsTuple and selector[0] in SELECTOR_OPS:
                if D:
                    self.debug("found %s operator in selector, %s",
                               color.bold(selector[0]), color.bold(selector))
                if type(fst) is dict:
                    fst = [fst]
                # TODO move it to tree building phase
                if type(selector[1]) is tuple and selector[1][0] == "name":
                    selector = (selector[0], selector[1][1], selector[2])
                selector0 = selector[0]
                selector1 = selector[1]
                selector2 = selector[2]

                def exeSelector(fst):
                    for i in fst:
                        if D:
                            self.debug("setting self.current to %s",
                                       color.bold(i))
                            self.debug(
                                "    s0: %s\n    s1: %s\n    s2: %s\n    Current: %s",
                                selector0, selector1, selector2, i)
                        self.current = i
                        if selector0 == "fn":
                            yield self._execute_node(selector)
                        # elif type(selector1) in STR_TYPES and False:
                        # 	if D: self.debug("found string %s", type(i))
                        # 	try:
                        # 		if self._execute_node((selector0,i[selector1],selector2)):
                        # 			yield i
                        # 			if D: self.debug("appended")
                        # 		if D: self.debug("discarded")
                        # 	except Exception as e:
                        # 		if D: self.debug("discarded, Exception: %s",color.bold(e))
                        else:
                            try:
                                # TODO optimize an event when @ is not used. self._execute_node(selector1) can be cached
                                if self._execute_node(
                                    (selector0, self._execute_node(selector1),
                                     self._execute_node(selector2))):
                                    yield i
                                    if D: self.debug("appended %s", i)
                                elif D: self.debug("discarded")
                            except Exception:
                                if D: self.debug("discarded")

                if type(fst) == generator:
                    for i in list(fst):
                        return exeSelector(i)
                else:
                    # if D and nodeList: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
                    return exeSelector(fst)
            self.current = fst
            snd = self._execute_node(node[2])
            typefst = type(fst)
            if typefst in [tuple] + ITER_TYPES + STR_TYPES:
                typesnd = type(snd)
                # nodes[N]
                if typesnd in NUM_TYPES or typesnd is str and snd.isdigit():
                    n = int(snd)
                    if D:
                        self.info("getting %sth element from '%s'",
                                  color.bold(n), color.bold(fst))
                    if typefst in (generator, chain):
                        if n > 0:
                            return skip(fst, n)
                        elif n == 0:
                            return next(fst)
                        else:
                            fst = list(fst)
                    else:
                        try:
                            return fst[n]
                        except (IndexError, TypeError):
                            return None
                # $.*['string']==$.string
                if type(snd) in STR_TYPES:
                    return self._execute_node((".", fst, snd))
                else:
                    # $.*[@.string] - bad syntax, but allowed
                    return snd
            else:
                try:
                    if D: self.debug("returning %s", color.bold(fst[snd]))
                    return fst[snd]
                except KeyError:
                    # CHECK - is it ok to do that or should it be ProgrammingError?
                    if D: self.debug("returning an empty list")
                    return []
        raise ProgrammingError("Wrong usage of " + color.bold("[") +
                               " operator")
Example #4
0
    def exe(self, node):
        """
            node[0] - operator name
            node[1:] - params
        """
        types = [str, timeutils.datetime.time, timeutils.datetime.date, timeutils.datetime.datetime]
        try:
            types += [unicode]
        except:
            pass
        if self.D:
            self.start("executing node '%s'", node)
        type_node = type(node)
        if node is None or type_node in TYPES:
            return node
        elif type_node in types:
            return node
        elif type_node is list:
            return (self.exe(n) for n in node)
        elif type_node is dict:
            ret = {}
            for i in node.items():
                ret[self.exe(i[0])] = self.exe(i[1])
            return ret
        op = node[0]
        if op in OP_TOKENS.keys():
            return self.lexer[op](*node)
        elif op == "(root)":  # this is $
            return self.data
        # elif op=="(node)":# this is !
        #     if D: self.debug("returning node %s",self.node)
        #     return self.node
        elif op == "(current)":  # this is @
            if self.D:
                self.debug("returning current node %s", self.current)
            return self.current
        elif op == "name":
            return node[1]
        elif op == ".":
            fst = node[1]
            if type(fst) is tuple:
                fst = self.exe(fst)
            typefst = type(fst)
            if self.D:
                self.debug(color.op(".") + " left is '%s'", fst)
            # try:
            if node[2][0] == "*":
                if self.D:
                    self.end(color.op(".") + " returning '%s'", typefst in ITER_TYPES and fst or [fst])
                return fst  # typefst in ITER_TYPES and fst or [fst]
            # except:
            #     pass
            snd = self.exe(node[2])
            if self.D:
                self.debug(color.op(".") + " right is '%s'", snd)
            if typefst in ITER_TYPES:
                if self.D:
                    self.debug(color.op(".") + " filtering %s by %s", color.bold(fst), color.bold(snd))
                if type(snd) in ITER_TYPES:
                    return filter_dict(fst, list(snd))
                else:
                    # if D: self.debug(list(fst))
                    return (e[snd] for e in fst if type(e) is dict and snd in e)
            try:
                if self.D:
                    self.end(color.op(".") + " returning '%s'", fst.get(snd))
                return fst.get(snd)
            except Exception:
                if isinstance(fst, object):
                    return self.object_getter(fst, snd)
                if self.D:
                    self.end(color.op(".") + " returning '%s'", color.bold(fst))
                return fst
        elif op == "..":
            fst = flatten(self.exe(node[1]))
            if node[2][0] == "*":
                if self.D:
                    self.debug(color.op("..") + " returning '%s'", color.bold(fst))
                return fst
            # reduce objects to selected attributes
            snd = self.exe(node[2])
            if self.D:
                self.debug(color.op("..") + " finding all %s in %s", color.bold(snd), color.bold(fst))
            if type(snd) in ITER_TYPES:
                ret = filter_dict(fst, list(snd))
                if self.D:
                    self.debug(color.op("..") + " returning %s", color.bold(ret))
                return ret
            else:
                ret = chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))
                # print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
                if self.D:
                    self.debug(color.op("..") + " returning %s", color.bold(ret))
                return ret
        elif op == "[":
            len_node = len(node)
            # TODO move it to tree generation phase
            if len_node is 1:  # empty list
                if self.D:
                    self.debug("returning an empty list")
                return []
            if len_node is 2:  # list - preserved to catch possible event of leaving it as '[' operator
                if self.D:
                    self.debug("doing list mapping")
                return [self.exe(x) for x in node[1]]
            if len_node is 3:  # selector used []
                fst = self.exe(node[1])
                # check against None
                if not fst:
                    return fst
                selector = node[2]
                if self.D:
                    self.debug("found '%s' selector. executing on %s", color.bold(selector), color.bold(fst))
                selectorIsTuple = type(selector) is tuple

                if selectorIsTuple and selector[0] is "[":
                    nodeList = []
                    nodeList_append = nodeList.append
                    for i in fst:
                        if self.D:
                            self.debug("setting self.current to %s", color.bold(i))
                        self.current = i
                        nodeList_append(self.exe((selector[0], self.exe(selector[1]), self.exe(selector[2]))))
                    if self.D:
                        self.debug("returning %s objects: %s", color.bold(len(nodeList)), color.bold(nodeList))
                    return nodeList

                if selectorIsTuple and selector[0] == "(current)":
                    if self.D:
                        self.warning(color.bold("$.*[@]") + " is eqivalent to " + color.bold("$.*") + "!")
                    return fst

                if selectorIsTuple and selector[0] in SELECTOR_OPS:
                    if self.D:
                        self.debug("found %s operator in selector, %s", color.bold(selector[0]),
                                     color.bold(selector))
                    if type(fst) is dict:
                        fst = [fst]
                    # TODO move it to tree building phase
                    if type(selector[1]) is tuple and selector[1][0] == "name":
                        selector = (selector[0], selector[1][1], selector[2])
                    # if D and nodeList: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
                    return self.exeSelector(fst, selector)
                self.current = fst
                snd = self.exe(node[2])
                typefst = type(fst)
                if typefst in [tuple] + ITER_TYPES + STR_TYPES:
                    typesnd = type(snd)
                    # nodes[N]
                    if typesnd in NUM_TYPES or typesnd is str and snd.isdigit():
                        n = int(snd)
                        if self.D:
                            self.info("getting %sth element from '%s'", color.bold(n), color.bold(fst))
                        if typefst in (generator, chain):
                            if n > 0:
                                return skip(fst, n)
                            elif n == 0:
                                return next(fst)
                            else:
                                fst = list(fst)
                        else:
                            try:
                                return fst[n]
                            except (IndexError, TypeError):
                                return None
                    # $.*['string']==$.string
                    if type(snd) in STR_TYPES:
                        return self.exe((".", fst, snd))
                    else:
                        # $.*[@.string] - bad syntax, but allowed
                        return snd
                else:
                    try:
                        if self.D:
                            self.debug("returning %s", color.bold(fst[snd]))
                        return fst[snd]
                    except KeyError:
                        # CHECK - is it ok to do that or should it be ProgrammingError?
                        if self.D:
                            self.debug("returning an empty list")
                        return []
            raise ProgrammingError("Wrong usage of " + color.bold("[") + " operator")
        elif op == "fn":
            # Built-in functions
            fnName = node[1]
            args = None
            try:
                args = [self.exe(x) for x in node[2:]]
            except IndexError:
                if self.D:
                    self.debug("NOT ERROR: can't map '%s' with '%s'", node[2:], self.exe)
            if fnName in FN_TOKENS.keys():
                return self.functions[fnName](*args)
            elif fnName in self._REGISTERED_FUNCTIONS:
                return self._REGISTERED_FUNCTIONS[fnName](*args)
            else:
                raise ProgrammingError("Function " + color.bold(fnName) + " does not exist.")
        else:
            return node
Example #5
0
        def exe(node):
            """
				node[0] - operator name
				node[1:] - params
			"""
            if D: self.start("executing node '%s'", node)
            type_node = type(node)
            if node is None or type_node in TYPES:
                return node
            elif type_node is list:
                return list(map(exe, node))
            elif type_node is dict:
                ret = {}
                for i in node.items():
                    ret[exe(i[0])] = exe(i[1])
                return ret
            op = node[0]
            if op == "or":
                if D: self.debug("%s or %s", node[1], node[2])
                return exe(node[1]) or exe(node[2])
            elif op == "and":
                if D: self.debug("%s and %s", node[1], node[2])
                return exe(node[1]) and exe(node[2])
            elif op == "+":
                if len(node) > 2:
                    fst = exe(node[1])
                    snd = exe(node[2])
                    if fst is None:
                        return snd
                    if snd is None:
                        return fst
                    typefst = type(fst)
                    typesnd = type(snd)
                    if typefst is dict:
                        try:
                            fst.update(snd)
                        except:
                            if type(snd) is not dict:
                                raise ProgrammingError(
                                    "Can't add value of type %s to %s" %
                                    (bold(
                                        PY_TYPES_MAP.get(
                                            type(snd).__name__,
                                            type(snd).__name__)),
                                     bold("object")))
                        return fst
                    if typefst is list and typesnd is list:
                        if D:
                            self.debug("both sides are lists, returning '%s'",
                                       fst + snd)
                        return fst + snd
                    if typefst in ITER_TYPES or typesnd in ITER_TYPES:
                        if typefst not in ITER_TYPES:
                            fst = [fst]
                        elif typesnd not in ITER_TYPES:
                            snd = [snd]
                        if D:
                            self.debug(
                                "at least one side is generator and other is iterable, returning chain"
                            )
                        return chain(fst, snd)
                    if typefst in (int, float):
                        try:
                            return fst + snd
                        except:
                            return fst + float(snd)
                    if typefst in STR_TYPES or typesnd in STR_TYPES:
                        if D:
                            self.info("doing string comparison '%s' is '%s'",
                                      fst, snd)
                        if sys.version < "3":
                            if typefst is unicode:
                                fst = fst.encode("utf-8")
                            if typesnd is unicode:
                                snd = snd.encode("utf-8")
                        return str(fst) + str(snd)
                    try:
                        timeType = timeutils.datetime.time
                        if typefst is timeType and typesnd is timeType:
                            return timeutils.addTimes(fst, snd)
                    except:
                        pass
                    if D:
                        self.debug("standard addition, returning '%s'",
                                   fst + snd)
                    return fst + snd
                else:
                    return exe(node[1])
            elif op == "-":
                #TODO move -N to tree builder!
                if len(node) > 2:
                    fst = exe(node[1])
                    snd = exe(node[2])
                    try:
                        return fst - snd
                    except:
                        typefst = type(fst)
                        typesnd = type(snd)
                        timeType = timeutils.datetime.time
                        if typefst is timeType and typesnd is timeType:
                            return timeutils.subTimes(fst, snd)
                else:
                    return -exe(node[1])
            elif op == "*":
                return exe(node[1]) * exe(node[2])
            elif op == "%":
                return exe(node[1]) % exe(node[2])
            elif op == "/":
                #print(node[1])
                #print(exe(node[1]))
                return exe(node[1]) / float(exe(node[2]))
            elif op == ">":
                if D: self.debug("%s > %s", node[1], node[2])
                return exe(node[1]) > exe(node[2])
            elif op == "<":
                return exe(node[1]) < exe(node[2])
            elif op == ">=":
                return exe(node[1]) >= exe(node[2])
            elif op == "<=":
                return exe(node[1]) <= exe(node[2])
            #TODO this algorithm produces 3 for 1<2<3 and should be true
            #elif op in "<=>=":
            #	fst=exe(node[1])
            #	snd=exe(node[2])
            #	if op==">":
            #		return fst > snd and snd or False
            #	elif op=="<":
            #		return fst < snd and snd or False
            #	elif op==">=":
            #		return fst >= snd and snd or False
            #	elif op=="<=":
            #		return fst <= snd and snd or False
            elif op == "not":
                fst = exe(node[1])
                if D: self.debug("doing not '%s'", fst)
                return not fst
            elif op == "in":
                fst = exe(node[1])
                snd = exe(node[2])
                if D: self.debug("doing '%s' in '%s'", node[1], node[2])
                if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
                    return any(x in max(fst, snd, key=len)
                               for x in min(fst, snd, key=len))
                return exe(node[1]) in exe(node[2])
            elif op == "not in":
                fst = exe(node[1])
                snd = exe(node[2])
                if D: self.debug("doing '%s' not in '%s'", node[1], node[2])
                if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
                    return not any(x in max(fst, snd, key=len)
                                   for x in min(fst, snd, key=len))
                return exe(node[1]) not in exe(node[2])
            elif op in ("is", "is not"):
                if D: self.debug("found operator '%s'", op)
                try:
                    fst = exe(node[1])
                except Exception as e:
                    if D:
                        self.debug(
                            "NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",
                            node[1], str(e))
                    fst = node[1]
                try:
                    snd = exe(node[2])
                except Exception as e:
                    if D:
                        self.debug(
                            "NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",
                            node[2], str(e))
                    snd = node[2]
                if op is "is" and fst == snd:
                    return True
                if op is "is not" and fst != snd:
                    return True
                typefst = type(fst)
                typesnd = type(snd)
                if D:
                    self.debug("type fst: '%s', type snd: '%s'", typefst,
                               typesnd)
                if typefst in STR_TYPES:
                    if D:
                        self.info("doing string comparison '\"%s\" is \"%s\"'",
                                  fst, snd)
                    ret = fst == str(snd)
                elif typefst is float:
                    if D:
                        self.info("doing float comparison '%s is %s'", fst,
                                  snd)
                    ret = abs(fst - float(snd)) < EPSILON
                elif typefst is int:
                    if D:
                        self.info("doing integer comparison '%s is %s'", fst,
                                  snd)
                    ret = fst == int(snd)
                elif typefst is list and typesnd is list:
                    if D:
                        self.info("doing array comparison '%s' is '%s'", fst,
                                  snd)
                    ret = fst == snd
                elif typefst is dict and typesnd is dict:
                    if D:
                        self.info("doing object comparison '%s' is '%s'", fst,
                                  snd)
                    ret = fst == snd
                else:
                    try:
                        global ObjectId
                        if not ObjectId:
                            from bson.objectid import ObjectId
                        if typefst is ObjectId or typesnd is ObjectId:
                            if D:
                                self.info(
                                    "doing MongoDB objectID comparison '%s' is '%s'",
                                    fst, snd)
                            ret = str(fst) == str(snd)
                        else:
                            if D:
                                self.info(
                                    "doing standard comparison '%s' is '%s'",
                                    fst, snd)
                            ret = fst is snd
                    except:
                        pass
                if op == "is not":
                    if D: self.info("'is not' found. Returning %s", not ret)
                    return not ret
                else:
                    if D:
                        self.info("returning '%s' is '%s'='%s'", fst, snd, ret)
                    return ret
            elif op == "(literal)":
                fstLetter = node[1][0]
                if fstLetter is "'":
                    return node[1][1:-1]
                elif fstLetter.isdigit:
                    return int(node[1])
            elif op == "(root)":  # this is $
                return self.data
            elif op == "(node)":  # this is !
                if D: self.debug("returning node %s", self.node)
                return self.node
            elif op == "(current)":  # this is @
                if D: self.debug("returning current node %s", self.current)
                return self.current
            elif op == "name":
                return node[1]
            elif op == ".":
                fst = node[1]
                if type(fst) is tuple:
                    fst = exe(fst)
                typefst = type(fst)
                if D: self.debug("left is '%s'", fst)
                if node[2][0] == "*":
                    if D:
                        self.end("returning '%s'",
                                 typefst in ITER_TYPES and fst or [fst])
                    return typefst in ITER_TYPES and fst or [fst]
                snd = exe(node[2])
                if D: self.debug("right is '%s'", snd)
                if typefst in ITER_TYPES:
                    ret = []
                    ret_append = ret.append
                    for i in fst:
                        try:
                            ret_append(i[snd])
                        except:
                            pass
                    if D: self.end(". returning '%s'", ret)
                    return ret
                try:
                    if D: self.end(". returning '%s'", fst.get(snd))
                    return fst.get(snd)
                except:
                    if isinstance(fst, object):
                        try:
                            return fst.__getattribute__(snd)
                        except:
                            pass
                    if D: self.end(". returning '%s'", fst)
                    return fst
            elif op == "..":
                fst = dicttree.flatten(exe(node[1]))
                if node[2][0] == "*":
                    if D: self.debug("returning '%s'", fst)
                    return fst
                ret = []
                snd = exe(node[2])
                for i in fst:
                    try:
                        ret.append(i[snd])
                    except:
                        pass
                if D: self.debug("returning '%s'", ret)
                return len(ret) is 1 and ret[0] or ret
            #TODO move it to tree generation phase
            elif op == "{":
                return {}
            elif op == "[":
                len_node = len(node)
                #TODO move it to tree generation phase
                if len_node is 1:  # empty list
                    if D: self.debug("returning an empty list")
                    return []
                if len_node is 2:  # list - preserved to catch possible event of leaving it as '[' operator
                    #TODO yielding is not possible here
                    #if type(node[1]) in (generator,chain):
                    #	for i in node[1]:
                    #		yield exe(i)
                    if D: self.debug("doing list mapping")
                    return list(map(exe, node[1]))
                if len_node is 3:  # selector used []
                    fst = exe(node[1])
                    # check against None
                    if not fst:
                        return fst
                    selector = node[2]
                    if D:
                        self.debug("found '%s' selector for '%s'", selector,
                                   fst)

                    if type(selector) is tuple and selector[0] is "[":
                        nodeList = []
                        nodeList_append = nodeList.append
                        for i in fst:
                            if D: self.debug("setting self.current to '%s'", i)
                            self.current = i
                            nodeList_append(
                                exe((selector[0], exe(selector[1]),
                                     exe(selector[2]))))
                        if D:
                            self.debug("returning '%s' objects: '%s'",
                                       len(nodeList), nodeList)
                        return nodeList

                    #if type(selector) is tuple and selector[0]=="fn":
                    #	for i in fst:

                    if type(selector) is tuple and selector[0] == "(current)":
                        if D:
                            self.warning(
                                bold("$.*[@]") + " is eqivalent to " +
                                bold("$.*") + "!")
                        return fst

                    if type(selector) is tuple and selector[0] in SELECTOR_OPS:
                        if D:
                            self.debug("found '%s' operator in selector",
                                       selector[0])
                        nodeList = []
                        nodeList_append = nodeList.append
                        if type(fst) is dict:
                            fst = [fst]
                        for i in fst:
                            if D: self.debug("setting self.current to '%s'", i)
                            self.current = i
                            #TODO move it to tree building phase
                            if type(selector[1]
                                    ) is tuple and selector[1][0] == "name":
                                selector = (selector[0], selector[1][1],
                                            selector[2])
                            if selector[0] == "fn":
                                nodeList_append(exe(selector))
                            elif type(selector[1]) in STR_TYPES:
                                try:
                                    if exe((selector[0], i[selector[1]],
                                            selector[2])):
                                        nodeList_append(i)
                                        if D: self.debug("appended")
                                    if D: self.debug("discarded")
                                except Exception as e:
                                    if D:
                                        self.debug("discarded, Exception: %s",
                                                   e)
                            else:
                                try:
                                    #TODO optimize an event when @ is not used. exe(selector[1]) can be cached
                                    if exe((selector[0], exe(selector[1]),
                                            exe(selector[2]))):
                                        nodeList_append(i)
                                        if D: self.debug("appended")
                                    if D: self.debug("discarded")
                                except:
                                    if D: self.debug("discarded")
                        if D:
                            self.debug("returning '%s' objects: '%s'",
                                       len(nodeList), nodeList)
                        return nodeList
                    snd = exe(node[2])
                    typefst = type(fst)
                    if typefst in [tuple] + ITER_TYPES + STR_TYPES:
                        typesnd = type(snd)
                        # nodes[N]
                        if typesnd in NUM_TYPES or typesnd is str and snd.isdigit(
                        ):
                            n = int(snd)
                            if D:
                                self.debug("getting %sth element from '%s'", n,
                                           snd)
                            if typefst in (generator, chain):
                                if n > 0:
                                    return skip(fst, n)
                                elif n == 0:
                                    return list(next(fst))[0]
                                else:
                                    fst = list(fst)
                            else:
                                try:
                                    return fst[n]
                                except:
                                    return None
                        # $.*['string']==$.string
                        return exe((".", fst, snd))
                    else:
                        try:
                            if D: self.debug("returning '%s'", fst[snd])
                            return fst[snd]
                        except:
                            #CHECK - is it ok to do that or should it be ProgrammingError?
                            if D: self.debug("returning an empty list")
                            return []
                raise ProgrammingError("Wrong usage of the '[' operator")
            elif op == "fn":
                """ Built-in functions """
                fnName = node[1]
                args = None
                try:
                    args = list(map(exe, node[2:]))
                except IndexError as e:
                    if D:
                        self.debug("NOT ERROR: can't map '%s' with '%s'",
                                   node[2:], exe)
                #arithmetic
                if fnName == "sum":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    return sum(
                        map(lambda x: type(x) in NUM_TYPES and x or exe(x),
                            args))
                elif fnName == "max":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    return max(
                        map(lambda x: type(x) in NUM_TYPES and x or exe(x),
                            args))
                elif fnName == "min":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    return min(
                        map(lambda x: type(x) in NUM_TYPES and x or exe(x),
                            args))
                elif fnName == "avg":
                    args = args[0]
                    if type(args) in NUM_TYPES:
                        return args
                    if type(args) not in ITER_TYPES:
                        raise Exception("Argument for avg() is not iterable")
                    try:
                        return sum(args) / float(len(args))
                    except TypeError:
                        args = filter(lambda x: type(x) in NUM_TYPES, args)
                        self.warning("Some items in array were ommited")
                        return sum(args) / float(len(args))
                elif fnName == "round":
                    return round(*args)
                #casting
                elif fnName == "int":
                    return int(args[0])
                elif fnName == "float":
                    return float(args[0])
                elif fnName == "str":
                    return str(py2JSON(args[0]))
                elif fnName in ("list", "array"):
                    try:
                        a = args[0]
                    except:
                        return []
                    targs = type(a)
                    if targs is timeutils.datetime.datetime:
                        return timeutils.date2list(a) + timeutils.time2list(a)
                    if targs is timeutils.datetime.date:
                        return timeutils.date2list(a)
                    if targs is timeutils.datetime.time:
                        return timeutils.time2list(a)
                    return list(a)
                #string
                elif fnName == "escape":
                    global escape, escapeDict
                    if not escape:
                        from objectpath.utils.xmlextras import escape, escapeDict
                    return escape(args[0], escapeDict)
                elif fnName == "upper":
                    return args[0].upper()
                elif fnName == "lower":
                    return args[0].lower()
                elif fnName == "capitalize":
                    return args[0].capitalize()
                elif fnName == "title":
                    return args[0].title()
                elif fnName == "split":
                    return args[0].split(*args[1:])
                elif fnName == "unescape":
                    global unescape, unescapeDict
                    if not unescape:
                        from objectpath.utils.xmlextras import unescape, unescapeDict
                    return unescape(args[0], unescapeDict)
                elif fnName == "replace":
                    if sys.version < "3" and type(args[0]) is unicode:
                        args[0] = args[0].encode("utf8")
                    return str.replace(args[0], args[1], args[2])
                elif fnName == "REsub":
                    return re.sub(args[1], args[2], args[0])
                #array
                elif fnName == "sort":
                    if D: self.debug("doing sort on '%s'", args)
                    if not args:
                        return args
                    if type(args) in (generator, chain):
                        args = list(args)
                    if len(args) > 1:
                        key = args[1]
                        a = {"key": lambda x: x.get(key)}
                        args = args[0]
                    else:
                        a = {}
                        args = args[0]
                    if type(args) is not list:
                        return args
                    args.sort(**a)
                    return args
                elif fnName == "reverse":
                    args = args[0]
                    if type(args) in (generator, chain):
                        args = list(args)
                    args.reverse()
                    return args
                elif fnName in ("count", "len"):
                    args = args[0]
                    if args in (True, False, None):
                        return args
                    if type(args) in ITER_TYPES:
                        return len(list(args))
                    return len(args)
                elif fnName == "join":
                    try:
                        joiner = args[1]
                    except:
                        joiner = ""
                    try:
                        return joiner.join(args[0])
                    except:
                        try:
                            return joiner.join(map(str, args[0]))
                        except:
                            return args[0]
                #time
                elif fnName in ("now", "age", "time", "date", "dateTime"):
                    if fnName == "now":
                        return timeutils.now()
                    if fnName == "date":
                        return timeutils.date(args)
                    if fnName == "time":
                        return timeutils.time(args)
                    if fnName == "dateTime":
                        return timeutils.dateTime(args)
                    if fnName == "age":
                        return timeutils.age(args[0],
                                             len(args) > 1 and args[1] or "en")
                elif fnName == "toMillis":
                    args = args[0]
                    if args.utcoffset() is not None:
                        args = args - args.utcoffset()
                    global calendar
                    if not calendar:
                        import calendar
                    return int(
                        calendar.timegm(args.timetuple()) * 1000 +
                        args.microsnd / 1000)
                elif fnName == "localize":
                    if type(args[0]) is timeutils.datetime.datetime:
                        return timeutils.UTC2local(*args)
                #polygons
                elif fnName == "area":

                    def segments(p):
                        p = list(map(lambda x: x[0:2], p))
                        return zip(p, p[1:] + [p[0]])

                    return 0.5 * abs(
                        sum(x0 * y1 - x1 * y0
                            for ((x0, y0), (x1, y1)) in segments(args[0])))
                #misc
                elif fnName == "keys":
                    try:
                        return args[0].keys()
                    except AttributeError as e:
                        raise Exception("Argument is not " + bold("object") +
                                        " but %s in keys()" %
                                        bold(type(args[0]).__name__))
                elif fnName == "type":
                    ret = type(args[0])
                    if ret in ITER_TYPES:
                        return "array"
                    if ret is dict:
                        return "object"
                    return ret.__name__
                else:
                    raise ProgrammingError("Function '" + fnName +
                                           "' does not exist.")
            else:
                return node
Example #6
0
		def exe(node):
			"""
				node[0] - operator name
				node[1:] - params
			"""
			if D: self.start("executing node '%s'", node)
			type_node=type(node)
			if node is None or type_node in TYPES:
				return node
			elif type_node is list:
				return list(map(exe,node))
			elif type_node is dict:
				ret={}
				for i in node.items():
					ret[exe(i[0])]=exe(i[1])
				return ret
			op=node[0]
			if op=="or":
				if D: self.debug("%s or %s", node[1],node[2])
				return exe(node[1]) or exe(node[2])
			elif op=="and":
				if D: self.debug("%s and %s", node[1],node[2])
				return exe(node[1]) and exe(node[2])
			elif op=="+":
				if len(node)>2:
					fst=exe(node[1])
					snd=exe(node[2])
					if fst is None:
						return snd
					if snd is None:
						return fst
					typefst=type(fst)
					typesnd=type(snd)
					if typefst is dict:
						try:
							fst.update(snd)
						except:
							if type(snd) is not dict:
								raise ProgrammingError("Can't add value of type %s to %s" % (bold(PY_TYPES_MAP.get(type(snd).__name__,type(snd).__name__)), bold("object")))
						return fst
					if typefst is list and typesnd is list:
						if D: self.debug("both sides are lists, returning '%s'",fst+snd)
						return fst+snd
					if typefst in ITER_TYPES or typesnd in ITER_TYPES:
						if typefst not in ITER_TYPES:
							fst=[fst]
						elif typesnd not in ITER_TYPES:
							snd=[snd]
						if D: self.debug("at least one side is generator and other is iterable, returning chain")
						return chain(fst,snd)
					if typefst in (int,float):
						try:
							return fst+snd
						except:
							return fst+float(snd)
					if typefst in STR_TYPES or typesnd in STR_TYPES:
						if D: self.info("doing string comparison '%s' is '%s'",fst,snd)
						if sys.version < "3":
							if typefst is unicode:
								fst=fst.encode("utf-8")
							if typesnd is unicode:
								snd=snd.encode("utf-8")
						return str(fst)+str(snd)
					try:
						timeType=timeutils.datetime.time
						if typefst is timeType and typesnd is timeType:
							return timeutils.addTimes(fst,snd)
					except:
						pass
					if D: self.debug("standard addition, returning '%s'",fst+snd)
					return fst + snd
				else:
					return exe(node[1])
			elif op=="-":
				#TODO move -N to tree builder!
				if len(node)>2:
					fst=exe(node[1])
					snd=exe(node[2])
					try:
						return fst-snd
					except:
						typefst=type(fst)
						typesnd=type(snd)
						timeType=timeutils.datetime.time
						if typefst is timeType and typesnd is timeType:
							return timeutils.subTimes(fst,snd)
				else:
					return - exe(node[1])
			elif op=="*":
				return exe(node[1]) * exe(node[2])
			elif op=="%":
				return exe(node[1]) % exe(node[2])
			elif op=="/":
				#print(node[1])
				#print(exe(node[1]))
				return exe(node[1]) / float(exe(node[2]))
			elif op==">":
				if D: self.debug("%s > %s", node[1],node[2])
				return exe(node[1]) > exe(node[2])
			elif op=="<":
				return exe(node[1]) < exe(node[2])
			elif op==">=":
				return exe(node[1]) >= exe(node[2])
			elif op=="<=":
				return exe(node[1]) <= exe(node[2])
			#TODO this algorithm produces 3 for 1<2<3 and should be true
			#elif op in "<=>=":
			#	fst=exe(node[1])
			#	snd=exe(node[2])
			#	if op==">":
			#		return fst > snd and snd or False
			#	elif op=="<":
			#		return fst < snd and snd or False
			#	elif op==">=":
			#		return fst >= snd and snd or False
			#	elif op=="<=":
			#		return fst <= snd and snd or False
			elif op=="not":
				fst=exe(node[1])
				if D: self.debug("doing not '%s'",fst)
				return not fst
			elif op=="in":
				fst=exe(node[1])
				snd=exe(node[2])
				if D: self.debug("doing '%s' in '%s'",node[1],node[2])
				if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
					return any(x in max(fst,snd,key=len) for x in min(fst,snd,key=len))
				return exe(node[1]) in exe(node[2])
			elif op=="not in":
				fst=exe(node[1])
				snd=exe(node[2])
				if D: self.debug("doing '%s' not in '%s'",node[1],node[2])
				if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
					return not any(x in max(fst,snd,key=len) for x in min(fst,snd,key=len))
				return exe(node[1]) not in exe(node[2])
			elif op in ("is","is not"):
				if D: self.debug("found operator '%s'",op)
				try:
					fst=exe(node[1])
				except Exception as e:
					if D: self.debug("NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",node[1],str(e))
					fst=node[1]
				try:
					snd=exe(node[2])
				except Exception as e:
					if D: self.debug("NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",node[2],str(e))
					snd=node[2]
				if op is "is" and fst == snd:
					return True
				if op is "is not" and fst != snd:
					return True
				typefst=type(fst)
				typesnd=type(snd)
				if D: self.debug("type fst: '%s', type snd: '%s'",typefst,typesnd)
				if typefst in STR_TYPES:
					if D: self.info("doing string comparison '\"%s\" is \"%s\"'",fst,snd)
					ret=fst==str(snd)
				elif typefst is float:
					if D: self.info("doing float comparison '%s is %s'",fst,snd)
					ret=abs(fst-float(snd))<EPSILON
				elif typefst is int:
					if D: self.info("doing integer comparison '%s is %s'",fst,snd)
					ret=fst==int(snd)
				elif typefst is list and typesnd is list:
					if D: self.info("doing array comparison '%s' is '%s'",fst,snd)
					ret=fst==snd
				elif typefst is dict and typesnd is dict:
					if D: self.info("doing object comparison '%s' is '%s'",fst,snd)
					ret=fst==snd
				else:
					try:
						global ObjectId
						if not ObjectId:
							from bson.objectid import ObjectId
						if typefst is ObjectId or typesnd is ObjectId:
							if D: self.info("doing MongoDB objectID comparison '%s' is '%s'",fst,snd)
							ret=str(fst)==str(snd)
						else:
							if D: self.info("doing standard comparison '%s' is '%s'",fst,snd)
							ret=fst is snd
					except:
						pass
				if op=="is not":
					if D: self.info("'is not' found. Returning %s",not ret)
					return not ret
				else:
					if D: self.info("returning '%s' is '%s'='%s'",fst,snd,ret)
					return ret
			elif op=="(literal)":
				fstLetter=node[1][0]
				if fstLetter is "'":
					return node[1][1:-1]
				elif fstLetter.isdigit:
					return int(node[1])
			elif op=="(root)":# this is $
				return self.data
			elif op=="(node)":# this is !
				if D: self.debug("returning node %s",self.node)
				return self.node
			elif op=="(current)":# this is @
				if D: self.debug("returning current node %s",self.current)
				return self.current
			elif op=="name":
				return node[1]
			elif op==".":
				fst=node[1]
				if type(fst) is tuple:
					fst=exe(fst)
				typefst=type(fst)
				if D: self.debug("left is '%s'",fst)
				if node[2][0] == "*":
					if D: self.end("returning '%s'",typefst in ITER_TYPES and fst or [fst])
					return typefst in ITER_TYPES and fst or [fst]
				snd=exe(node[2])
				if D: self.debug("right is '%s'",snd)
				if typefst in ITER_TYPES:
					ret=[]
					ret_append=ret.append
					for i in fst:
						try:
							ret_append(i[snd])
						except:
							pass
					if D: self.end(". returning '%s'",ret)
					return ret
				try:
					if D: self.end(". returning '%s'",fst.get(snd))
					return fst.get(snd)
				except:
					if isinstance(fst,object):
						try:
							return fst.__getattribute__(snd)
						except:
							pass
					if D: self.end(". returning '%s'",fst)
					return fst
			elif op=="..":
				fst=dicttree.flatten(exe(node[1]))
				if node[2][0]=="*":
					if D: self.debug("returning '%s'",fst)
					return fst
				ret=[]
				snd=exe(node[2])
				for i in fst:
					try:
						ret.append(i[snd])
					except:
						pass
				if D: self.debug("returning '%s'",ret)
				return len(ret) is 1 and ret[0] or ret
			#TODO move it to tree generation phase
			elif op=="{":
				return {}
			elif op=="[":
				len_node=len(node)
				#TODO move it to tree generation phase
				if len_node is 1: # empty list
					if D: self.debug("returning an empty list")
					return []
				if len_node is 2: # list - preserved to catch possible event of leaving it as '[' operator
					#TODO yielding is not possible here
					#if type(node[1]) in (generator,chain):
					#	for i in node[1]:
					#		yield exe(i)
					if D: self.debug("doing list mapping")
					return list(map(exe,node[1]))
				if len_node is 3: # selector used []
					fst=exe(node[1])
					# check against None
					if not fst:
						return fst
					selector=node[2]
					if D: self.debug("found '%s' selector for '%s'",selector,fst)

					if type(selector) is tuple and selector[0] is "[":
						nodeList=[]
						nodeList_append=nodeList.append
						for i in fst:
							if D: self.debug("setting self.current to '%s'",i)
							self.current=i
							nodeList_append(exe((selector[0],exe(selector[1]),exe(selector[2]))))
						if D: self.debug("returning '%s' objects: '%s'",len(nodeList),nodeList)
						return nodeList

					#if type(selector) is tuple and selector[0]=="fn":
					#	for i in fst:

					if type(selector) is tuple and selector[0] == "(current)":
						if D: self.warning(bold("$.*[@]")+" is eqivalent to "+bold("$.*")+"!")
						return fst

					if type(selector) is tuple and selector[0] in SELECTOR_OPS:
						if D: self.debug("found '%s' operator in selector",selector[0])
						nodeList=[]
						nodeList_append=nodeList.append
						if type(fst) is dict:
							fst=[fst]
						for i in fst:
							if D: self.debug("setting self.current to '%s'",i)
							self.current=i
							#TODO move it to tree building phase
							if type(selector[1]) is tuple and selector[1][0]=="name":
								selector=(selector[0],selector[1][1],selector[2])
							if selector[0]=="fn":
								nodeList_append(exe(selector))
							elif type(selector[1]) in STR_TYPES:
								try:
									if exe((selector[0],i[selector[1]],selector[2])):
										nodeList_append(i)
										if D: self.debug("appended")
									if D: self.debug("discarded")
								except Exception as e:
									if D: self.debug("discarded, Exception: %s",e)
							else:
								try:
									#TODO optimize an event when @ is not used. exe(selector[1]) can be cached
									if exe((selector[0],exe(selector[1]),exe(selector[2]))):
										nodeList_append(i)
										if D: self.debug("appended")
									if D: self.debug("discarded")
								except:
									if D: self.debug("discarded")
						if D: self.debug("returning '%s' objects: '%s'",len(nodeList),nodeList)
						return nodeList
					snd=exe(node[2])
					typefst=type(fst)
					if typefst in [tuple]+ITER_TYPES+STR_TYPES:
						typesnd=type(snd)
						# nodes[N]
						if typesnd in NUM_TYPES or typesnd is str and snd.isdigit():
							n=int(snd)
							if D: self.debug("getting %sth element from '%s'",n,snd)
							if typefst in (generator,chain):
								if n>0:
									return skip(fst,n)
								elif n==0:
									return list(next(fst))[0]
								else:
									fst=list(fst)
							else:
								try:
									return fst[n]
								except:
									return None
						# $.*['string']==$.string
						return exe((".",fst,snd))
					else:
						try:
							if D: self.debug("returning '%s'",fst[snd])
							return fst[snd]
						except:
							#CHECK - is it ok to do that or should it be ProgrammingError?
							if D: self.debug("returning an empty list")
							return []
				raise ProgrammingError("Wrong usage of the '[' operator")
			elif op=="fn":
				""" Built-in functions """
				fnName=node[1]
				args=None
				try:
					args=list(map(exe,node[2:]))
				except IndexError as e:
					if D: self.debug("NOT ERROR: can't map '%s' with '%s'",node[2:],exe)
				#arithmetic
				if fnName=="sum":
					args=args[0]
					if type(args) in NUM_TYPES:
						return args
					return sum(map(lambda x:type(x) in NUM_TYPES and x or exe(x), args))
				elif fnName=="max":
					args=args[0]
					if type(args) in NUM_TYPES:
						return args
					return max(map(lambda x:type(x) in NUM_TYPES and x or exe(x), args))
				elif fnName=="min":
					args=args[0]
					if type(args) in NUM_TYPES:
						return args
					return min(map(lambda x:type(x) in NUM_TYPES and x or exe(x), args))
				elif fnName=="avg":
					args=args[0]
					if type(args) in NUM_TYPES:
						return args
					if type(args) not in ITER_TYPES:
						raise Exception("Argument for avg() is not iterable")
					try:
						return sum(args)/float(len(args))
					except TypeError:
						args=filter(lambda x: type(x) in NUM_TYPES, args)
						self.warning("Some items in array were ommited")
						return sum(args)/float(len(args))
				elif fnName=="round":
					return round(*args)
				#casting
				elif fnName=="int":
					return int(args[0])
				elif fnName=="float":
					return float(args[0])
				elif fnName=="str":
					return str(py2JSON(args[0]))
				elif fnName in ("list","array"):
					try:
						a=args[0]
					except:
						return []
					targs=type(a)
					if targs is timeutils.datetime.datetime:
						return timeutils.date2list(a)+timeutils.time2list(a)
					if targs is timeutils.datetime.date:
						return timeutils.date2list(a)
					if targs is timeutils.datetime.time:
						return timeutils.time2list(a)
					return list(a)
				#string
				elif fnName=="escape":
					global escape,escapeDict
					if not escape:
						from objectpath.utils.xmlextras import escape, escapeDict
					return escape(args[0],escapeDict)
				elif fnName=="upper":
					return args[0].upper()
				elif fnName=="lower":
					return args[0].lower()
				elif fnName=="capitalize":
					return args[0].capitalize()
				elif fnName=="title":
					return args[0].title()
				elif fnName=="split":
					return args[0].split(*args[1:])
				elif fnName=="unescape":
					global unescape,unescapeDict
					if not unescape:
						from objectpath.utils.xmlextras import unescape, unescapeDict
					return unescape(args[0],unescapeDict)
				elif fnName=="replace":
					if sys.version < "3" and type(args[0]) is unicode:
						args[0]=args[0].encode("utf8")
					return str.replace(args[0],args[1],args[2])
				elif fnName=="REsub":
					return re.sub(args[1],args[2],args[0])
				#array
				elif fnName=="sort":
					if D: self.debug("doing sort on '%s'",args)
					if not args:
						return args
					if type(args) in (generator,chain):
						args=list(args)
					if len(args)>1:
						key=args[1]
						a={"key":lambda x: x.get(key)}
						args=args[0]
					else:
						a={}
						args=args[0]
					if type(args) is not list:
						return args
					args.sort(**a)
					return args
				elif fnName=="reverse":
					args=args[0]
					if type(args) in (generator,chain):
						args=list(args)
					args.reverse()
					return args
				elif fnName in ("count","len"):
					args=args[0]
					if args in (True,False,None):
						return args
					if type(args) in ITER_TYPES:
						return len(list(args))
					return len(args)
				elif fnName=="join":
					try:
						joiner=args[1]
					except:
						joiner=""
					try:
						return joiner.join(args[0])
					except:
						try:
							return joiner.join(map(str,args[0]))
						except:
							return args[0]
				#time
				elif fnName in ("now","age","time","date","dateTime"):
					if fnName=="now":
						return timeutils.now()
					if fnName=="date":
						return timeutils.date(args)
					if fnName=="time":
						return timeutils.time(args)
					if fnName=="dateTime":
						return timeutils.dateTime(args)
					if fnName=="age":
						return timeutils.age(args[0],len(args)>1 and args[1] or "en")
				elif fnName=="toMillis":
					args=args[0]
					if args.utcoffset() is not None:
						args=args-args.utcoffset()
					global calendar
					if not calendar:
						import calendar
					return int(calendar.timegm(args.timetuple()) * 1000 + args.microsnd / 1000)
				elif fnName=="localize":
					if type(args[0]) is timeutils.datetime.datetime:
						return timeutils.UTC2local(*args)
				#polygons
				elif fnName=="area":
					def segments(p):
						p=list(map(lambda x: x[0:2],p))
						return zip(p, p[1:] + [p[0]])
					return 0.5 * abs(sum(x0*y1 - x1*y0
						for ((x0, y0), (x1, y1)) in segments(args[0])))
				#misc
				elif fnName=="keys":
					try:
						return args[0].keys()
					except AttributeError as e:
						raise Exception("Argument is not "+bold("object")+" but %s in keys()"%bold(type(args[0]).__name__))
				elif fnName=="type":
					ret=type(args[0])
					if ret in ITER_TYPES:
						return "array"
					if ret is dict:
						return "object"
					return ret.__name__
				else:
					raise ProgrammingError("Function '"+fnName+"' does not exist.")
			else:
				return node