def _execute_dot_dot(self, node):
D = self.D
fst = flatten(self._execute_node(node[1]))
if node[2][0] == "*":
if D:
self.debug(color.op("..") + " returning '%s'", color.bold(fst))
return fst
# reduce objects to selected attributes
snd = self._execute_node(node[2])
if D:
self.debug(
color.op("..") + " finding all %s in %s", color.bold(snd),
color.bold(self.cleanOutput(fst)))
if type(snd) in ITER_TYPES:
ret = filter_dict(fst, list(snd))
if D: self.debug(color.op("..") + " returning %s", color.bold(ret))
return ret
else:
ret = chain.from_iterable(
type(x) in ITER_TYPES and x or [x]
for x in (e[snd] for e in fst if snd in e))
# print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
if D:
self.debug(
color.op("..") + " returning %s",
color.bold(self.cleanOutput(ret)))
return ret
def exe(node):
"""
node[0] - operator name
node[1:] - params
"""
types = [
str, timeutils.datetime.time, timeutils.datetime.date,
timeutils.datetime.datetime
]
try:
types += [unicode]
except:
pass
if D:
self.start("executing node %s",
color.bold(self.cleanOutput(node)))
type_node = type(node)
if node is None or type_node in TYPES:
return node
elif type_node in types:
return node
elif type_node is list:
return (exe(n) for n in node)
elif type_node is dict:
ret = {}
for i in node.items():
ret[exe(i[0])] = exe(i[1])
return ret
op = node[0]
if op == "or":
if D: self.debug("%s or %s", node[1], node[2])
return exe(node[1]) or exe(node[2])
elif op == "and":
if D: self.debug("%s and %s", node[1], node[2])
return exe(node[1]) and exe(node[2])
elif op == "+":
if len(node) > 2:
fst = exe(node[1])
snd = exe(node[2])
if None in (fst, snd):
return fst or snd
typefst = type(fst)
typesnd = type(snd)
if typefst is dict:
try:
fst.update(snd)
except Exception:
if type(snd) is not dict:
raise ProgrammingError(
"Can't add value of type %s to %s" %
(color.bold(
PY_TYPES_MAP.get(
type(snd).__name__,
type(snd).__name__)),
color.bold("object")))
return fst
if typefst is list and typesnd is list:
if D:
self.debug("both sides are lists, returning '%s'",
fst + snd)
return fst + snd
if typefst in ITER_TYPES or typesnd in ITER_TYPES:
if typefst not in ITER_TYPES:
fst = [fst]
elif typesnd not in ITER_TYPES:
snd = [snd]
if D:
self.debug(
"at least one side is a generator and the other is an iterable, returning chain"
)
return chain(fst, snd)
if typefst in NUM_TYPES:
try:
return fst + snd
except Exception:
return fst + float(snd)
if typefst in STR_TYPES or typesnd in STR_TYPES:
if D:
self.info("doing string comparison '%s' is '%s'",
fst, snd)
if sys.version_info[0] < 3:
if typefst is unicode:
fst = fst.encode("utf-8")
if typesnd is unicode:
snd = snd.encode("utf-8")
return str(fst) + str(snd)
try:
timeType = timeutils.datetime.time
if typefst is timeType and typesnd is timeType:
return timeutils.addTimes(fst, snd)
except Exception:
pass
if D:
self.debug("standard addition, returning '%s'",
fst + snd)
return fst + snd
else:
return exe(node[1])
elif op == "-":
if len(node) > 2:
fst = exe(node[1])
snd = exe(node[2])
try:
return fst - snd
except Exception:
typefst = type(fst)
typesnd = type(snd)
timeType = timeutils.datetime.time
if typefst is timeType and typesnd is timeType:
return timeutils.subTimes(fst, snd)
else:
return -exe(node[1])
elif op == "*":
return exe(node[1]) * exe(node[2])
elif op == "%":
return exe(node[1]) % exe(node[2])
elif op == "/":
return exe(node[1]) / float(exe(node[2]))
elif op == ">":
if D:
self.debug("%s > %s, %s", node[1], node[2],
node[1] > node[2])
return exe(node[1]) > exe(node[2])
elif op == "<":
return exe(node[1]) < exe(node[2])
elif op == ">=":
return exe(node[1]) >= exe(node[2])
elif op == "<=":
return exe(node[1]) <= exe(node[2])
# TODO this algorithm produces 3 for 1<2<3 and should be true
# elif op in "<=>=":
# fst=exe(node[1])
# snd=exe(node[2])
# if op==">":
# return fst > snd and snd or False
# elif op=="<":
# return fst < snd and snd or False
# elif op==">=":
# return fst >= snd and snd or False
# elif op=="<=":
# return fst <= snd and snd or False
elif op == "not":
fst = exe(node[1])
if D: self.debug("doing not '%s'", fst)
return not fst
elif op == "in":
fst = exe(node[1])
snd = exe(node[2])
if D: self.debug("doing '%s' in '%s'", node[1], node[2])
if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
return any(x in max(fst, snd, key=len)
for x in min(fst, snd, key=len))
return exe(node[1]) in exe(node[2])
elif op == "not in":
fst = exe(node[1])
snd = exe(node[2])
if D: self.debug("doing '%s' not in '%s'", node[1], node[2])
if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES:
return not any(x in max(fst, snd, key=len)
for x in min(fst, snd, key=len))
return exe(node[1]) not in exe(node[2])
elif op in ("is", "is not"):
if D: self.debug("found operator '%s'", op)
# try:
fst = exe(node[1])
# except Exception as e:
# if D: self.debug("NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",node[1],str(e))
# fst=node[1]
# try:
snd = exe(node[2])
# except Exception as e:
# if D: self.debug("NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",node[2],str(e))
# snd=node[2]
if op == "is" and fst == snd:
return True
# this doesn't work for 3 is not '3'
# if op == "is not" and fst != snd:
# return True
typefst = type(fst)
typesnd = type(snd)
if D:
self.debug("type fst: '%s', type snd: '%s'", typefst,
typesnd)
if typefst in STR_TYPES:
if D:
self.info("doing string comparison '\"%s\" is \"%s\"'",
fst, snd)
ret = str(fst) == str(snd)
elif typefst is float or typesnd is float:
if D:
self.info("doing float comparison '%s is %s'", fst,
snd)
try:
ret = abs(float(fst) - float(snd)) < EPSILON
except:
ret = False
elif typefst is int or typesnd is int:
if D:
self.info("doing integer comparison '%s is %s'", fst,
snd)
try:
ret = int(fst) == int(snd)
except:
ret = False
elif typefst is list and typesnd is list:
if D:
self.info("doing array comparison '%s' is '%s'", fst,
snd)
ret = fst == snd
elif typefst is dict and typesnd is dict:
if D:
self.info("doing object comparison '%s' is '%s'", fst,
snd)
ret = fst == snd
elif fst is None or snd is None:
if fst is None and snd is None:
# this executes only for "is not"
ret = True
else:
ret = (fst or snd) is None
if D:
self.info("doing None comparison %s is %s = %s",
color.bold(fst), color.bold(snd),
color.bold(not not (fst or snd)))
else:
if D:
self.info("can't compare %s and %s. Returning False",
self.cleanOutput(fst), self.cleanOutput(snd))
ret = False
# else:
# try:
# global ObjectId
# if not ObjectId:
# from bson.objectid import ObjectId
# if typefst is ObjectId or typesnd is ObjectId:
# if D: self.info("doing MongoDB objectID comparison '%s' is '%s'",fst,snd)
# ret=str(fst)==str(snd)
# else:
# if D: self.info("doing standard comparison '%s' is '%s'",fst,snd)
# ret=fst is snd
# except Exception:
# pass
if op == "is not":
if D: self.info("'is not' found. Returning %s", not ret)
return not ret
else:
if D:
self.info("returning %s is %s => %s",
color.bold(self.cleanOutput(fst)),
color.bold(self.cleanOutput(snd)),
color.bold(ret))
return ret
elif op == "re":
return re.compile(exe(node[1]))
elif op == "matches":
fst = exe(node[1])
snd = exe(node[2])
if type(fst) not in STR_TYPES + [RE_TYPE]:
raise Exception(
"operator " + color.bold("matches") +
" expects regexp on the left. Example: 'a.*d' matches 'abcd'"
)
if type(snd) in ITER_TYPES:
for i in snd:
if not not re.match(fst, i):
return True
return False
else:
# regex matches string
return not not re.match(fst, snd)
# elif op=="(literal)":
# fstLetter=node[1][0]
# if fstLetter is "'":
# return node[1][1:-1]
# elif fstLetter.isdigit:
# return int(node[1])
elif op == "(root)": # this is $
return self.data
# elif op=="(node)":# this is !
# if D: self.debug("returning node %s",self.node)
# return self.node
elif op == "(current)": # this is @
if D:
self.debug("returning current node: \n %s",
color.bold(self.current))
return self.current
elif op == "name":
return node[1]
elif op == ".":
fst = node[1]
if type(fst) is tuple:
fst = exe(fst)
typefst = type(fst)
if D:
self.debug(
color.op(".") + " left is '%s'",
color.bold(self.cleanOutput(fst)))
# try:
if node[2][0] == "*":
if D:
self.end(
color.op(".") + " returning '%s'",
color.bold(typefst in ITER_TYPES and fst or [fst]))
return fst # typefst in ITER_TYPES and fst or [fst]
# except:
# pass
snd = exe(node[2])
if D:
self.debug(
color.op(".") + " right is '%s'", color.bold(snd))
if typefst in ITER_TYPES:
if D:
self.debug(
color.op(".") + " filtering %s by %s",
color.bold(self.cleanOutput(fst)), color.bold(snd))
if type(snd) in ITER_TYPES:
return filter_dict(fst, list(snd))
else:
# if D: self.debug(list(fst))
return (e[snd] for e in fst
if type(e) is dict and snd in e)
try:
if D:
self.end(
color.op(".") + " returning '%s'", fst.get(snd))
return fst.get(snd)
except Exception:
if isinstance(fst, object):
return self.object_getter(fst, snd)
if D:
self.end(
color.op(".") + " returning '%s'", color.bold(fst))
return fst
elif op == "..":
fst = flatten(exe(node[1]))
if node[2][0] == "*":
if D:
self.debug(
color.op("..") + " returning '%s'",
color.bold(fst))
return fst
# reduce objects to selected attributes
snd = exe(node[2])
if D:
self.debug(
color.op("..") + " finding all %s in %s",
color.bold(snd), color.bold(self.cleanOutput(fst)))
if type(snd) in ITER_TYPES:
ret = filter_dict(fst, list(snd))
if D:
self.debug(
color.op("..") + " returning %s", color.bold(ret))
return ret
else:
ret = chain(*(type(x) in ITER_TYPES and x or [x]
for x in (e[snd] for e in fst if snd in e)))
# print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
if D:
self.debug(
color.op("..") + " returning %s",
color.bold(self.cleanOutput(ret)))
return ret
elif op == "[":
len_node = len(node)
# TODO move it to tree generation phase
if len_node is 1: # empty list
if D: self.debug("returning an empty list")
return []
if len_node is 2: # list - preserved to catch possible event of leaving it as '[' operator
if D: self.debug("doing list mapping")
return [exe(x) for x in node[1]]
if len_node is 3: # selector used []
fst = exe(node[1])
# check against None
if not fst:
return fst
selector = node[2]
if D:
self.debug(
"\n found selector '%s'.\n executing on %s",
color.bold(selector), color.bold(fst))
selectorIsTuple = type(selector) is tuple
if selectorIsTuple and selector[0] is "[":
nodeList = []
nodeList_append = nodeList.append
for i in fst:
if D:
self.debug("setting self.current to %s",
color.bold(i))
self.current = i
nodeList_append(
exe((selector[0], exe(selector[1]),
exe(selector[2]))))
if D:
self.debug("returning %s objects: %s",
color.bold(len(nodeList)),
color.bold(nodeList))
return nodeList
if selectorIsTuple and selector[0] == "(current)":
if D:
self.warning(
color.bold("$.*[@]") + " is eqivalent to " +
color.bold("$.*") + "!")
return fst
if selectorIsTuple and selector[0] in SELECTOR_OPS:
if D:
self.debug("found %s operator in selector, %s",
color.bold(selector[0]),
color.bold(selector))
if type(fst) is dict:
fst = [fst]
# TODO move it to tree building phase
if type(selector[1]
) is tuple and selector[1][0] == "name":
selector = (selector[0], selector[1][1],
selector[2])
selector0 = selector[0]
selector1 = selector[1]
selector2 = selector[2]
def exeSelector(fst):
for i in fst:
if D:
self.debug("setting self.current to %s",
color.bold(i))
self.debug(
" s0: %s\n s1: %s\n s2: %s\n Current: %s",
selector0, selector1, selector2, i)
self.current = i
if selector0 == "fn":
yield exe(selector)
# elif type(selector1) in STR_TYPES and False:
# if D: self.debug("found string %s", type(i))
# try:
# if exe((selector0,i[selector1],selector2)):
# yield i
# if D: self.debug("appended")
# if D: self.debug("discarded")
# except Exception as e:
# if D: self.debug("discarded, Exception: %s",color.bold(e))
else:
try:
# TODO optimize an event when @ is not used. exe(selector1) can be cached
if exe((selector0, exe(selector1),
exe(selector2))):
yield i
if D: self.debug("appended %s", i)
elif D: self.debug("discarded")
except Exception:
if D: self.debug("discarded")
# if D and nodeList: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
return exeSelector(fst)
self.current = fst
snd = exe(node[2])
typefst = type(fst)
if typefst in [tuple] + ITER_TYPES + STR_TYPES:
typesnd = type(snd)
# nodes[N]
if typesnd in NUM_TYPES or typesnd is str and snd.isdigit(
):
n = int(snd)
if D:
self.info("getting %sth element from '%s'",
color.bold(n), color.bold(fst))
if typefst in (generator, chain):
if n > 0:
return skip(fst, n)
elif n == 0:
return next(fst)
else:
fst = list(fst)
else:
try:
return fst[n]
except (IndexError, TypeError):
return None
# $.*['string']==$.string
if type(snd) in STR_TYPES:
return exe((".", fst, snd))
else:
# $.*[@.string] - bad syntax, but allowed
return snd
else:
try:
if D:
self.debug("returning %s",
color.bold(fst[snd]))
return fst[snd]
except KeyError:
# CHECK - is it ok to do that or should it be ProgrammingError?
if D: self.debug("returning an empty list")
return []
raise ProgrammingError("Wrong usage of " + color.bold("[") +
" operator")
elif op == "fn":
# Built-in functions
fnName = node[1]
args = None
try:
args = [exe(x) for x in node[2:]]
except IndexError:
if D:
self.debug("NOT ERROR: can't map '%s' with '%s'",
node[2:], exe)
# arithmetic
if fnName == "sum":
args = args[0]
if type(args) in NUM_TYPES:
return args
return sum((x for x in args if type(x) in NUM_TYPES))
elif fnName == "max":
args = args[0]
if type(args) in NUM_TYPES:
return args
return max((x for x in args if type(x) in NUM_TYPES))
elif fnName == "min":
args = args[0]
if type(args) in NUM_TYPES:
return args
return min((x for x in args if type(x) in NUM_TYPES))
elif fnName == "avg":
args = args[0]
if type(args) in NUM_TYPES:
return args
if type(args) not in ITER_TYPES:
raise Exception("Argument for avg() is not an array")
else:
args = list(args)
try:
return sum(args) / float(len(args))
except TypeError:
args = [x for x in args if type(x) in NUM_TYPES]
self.warning("Some items in array were ommited")
return sum(args) / float(len(args))
elif fnName == "round":
return round(*args)
# casting
elif fnName == "int":
return int(args[0])
elif fnName == "float":
return float(args[0])
elif fnName == "str":
return str(py2JSON(args[0]))
elif fnName in ("list", "array"):
try:
a = args[0]
except IndexError:
return []
targs = type(a)
if targs is timeutils.datetime.datetime:
return timeutils.date2list(a) + timeutils.time2list(a)
if targs is timeutils.datetime.date:
return timeutils.date2list(a)
if targs is timeutils.datetime.time:
return timeutils.time2list(a)
return list(a)
# string
elif fnName == "upper":
return args[0].upper()
elif fnName == "lower":
return args[0].lower()
elif fnName == "capitalize":
return args[0].capitalize()
elif fnName == "title":
return args[0].title()
elif fnName == "split":
return args[0].split(*args[1:])
elif fnName == "slice":
if args and type(args[1]) not in ITER_TYPES:
raise ExecutionError(
"Wrong usage of slice(STRING, ARRAY). Second argument is not an array but %s."
% color.bold(type(args[1]).__name__))
try:
pos = list(args[1])
if type(pos[0]) in ITER_TYPES:
if D:
self.debug("run slice() for a list of slicers")
return (args[0][x[0]:x[1]] for x in pos)
return args[0][pos[0]:pos[1]]
except IndexError:
if len(args) != 2:
raise ProgrammingError(
"Wrong usage of slice(STRING, ARRAY). Provided %s argument, should be exactly 2."
% len(args))
elif fnName == "escape":
global escape, escapeDict
if not escape:
from objectpath.utils import escape, escapeDict
return escape(args[0], escapeDict)
elif fnName == "unescape":
global unescape, unescapeDict
if not unescape:
from objectpath.utils import unescape, unescapeDict
return unescape(args[0], unescapeDict)
elif fnName == "replace":
if sys.version_info[0] < 3 and type(args[0]) is unicode:
args[0] = args[0].encode("utf8")
return str.replace(args[0], args[1], args[2])
# TODO this should be supported by /regex/
# elif fnName=="REsub":
# return re.sub(args[1],args[2],args[0])
elif fnName == "sort":
if len(args) > 1:
key = args[1]
a = {"key": lambda x: x.get(key, 0)}
else:
a = {}
args = args[0]
if D: self.debug("doing sort on '%s'", args)
try:
return sorted(args, **a)
except TypeError:
return args
elif fnName == "reverse":
args = args[0]
try:
args.reverse()
return args
except TypeError:
return args
elif fnName == "unique":
try:
return list(set(args[0]))
except TypeError:
return args[0]
elif fnName == "map":
return chain(
*map(lambda x: exe(("fn", args[0], x)), args[1]))
elif fnName in ("count", "len"):
args = args[0]
if args in (True, False, None):
return args
if type(args) in ITER_TYPES:
return len(list(args))
return len(args)
elif fnName == "join":
try:
joiner = args[1]
except Exception:
joiner = ""
try:
return joiner.join(args[0])
except TypeError:
try:
return joiner.join(map(str, args[0]))
except Exception:
return args[0]
# time
elif fnName in ("now", "age", "time", "date", "dateTime"):
if fnName == "now":
return timeutils.now()
if fnName == "date":
return timeutils.date(args)
if fnName == "time":
return timeutils.time(args)
if fnName == "dateTime":
return timeutils.dateTime(args)
# TODO move lang to localize() entirely!
if fnName == "age":
a = {}
if len(args) > 1:
a["reference"] = args[1]
if len(args) > 2:
a["lang"] = args[2]
return list(timeutils.age(args[0], **a))
elif fnName == "toMillis":
args = args[0]
if args.utcoffset() is not None:
args = args - args.utcoffset() # pylint: disable=E1103
global calendar
if not calendar:
import calendar
return int(
calendar.timegm(args.timetuple()) * 1000 +
args.microsecond / 1000)
elif fnName == "localize":
if type(args[0]) is timeutils.datetime.datetime:
return timeutils.UTC2local(*args)
# polygons
elif fnName == "area":
def segments(p):
p = list(map(lambda x: x[0:2], p))
return zip(p, p[1:] + [p[0]])
return 0.5 * abs(
sum(x0 * y1 - x1 * y0
for ((x0, y0), (x1, y1)) in segments(args[0])))
# misc
elif fnName == "keys":
try:
return list(args[0].keys())
except AttributeError:
raise ExecutionError(
"Argument is not " + color.bold("object") +
" but %s in keys()" %
color.bold(type(args[0]).__name__))
elif fnName == "values":
try:
return list(args[0].values())
except AttributeError:
raise ExecutionError(
"Argument is not " + color.bold("object") +
" but %s in values()" %
color.bold(type(args[0]).__name__))
elif fnName == "type":
ret = type(args[0])
if ret in ITER_TYPES:
return "array"
if ret is dict:
return "object"
return ret.__name__
elif fnName in self._REGISTERED_FUNCTIONS:
return self._REGISTERED_FUNCTIONS[fnName](*args)
else:
raise ProgrammingError("Function " + color.bold(fnName) +
" does not exist.")
else:
return node
def exe(node):
"""
node[0] - operator name
node[1:] - params
"""
if D: self.start("executing node '%s'", node)
type_node=type(node)
if node is None or isinstance(node, tuple(TYPES)):
return node
elif isinstance(node, list):
return (exe(n) for n in node)
elif isinstance(node, dict):
ret={}
for i in node.items():
ret[exe(i[0])]=exe(i[1])
return ret
op=node[0]
if op=="or":
if D: self.debug("%s or %s", node[1],node[2])
return exe(node[1]) or exe(node[2])
elif op=="and":
if D: self.debug("%s and %s", node[1],node[2])
return exe(node[1]) and exe(node[2])
elif op=="+":
if len(node)>2:
fst=exe(node[1])
snd=exe(node[2])
if None in (fst,snd):
return fst or snd
if isinstance(fst, dict):
try:
fst.update(snd)
except Exception:
if not isinstance(snd, dict):
raise ProgrammingError("Can't add value of type %s to %s" % (color.bold(PY_TYPES_MAP.get(type(snd).__name__, type(snd).__name__)), color.bold("object")))
return fst
if isinstance(fst, list) and isinstance(snd, list):
if D: self.debug("both sides are lists, returning '%s'",fst+snd)
return fst+snd
if isinstance(fst, tuple(ITER_TYPES)) or isinstance(snd, tuple(ITER_TYPES)):
if not isinstance(fst, tuple(ITER_TYPES)):
fst=[fst]
elif not isinstance(snd, tuple(ITER_TYPES)):
snd=[snd]
if D: self.debug("at least one side is generator and other is iterable, returning chain")
return chain(fst,snd)
if isinstance(fst, tuple(NUM_TYPES)):
try:
return fst+snd
except Exception:
return fst+float(snd)
if isinstance(fst, tuple(STR_TYPES)) or isinstance(snd, tuple(STR_TYPES)):
if D: self.info("doing string comparison '%s' is '%s'",fst,snd)
if sys.version_info.major < 3:
if isinstance(fst, unicode):
fst=fst.encode("utf-8")
if isinstance(snd, unicode):
snd=snd.encode("utf-8")
return str(fst)+str(snd)
try:
timeType=timeutils.datetime.time
if isinstance(fst, timeType) and isinstance(snd, timeType):
return timeutils.addTimes(fst,snd)
except Exception:
pass
if D: self.debug("standard addition, returning '%s'",fst+snd)
return fst + snd
else:
return exe(node[1])
elif op=="-":
if len(node)>2:
fst=exe(node[1])
snd=exe(node[2])
try:
return fst-snd
except Exception:
timeType=timeutils.datetime.time
if isinstance(fst, timeType) and isinstance(snd, timeType):
return timeutils.subTimes(fst,snd)
else:
return - exe(node[1])
elif op=="*":
return exe(node[1]) * exe(node[2])
elif op=="%":
return exe(node[1]) % exe(node[2])
elif op=="/":
return exe(node[1]) / float(exe(node[2]))
elif op==">":
if D: self.debug("%s > %s", node[1],node[2])
return exe(node[1]) > exe(node[2])
elif op=="<":
return exe(node[1]) < exe(node[2])
elif op==">=":
return exe(node[1]) >= exe(node[2])
elif op=="<=":
return exe(node[1]) <= exe(node[2])
# TODO this algorithm produces 3 for 1<2<3 and should be true
# elif op in "<=>=":
# fst=exe(node[1])
# snd=exe(node[2])
# if op==">":
# return fst > snd and snd or False
# elif op=="<":
# return fst < snd and snd or False
# elif op==">=":
# return fst >= snd and snd or False
# elif op=="<=":
# return fst <= snd and snd or False
elif op=="not":
fst=exe(node[1])
if D: self.debug("doing not '%s'",fst)
return not fst
elif op=="in":
fst=exe(node[1])
snd=exe(node[2])
if D: self.debug("doing '%s' in '%s'",node[1],node[2])
if isinstance(fst, tuple(ITER_TYPES)) and isinstance(snd, tuple(ITER_TYPES)):
return any(x in max(fst,snd,key=len) for x in min(fst,snd,key=len))
return exe(node[1]) in exe(node[2])
elif op=="not in":
fst=exe(node[1])
snd=exe(node[2])
if D: self.debug("doing '%s' not in '%s'",node[1],node[2])
if isinstance(fst, tuple(ITER_TYPES)) and isinstance(snd, tuple(ITER_TYPES)):
return not any(x in max(fst,snd,key=len) for x in min(fst,snd,key=len))
return exe(node[1]) not in exe(node[2])
elif op in ("is","is not"):
if D: self.debug("found operator '%s'",op)
# try:
fst=exe(node[1])
# except Exception as e:
# if D: self.debug("NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",node[1],str(e))
# fst=node[1]
# try:
snd=exe(node[2])
# except Exception as e:
# if D: self.debug("NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",node[2],str(e))
# snd=node[2]
if op == "is" and fst == snd:
return True
# this doesn't work for 3 is not '3'
# if op == "is not" and fst != snd:
# return True
if D: self.debug("type fst: '%s', type snd: '%s'",typefst,typesnd)
if isinstance(fst, tuple(STR_TYPES)):
if D: self.info("doing string comparison '\"%s\" is \"%s\"'",fst,snd)
ret=fst==str(snd)
elif isinstance(fst, float):
if D: self.info("doing float comparison '%s is %s'",fst,snd)
ret=abs(fst-float(snd))<EPSILON
elif isinstance(fst, int):
if D: self.info("doing integer comparison '%s is %s'",fst,snd)
ret=fst==int(snd)
elif isinstance(fst, list) and isinstance(snd, list):
if D: self.info("doing array comparison '%s' is '%s'",fst,snd)
ret=fst==snd
elif isinstance(fst, dict) and isinstance(snd, dict):
if D: self.info("doing object comparison '%s' is '%s'",fst,snd)
ret=fst==snd
# else:
# try:
# global ObjectId
# if not ObjectId:
# from bson.objectid import ObjectId
# if typefst is ObjectId or typesnd is ObjectId:
# if D: self.info("doing MongoDB objectID comparison '%s' is '%s'",fst,snd)
# ret=str(fst)==str(snd)
# else:
# if D: self.info("doing standard comparison '%s' is '%s'",fst,snd)
# ret=fst is snd
# except Exception:
# pass
if op=="is not":
if D: self.info("'is not' found. Returning %s",not ret)
return not ret
else:
if D: self.info("returning '%s' is '%s'='%s'",fst,snd,ret)
return ret
elif op=="re":
return re.compile(exe(node[1]))
elif op=="matches":
return not not re.match(exe(node[1]), exe(node[2]))
# elif op=="(literal)":
# fstLetter=node[1][0]
# if fstLetter is "'":
# return node[1][1:-1]
# elif fstLetter.isdigit:
# return int(node[1])
elif op=="(root)": # this is $
return self.data
# elif op=="(node)":# this is !
# if D: self.debug("returning node %s",self.node)
# return self.node
elif op=="(current)": # this is @
if D: self.debug("returning current node %s", self.current)
return self.current
elif op=="name":
return node[1]
elif op==".":
fst=node[1]
if isinstance(fst, tuple):
fst=exe(fst)
if D: self.debug(color.op(".")+" left is '%s'", fst)
# try:
if node[2][0] == "*":
if D: self.end(color.op(".")+" returning '%s'", isinstance(fst, tuple(ITER_TYPES)) and fst or [fst])
return fst # typefst in ITER_TYPES and fst or [fst]
# except:
# pass
snd=exe(node[2])
if D: self.debug(color.op(".")+" right is '%s'",snd)
if isinstance(fst, tuple(ITER_TYPES)):
if D: self.debug(color.op(".")+" filtering %s by %s",color.bold(fst),color.bold(snd))
if isinstance(snd, tuple(ITER_TYPES)):
return filter_dict(fst, list(snd))
else:
# if D: self.debug(list(fst))
return (e[snd] for e in fst if isinstance(e, dict) and snd in e)
try:
if D: self.end(color.op(".")+" returning '%s'",fst.get(snd))
return fst.get(snd)
except Exception:
if isinstance(fst,object):
return self.object_getter(fst, snd)
if D: self.end(color.op(".")+" returning '%s'", color.bold(fst))
return fst
elif op=="..":
fst=flatten(exe(node[1]))
if node[2][0]=="*":
if D: self.debug(color.op("..")+" returning '%s'", color.bold(fst))
return fst
# reduce objects to selected attributes
snd=exe(node[2])
if D: self.debug(color.op("..")+" finding all %s in %s", color.bold(snd), color.bold(fst))
if isinstance(snd, tuple(ITER_TYPES)):
ret=filter_dict(fst, list(snd))
if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
return ret
else:
ret=chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))
# print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
return ret
elif op=="[":
len_node=len(node)
# TODO move it to tree generation phase
if len_node is 1: # empty list
if D: self.debug("returning an empty list")
return []
if len_node is 2: # list - preserved to catch possible event of leaving it as '[' operator
if D: self.debug("doing list mapping")
return [exe(x) for x in node[1]]
if len_node is 3: # selector used []
fst=exe(node[1])
# check against None
if not fst:
return fst
selector=node[2]
if D: self.debug("found '%s' selector. executing on %s", color.bold(selector),color.bold(fst))
selectorIsTuple=isinstance(selector, tuple)
if selectorIsTuple and selector[0] is "[":
nodeList=[]
nodeList_append=nodeList.append
for i in fst:
if D: self.debug("setting self.current to %s",color.bold(i))
self.current=i
nodeList_append(exe((selector[0],exe(selector[1]),exe(selector[2]))))
if D: self.debug("returning %s objects: %s", color.bold(len(nodeList)),color.bold(nodeList))
return nodeList
if selectorIsTuple and selector[0] == "(current)":
if D: self.warning(color.bold("$.*[@]")+" is eqivalent to "+color.bold("$.*")+"!")
return fst
if selectorIsTuple and selector[0] in SELECTOR_OPS:
if D: self.debug("found %s operator in selector", color.bold(selector[0]))
if isinstance(fst, dict):
fst=[fst]
# TODO move it to tree building phase
if isinstance(selector[1], tuple) and selector[1][0]=="name":
selector=(selector[0],selector[1][1],selector[2])
selector0=selector[0]
selector1=selector[1]
selector2=selector[2]
def exeSelector(fst):
for i in fst:
if D: self.debug("setting self.current to %s", color.bold(i))
self.current=i
if selector0=="fn":
yield exe(selector)
elif isinstance(selector1, tuple(STR_TYPES)):
try:
if exe((selector0,i[selector1],selector2)):
yield i
if D: self.debug("appended")
if D: self.debug("discarded")
except Exception as e:
if D: self.debug("discarded, Exception: %s",color.bold(e))
else:
try:
# TODO optimize an event when @ is not used. exe(selector1) can be cached
if exe((selector0,exe(selector1),exe(selector2))):
yield i
if D: self.debug("appended")
if D: self.debug("discarded")
except Exception:
if D: self.debug("discarded")
if D: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
return exeSelector(fst)
self.current=fst
snd=exe(node[2])
if isinstance(fst, tuple([tuple]+ITER_TYPES+STR_TYPES)):
# nodes[N]
if isinstance(snd, tuple(NUM_TYPES)) or isinstance(snd, str) and snd.isdigit():
n=int(snd)
if D: self.info("getting %sth element from '%s'", color.bold(n), color.bold(fst))
if isinstance(fst, (generator,chain)):
if n>0:
return skip(fst,n)
elif n==0:
return next(fst)
else:
fst=list(fst)
else:
try:
return fst[n]
except (IndexError, TypeError):
return None
# $.*['string']==$.string
if isinstance(snd, tuple(STR_TYPES)):
return exe((".",fst,snd))
else:
# $.*[@.string] - bad syntax, but allowed
return snd
else:
try:
if D: self.debug("returning %s", color.bold(fst[snd]))
return fst[snd]
except KeyError:
# CHECK - is it ok to do that or should it be ProgrammingError?
if D: self.debug("returning an empty list")
return []
raise ProgrammingError("Wrong usage of "+color.bold("[")+" operator")
elif op=="fn":
# Built-in functions
fnName=node[1]
args=None
try:
args=[exe(x) for x in node[2:]]
except IndexError:
if D: self.debug("NOT ERROR: can't map '%s' with '%s'",node[2:],exe)
# arithmetic
if fnName=="sum":
args=args[0]
if isinstance(args, tuple(NUM_TYPES)):
return args
return sum((x for x in args if isinstance(x, tuple(NUM_TYPES))))
elif fnName=="max":
args=args[0]
if isinstance(args, tuple(NUM_TYPES)):
return args
return max((x for x in args if isinstance(x, tuple(NUM_TYPES))))
elif fnName=="min":
args=args[0]
if isinstance(args, tuple(NUM_TYPES)):
return args
return min((x for x in args if isinstance(x, tuple(NUM_TYPES))))
elif fnName=="avg":
args=args[0]
if isinstance(args, tuple(NUM_TYPES)):
return args
if not isinstance(args, tuple(ITER_TYPES)):
raise Exception("Argument for avg() is not an array")
else:
args=list(args)
try:
return sum(args)/float(len(args))
except TypeError:
args=[x for x in args if isinstance(x, tuple(NUM_TYPES))]
self.warning("Some items in array were ommited")
return sum(args)/float(len(args))
elif fnName=="round":
return round(*args)
# casting
elif fnName=="int":
return int(args[0])
elif fnName=="float":
return float(args[0])
elif fnName=="str":
return str(py2JSON(args[0]))
elif fnName in ("list","array"):
try:
a=args[0]
except IndexError:
return []
if isinstance(a, timeutils.datetime.datetime):
return timeutils.date2list(a)+timeutils.time2list(a)
if isinstance(a, timeutils.datetime.date):
return timeutils.date2list(a)
if isinstance(a, timeutils.datetime.time):
return timeutils.time2list(a)
return list(a)
# string
elif fnName=="upper":
return args[0].upper()
elif fnName=="lower":
return args[0].lower()
elif fnName=="capitalize":
return args[0].capitalize()
elif fnName=="title":
return args[0].title()
elif fnName=="split":
return args[0].split(*args[1:])
elif fnName=="slice":
if args and not isinstance(args[1], tuple(ITER_TYPES)):
raise ExecutionError("Wrong usage of slice(STRING, ARRAY). Second argument is not an array but %s."%color.bold(type(args[1]).__name__))
try:
pos=list(args[1])
if isinstance(pos[0], tuple(ITER_TYPES)):
if D: self.debug("run slice() for a list of slicers")
return (args[0][x[0]:x[1]] for x in pos)
return args[0][pos[0]:pos[1]]
except IndexError:
if len(args)!=2:
raise ProgrammingError("Wrong usage of slice(STRING, ARRAY). Provided %s argument, should be exactly 2."%len(args))
elif fnName=="escape":
global escape,escapeDict
if not escape:
from objectpath.utils import escape, escapeDict
return escape(args[0],escapeDict)
elif fnName=="unescape":
global unescape,unescapeDict
if not unescape:
from objectpath.utils import unescape, unescapeDict
return unescape(args[0],unescapeDict)
elif fnName=="replace":
if sys.version_info.major < 3 and isinstance(args[0], unicode):
args[0]=args[0].encode("utf8")
return str.replace(args[0],args[1],args[2])
# TODO this should be supported by /regex/
# elif fnName=="REsub":
# return re.sub(args[1],args[2],args[0])
elif fnName=="sort":
if len(args)>1:
key=args[1]
a={"key":lambda x: x.get(key, 0)}
else:
a={}
args=args[0]
if D: self.debug("doing sort on '%s'",args)
try:
return sorted(args,**a)
except TypeError:
return args
elif fnName=="reverse":
args=args[0]
try:
args.reverse()
return args
except TypeError:
return args
elif fnName=="map":
return map(lambda x: exe(("fn",args[0],x)), args[1])
elif fnName in ("count","len"):
args=args[0]
if args in (True,False,None):
return args
if isinstance(args, tuple(ITER_TYPES)):
return len(list(args))
return len(args)
elif fnName=="join":
try:
joiner=args[1]
except Exception:
joiner=""
try:
return joiner.join(args[0])
except TypeError:
try:
return joiner.join(map(str,args[0]))
except Exception:
return args[0]
# time
elif fnName in ("now","age","time","date","dateTime"):
if fnName=="now":
return timeutils.now()
if fnName=="date":
return timeutils.date(args)
if fnName=="time":
return timeutils.time(args)
if fnName=="dateTime":
return timeutils.dateTime(args)
# TODO move lang to localize() entirely!
if fnName=="age":
a={}
if len(args)>1:
a["reference"]=args[1]
if len(args)>2:
a["lang"]=args[2]
return list(timeutils.age(args[0],**a))
elif fnName=="toMillis":
args=args[0]
if args.utcoffset() is not None:
args=args-args.utcoffset() # pylint: disable=E1103
global calendar
if not calendar:
import calendar
return int(calendar.timegm(args.timetuple()) * 1000 + args.microsecond / 1000)
elif fnName=="localize":
if isinstance(args[0], timeutils.datetime.datetime):
return timeutils.UTC2local(*args)
# polygons
elif fnName=="area":
def segments(p):
p=list(map(lambda x: x[0:2],p))
return zip(p, p[1:] + [p[0]])
return 0.5 * abs(sum(x0*y1 - x1*y0
for ((x0, y0), (x1, y1)) in segments(args[0])))
# misc
elif fnName=="keys":
try:
return list(args[0].keys())
except AttributeError:
raise ExecutionError("Argument is not "+color.bold("object")+" but %s in keys()"%color.bold(type(args[0]).__name__))
elif fnName=="type":
if isinstance(args[0], tuple(ITER_TYPES)):
return "array"
if isinstance(args[0], dict):
return "object"
return args[0].__class__.__name__
elif fnName in self._REGISTERED_FUNCTIONS:
return self._REGISTERED_FUNCTIONS[fnName](*args)
else:
raise ProgrammingError("Function "+color.bold(fnName)+" does not exist.")
else:
return node
def exe(self, node):
"""
node[0] - operator name
node[1:] - params
"""
types = [str, timeutils.datetime.time, timeutils.datetime.date, timeutils.datetime.datetime]
try:
types += [unicode]
except:
pass
if self.D:
self.start("executing node '%s'", node)
type_node = type(node)
if node is None or type_node in TYPES:
return node
elif type_node in types:
return node
elif type_node is list:
return (self.exe(n) for n in node)
elif type_node is dict:
ret = {}
for i in node.items():
ret[self.exe(i[0])] = self.exe(i[1])
return ret
op = node[0]
if op in OP_TOKENS.keys():
return self.lexer[op](*node)
elif op == "(root)": # this is $
return self.data
# elif op=="(node)":# this is !
# if D: self.debug("returning node %s",self.node)
# return self.node
elif op == "(current)": # this is @
if self.D:
self.debug("returning current node %s", self.current)
return self.current
elif op == "name":
return node[1]
elif op == ".":
fst = node[1]
if type(fst) is tuple:
fst = self.exe(fst)
typefst = type(fst)
if self.D:
self.debug(color.op(".") + " left is '%s'", fst)
# try:
if node[2][0] == "*":
if self.D:
self.end(color.op(".") + " returning '%s'", typefst in ITER_TYPES and fst or [fst])
return fst # typefst in ITER_TYPES and fst or [fst]
# except:
# pass
snd = self.exe(node[2])
if self.D:
self.debug(color.op(".") + " right is '%s'", snd)
if typefst in ITER_TYPES:
if self.D:
self.debug(color.op(".") + " filtering %s by %s", color.bold(fst), color.bold(snd))
if type(snd) in ITER_TYPES:
return filter_dict(fst, list(snd))
else:
# if D: self.debug(list(fst))
return (e[snd] for e in fst if type(e) is dict and snd in e)
try:
if self.D:
self.end(color.op(".") + " returning '%s'", fst.get(snd))
return fst.get(snd)
except Exception:
if isinstance(fst, object):
return self.object_getter(fst, snd)
if self.D:
self.end(color.op(".") + " returning '%s'", color.bold(fst))
return fst
elif op == "..":
fst = flatten(self.exe(node[1]))
if node[2][0] == "*":
if self.D:
self.debug(color.op("..") + " returning '%s'", color.bold(fst))
return fst
# reduce objects to selected attributes
snd = self.exe(node[2])
if self.D:
self.debug(color.op("..") + " finding all %s in %s", color.bold(snd), color.bold(fst))
if type(snd) in ITER_TYPES:
ret = filter_dict(fst, list(snd))
if self.D:
self.debug(color.op("..") + " returning %s", color.bold(ret))
return ret
else:
ret = chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))
# print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
if self.D:
self.debug(color.op("..") + " returning %s", color.bold(ret))
return ret
elif op == "[":
len_node = len(node)
# TODO move it to tree generation phase
if len_node is 1: # empty list
if self.D:
self.debug("returning an empty list")
return []
if len_node is 2: # list - preserved to catch possible event of leaving it as '[' operator
if self.D:
self.debug("doing list mapping")
return [self.exe(x) for x in node[1]]
if len_node is 3: # selector used []
fst = self.exe(node[1])
# check against None
if not fst:
return fst
selector = node[2]
if self.D:
self.debug("found '%s' selector. executing on %s", color.bold(selector), color.bold(fst))
selectorIsTuple = type(selector) is tuple
if selectorIsTuple and selector[0] is "[":
nodeList = []
nodeList_append = nodeList.append
for i in fst:
if self.D:
self.debug("setting self.current to %s", color.bold(i))
self.current = i
nodeList_append(self.exe((selector[0], self.exe(selector[1]), self.exe(selector[2]))))
if self.D:
self.debug("returning %s objects: %s", color.bold(len(nodeList)), color.bold(nodeList))
return nodeList
if selectorIsTuple and selector[0] == "(current)":
if self.D:
self.warning(color.bold("$.*[@]") + " is eqivalent to " + color.bold("$.*") + "!")
return fst
if selectorIsTuple and selector[0] in SELECTOR_OPS:
if self.D:
self.debug("found %s operator in selector, %s", color.bold(selector[0]),
color.bold(selector))
if type(fst) is dict:
fst = [fst]
# TODO move it to tree building phase
if type(selector[1]) is tuple and selector[1][0] == "name":
selector = (selector[0], selector[1][1], selector[2])
# if D and nodeList: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
return self.exeSelector(fst, selector)
self.current = fst
snd = self.exe(node[2])
typefst = type(fst)
if typefst in [tuple] + ITER_TYPES + STR_TYPES:
typesnd = type(snd)
# nodes[N]
if typesnd in NUM_TYPES or typesnd is str and snd.isdigit():
n = int(snd)
if self.D:
self.info("getting %sth element from '%s'", color.bold(n), color.bold(fst))
if typefst in (generator, chain):
if n > 0:
return skip(fst, n)
elif n == 0:
return next(fst)
else:
fst = list(fst)
else:
try:
return fst[n]
except (IndexError, TypeError):
return None
# $.*['string']==$.string
if type(snd) in STR_TYPES:
return self.exe((".", fst, snd))
else:
# $.*[@.string] - bad syntax, but allowed
return snd
else:
try:
if self.D:
self.debug("returning %s", color.bold(fst[snd]))
return fst[snd]
except KeyError:
# CHECK - is it ok to do that or should it be ProgrammingError?
if self.D:
self.debug("returning an empty list")
return []
raise ProgrammingError("Wrong usage of " + color.bold("[") + " operator")
elif op == "fn":
# Built-in functions
fnName = node[1]
args = None
try:
args = [self.exe(x) for x in node[2:]]
except IndexError:
if self.D:
self.debug("NOT ERROR: can't map '%s' with '%s'", node[2:], self.exe)
if fnName in FN_TOKENS.keys():
return self.functions[fnName](*args)
elif fnName in self._REGISTERED_FUNCTIONS:
return self._REGISTERED_FUNCTIONS[fnName](*args)
else:
raise ProgrammingError("Function " + color.bold(fnName) + " does not exist.")
else:
return node