def testSum(self):
# Adding a constant to a list, adds it to each element.
query = q.Query(
"SELECT ages + 10 as sum FROM (bind('ages': [10, 20, 30]))")
value = list(solve.solve(query, {}).value)
sum = structured.resolve(value[0], "sum")
self.assertEqual(sum, [20, 30, 40])
# Adding a list to a list, pads the short list with zeros.
query = q.Query(
"SELECT ages + [10, 20] as sum FROM (bind('ages': [10, 20, 30]))")
value = list(solve.solve(query, {}).value)
sum = structured.resolve(value[0], "sum")
self.assertEqual(sum, [20, 40, 30])
# Repeated integers add item by item to the
query = q.Query(
"[5, 1, 2] + 10 + SELECT age FROM"
" (bind('age': 10, 'name': 'Tom'), bind('age': 8, 'name': 'Jerry'))"
" WHERE name == 'Jerry'")
self.assertEqual(solve.solve(query, {}).value, [23, 11, 12])
self.assertEqual(
solve.solve(q.Query("5 + 15 + 25"),
mocks.Process(1, None, None)).value, [45])
def testProduct(self):
# Multiplying a constant to a list, adds it to each element.
query = q.Query(
"SELECT ages * 10 as x FROM (bind('ages': [10, 20, 30]))")
value = list(solve.solve(query, {}).value)
x = structured.resolve(value[0], "x")
self.assertEqual(x, [100, 200, 300])
# Multiplying a constant to a list, multiply each element.
query = q.Query(
"10 * (SELECT age FROM (bind('age': 10), bind('age': 20)))")
value = list(solve.solve(query, {}).value)
self.assertEqual(value, [100, 200])
# Multiplying two subselects multiplies each element.
query = q.Query(
"""(SELECT age FROM (bind('age': 10), bind('age': 20))) *
(SELECT age FROM (bind('age': 20), bind('age': 30)))
""")
value = list(solve.solve(query, {}).value)
self.assertEqual(value, [200, 600])
self.assertEqual(
solve.solve(q.Query("5 * 5 * 5"),
mocks.Process(1, None, None)).value, [125])
def testDifference(self):
# Adding a constant to a list, adds it to each element.
query = q.Query(
"SELECT ages - 10 as diff FROM (bind('ages': [10, 20, 30]))")
value = list(solve.solve(query, {}).value)
diff = structured.resolve(value[0], "diff")
self.assertEqual(diff, [0, 10, 20])
# Subtracting numbers from non numbers just gives None.
query = q.Query('SELECT ages - 10 as diff FROM '
'(bind("ages": ["foo", "bar", "baz"]))')
value = list(solve.solve(query, {}).value)
diff = structured.resolve(value[0], "diff")
self.assertEqual(diff, [None, None, None])
def solve_resolve(expr, vars):
"""Use IStructured.resolve to get member (rhs) from the object (lhs).
This operation supports both scalars and repeated values on the LHS -
resolving from a repeated value implies a map-like operation and returns a
new repeated values.
"""
objs, _ = __solve_for_repeated(expr.lhs, vars)
member = solve(expr.rhs, vars).value
try:
results = [
structured.resolve(o, member) for o in repeated.getvalues(objs)
]
except (KeyError, AttributeError):
# Raise a better exception for the non-existent member.
raise errors.EfilterKeyError(root=expr.rhs,
key=member,
query=expr.source)
except (TypeError, ValueError):
# Is this a null object error?
if vars.locals is None:
raise errors.EfilterNoneError(
root=expr,
query=expr.source,
message="Cannot resolve member %r from a null." % member)
else:
raise
except NotImplementedError:
raise errors.EfilterError(
root=expr,
query=expr.source,
message="Cannot resolve members from a non-structured value.")
return Result(repeated.meld(*results), ())
def solve_resolve(expr, vars):
"""Use IStructured.resolve to get member (rhs) from the object (lhs)."""
objs = __within_lhs_as_repeated(expr.lhs, vars)
member = solve(expr.rhs, vars).value
try:
results = [structured.resolve(o, member)
for o in repeated.getvalues(objs)]
except (KeyError, AttributeError):
# Raise a better exception for the non-existent member.
raise errors.EfilterKeyError(root=expr.rhs, key=member,
query=expr.source)
except (TypeError, ValueError):
# Is this a null object error?
if vars is None:
raise errors.EfilterNoneError(
root=expr, query=expr.source,
message="Cannot resolve member %r from a null." % member)
else:
raise
except NotImplementedError:
raise errors.EfilterError(
root=expr, query=expr.source,
message="Cannot resolve members from a non-structured value.")
return Result(repeated.meld(*results), ())
def solve_var(expr, vars):
"""Returns the value of the var named in the expression."""
try:
return Result(structured.resolve(vars, expr.value), ())
except (KeyError, AttributeError) as e:
# Raise a better exception for accessing a non-existent member.
raise errors.EfilterKeyError(root=expr,
key=expr.value,
message=e,
query=expr.source)
except (TypeError, ValueError) as e:
# Raise a better exception for what is probably a null pointer error.
if vars.locals is None:
raise errors.EfilterNoneError(
root=expr,
query=expr.source,
message="Trying to access member %r of a null." % expr.value)
else:
raise errors.EfilterTypeError(root=expr,
query=expr.source,
message="%r (vars: %r)" % (e, vars))
except NotImplementedError as e:
raise errors.EfilterError(
root=expr,
query=expr.source,
message="Trying to access member %r of an instance of %r." %
(expr.value, type(vars)))
def collect_members(self, item, depth):
if depth > self.plugin_args.max_depth:
return
try:
for member in sorted(structured.getmembers(item)):
type_instance = structured.resolve(item, member)
# If it was given as a type, we need an instance here.
if isinstance(type_instance, type):
type_instance = type_instance()
try:
object_type = type_instance.obj_type
except AttributeError:
object_type = type(type_instance).__name__
yield dict(
Field=member,
Type=object_type,
depth=depth,
)
for x in self.collect_members(type_instance, depth + 1):
yield x
except (TypeError, NotImplementedError):
pass
def solve_resolve(expr, vars):
"""Use IStructured.resolve to get member (rhs) from the object (lhs).
This operation supports both scalars and repeated values on the LHS -
resolving from a repeated value implies a map-like operation and returns a
new repeated values.
"""
objs = __solve_for_repeated(expr.lhs, vars)
member = solve(expr.rhs, vars).value
try:
results = [structured.resolve(o, member)
for o in repeated.getvalues(objs)]
except (KeyError, AttributeError):
# Raise a better exception for the non-existent member.
raise errors.EfilterKeyError(root=expr.rhs, key=member,
query=expr.source)
except (TypeError, ValueError):
# Is this a null object error?
if vars.locals is None:
raise errors.EfilterNoneError(
root=expr, query=expr.source,
message="Cannot resolve member %r from a null." % member)
else:
raise
except NotImplementedError:
raise errors.EfilterError(
root=expr, query=expr.source,
message="Cannot resolve members from a non-structured value.")
return Result(repeated.meld(*results), ())
def testInterfaces(self):
rt = row_tuple.RowTuple(values=dict(foo="Foo", bar="Bar", car="Car"),
ordered_columns=["car", "bar", "foo"])
self.assertEqual(len(rt), 3)
self.assertEqual(counted.count(rt), 3)
self.assertEqual(list(rt), ["Car", "Bar", "Foo"])
self.assertEqual(associative.select(rt, 2), "Foo")
with self.assertRaises(KeyError):
associative.select(rt, "foo")
with self.assertRaises(KeyError):
structured.resolve(rt, 2)
self.assertEqual(structured.resolve(rt, "foo"), "Foo")
def resolve_Pointer(ptr, member):
"""Delegate to target of the pointer, if any."""
target_obj = ptr.deref()
if not target_obj:
ptr.session.logging.warn(
"Attempting to access member %r of a void pointer %r.", member, ptr)
if target_obj:
return structured.resolve(target_obj, member)
def resolve_Pointer(ptr, member):
"""Delegate to target of the pointer, if any."""
obj = ptr.deref()
if not obj:
ptr.session.logging.warn(
"Attempting to access member %r of a void pointer %r.", member, ptr)
if obj:
return structured.resolve(obj, member)
def __init__(self, other):
super(dict, self).__init__()
if not protocol.implements(other, structured.IStructured):
raise TypeError("Can only set scope from IStructured.")
# Copy the scope locally.
for key in structured.getmembers_runtime(other):
self[key] = structured.resolve(other, key)
def resolve(self, name):
"""Call IStructured.resolve across all scopes and return first hit."""
for scope in reversed(self.scopes):
try:
return structured.resolve(scope, name)
except (KeyError, AttributeError):
continue
raise AttributeError(name)
def resolve(self, name):
"""Call IStructured.resolve across all scopes and return first hit."""
for scope in reversed(self.scopes):
try:
return structured.resolve(scope, name)
except (KeyError, AttributeError):
continue
raise AttributeError(name)
def solve_resolve(expr, vars):
"""Use IStructured.resolve to get member (rhs) from the object (lhs).
This operation supports both scalars and repeated values on the LHS -
resolving from a repeated value implies a map-like operation and returns a
new repeated values.
"""
objs = solve(expr.lhs, vars).value
member = solve(expr.rhs, vars).value
results = []
if repeated.isrepeating(objs):
for o in repeated.getvalues(objs):
results.append(structured.resolve(o, member))
return Result(results, ())
return Result(structured.resolve(objs, member), ())
def collect_members(self, item, depth):
if depth > self.plugin_args.max_depth:
return
try:
for member in sorted(structured.getmembers(item)):
type_instance = structured.resolve(item, member)
# If it was given as a type, we need an instance here.
yield dict(
Field=member,
Type=self._determine_type_name(type_instance),
depth=depth,
)
for x in self.collect_members(type_instance, depth + 1):
yield x
except (TypeError, NotImplementedError):
pass
def collect_members(self, item, depth):
if depth > self.plugin_args.max_depth:
return
try:
for member in sorted(structured.getmembers(item)):
type_instance = structured.resolve(item, member)
# If it was given as a type, we need an instance here.
yield dict(
Field=member,
Type=self._determine_type_name(type_instance),
depth=depth,
)
for x in self.collect_members(type_instance, depth + 1):
yield x
except (TypeError, NotImplementedError):
pass
def __eq__(self, other):
if isinstance(other, type(self)):
return self.ordered_dict == other.ordered_dict
elif isinstance(other, structured.IStructured):
try:
other_members = structured.getmembers(other)
except NotImplementedError:
return None
members = sorted(self.ordered_dict.keys())
if members != sorted(other_members):
return False
vals = tuple([self.get(m) for m in members])
other_vals = tuple([structured.resolve(other, m) for m in members])
return vals == other_vals
elif isinstance(other, (tuple, list)):
return list(self) == list(other)
else:
return None
def __eq__(self, other):
if isinstance(other, type(self)):
return self.ordered_dict == other.ordered_dict
elif isinstance(other, structured.IStructured):
try:
other_members = structured.getmembers(other)
except NotImplementedError:
return None
members = sorted(self.ordered_dict.keys())
if members != sorted(other_members):
return False
vals = tuple([self.get(m) for m in members])
other_vals = tuple([structured.resolve(other, m) for m in members])
return vals == other_vals
elif isinstance(other, (tuple, list)):
return list(self) == list(other)
else:
return None
def convert_to_list(expr, repeated_list):
if not repeated.isrepeating(repeated_list):
return [repeated_list]
result = []
for element in repeated_list:
if element is not None:
# The output from a select is a repeated structured
# (dict). If it has a single member we just use that,
# otherwise we raise because the query is probably bad
# (it should only return a single column).
if structured.isstructured(element):
members = structured.getmembers(element)
if len(members) != 1:
raise errors.EfilterTypeError(
message="Expecting a single column in subselect - "
"got %s columns" % len(members),
query=expr.source)
element = structured.resolve(element, members[0])
result.append(element)
return result
def solve_var(expr, vars):
"""Returns the value of the var named in the expression."""
try:
return Result(structured.resolve(vars, expr.value), ())
except (KeyError, AttributeError) as e:
# Raise a better exception for accessing a non-existent member.
raise errors.EfilterKeyError(root=expr, key=expr.value, message=e,
query=expr.source)
except (TypeError, ValueError) as e:
# Raise a better exception for what is probably a null pointer error.
if vars.locals is None:
raise errors.EfilterNoneError(
root=expr, query=expr.source,
message="Trying to access member %r of a null." % expr.value)
else:
raise errors.EfilterTypeError(
root=expr, query=expr.source,
message="%r (vars: %r)" % (e, vars))
except NotImplementedError as e:
raise errors.EfilterError(
root=expr, query=expr.source,
message="Trying to access member %r of an instance of %r." %
(expr.value, type(vars)))
def _materialize_repeated_kwarg(self, kwargs):
"""Materialize the result of the args.
This is a shim between a repeated plugin arg and the efilter
stream. We handle the following cases.
1. EFilter LazyRepetition with unstructured elements (e.g. dicts).
2. EFilter LazyRepetition with structured elements. These are
usually returned from a subselect. In the special case
where the arg name is present in the structure
"""
result = {}
for k, v in six.iteritems(kwargs):
if not repeated.isrepeating(v):
result[k] = v
else:
expanded_value = []
for item in v:
if structured.isstructured(item):
members = structured.getmembers(item)
if len(members) == 1 or k in members:
# A single column in the subquery - just
# use that as the arg value. If the name
# emitted is the same as the expected arg
# name we also just take that one.
expanded_value.append(
structured.resolve(item, members[0]))
continue
expanded_value.append(item)
result[k] = expanded_value
return result
(name
for name, _ in s.getproperties()))
})
# This lets us get indices out of Arrays.
associative.IAssociative.implement(for_type=obj.Array,
implementations={
associative.select:
lambda obj, key: obj[key],
})
# This lets us do some_array.some_member.
structured.IStructured.implement(for_type=obj.Array,
implementations={structured.resolve: getattr})
# Pointer[key] is implemented as Pointer.dereference()[key].
associative.IAssociative.implement(
for_type=obj.Pointer,
implementations={
associative.select:
lambda ptr, key: associative.select(ptr.deref(), key),
})
# Pointer.member is implemented as Pointer.dereference().member.
structured.IStructured.implement(
for_type=obj.Pointer,
implementations={
structured.resolve:
lambda ptr, key: structured.resolve(ptr.deref(), key)
})
)
# This lets us do some_array.some_member.
structured.IStructured.implement(
for_type=obj.Array,
implementations={
structured.resolve: getattr
}
)
# Pointer[key] is implemented as Pointer.dereference()[key].
associative.IAssociative.implement(
for_type=obj.Pointer,
implementations={
associative.select:
lambda ptr, key: associative.select(ptr.deref(), key),
}
)
# Pointer.member is implemented as Pointer.dereference().member.
structured.IStructured.implement(
for_type=obj.Pointer,
implementations={
structured.resolve:
lambda ptr, key: structured.resolve(ptr.deref(), key)
}
)
