def testSum(self): # Adding a constant to a list, adds it to each element. query = q.Query( "SELECT ages + 10 as sum FROM (bind('ages': [10, 20, 30]))") value = list(solve.solve(query, {}).value) sum = structured.resolve(value[0], "sum") self.assertEqual(sum, [20, 30, 40]) # Adding a list to a list, pads the short list with zeros. query = q.Query( "SELECT ages + [10, 20] as sum FROM (bind('ages': [10, 20, 30]))") value = list(solve.solve(query, {}).value) sum = structured.resolve(value[0], "sum") self.assertEqual(sum, [20, 40, 30]) # Repeated integers add item by item to the query = q.Query( "[5, 1, 2] + 10 + SELECT age FROM" " (bind('age': 10, 'name': 'Tom'), bind('age': 8, 'name': 'Jerry'))" " WHERE name == 'Jerry'") self.assertEqual(solve.solve(query, {}).value, [23, 11, 12]) self.assertEqual( solve.solve(q.Query("5 + 15 + 25"), mocks.Process(1, None, None)).value, [45])
def testProduct(self): # Multiplying a constant to a list, adds it to each element. query = q.Query( "SELECT ages * 10 as x FROM (bind('ages': [10, 20, 30]))") value = list(solve.solve(query, {}).value) x = structured.resolve(value[0], "x") self.assertEqual(x, [100, 200, 300]) # Multiplying a constant to a list, multiply each element. query = q.Query( "10 * (SELECT age FROM (bind('age': 10), bind('age': 20)))") value = list(solve.solve(query, {}).value) self.assertEqual(value, [100, 200]) # Multiplying two subselects multiplies each element. query = q.Query( """(SELECT age FROM (bind('age': 10), bind('age': 20))) * (SELECT age FROM (bind('age': 20), bind('age': 30))) """) value = list(solve.solve(query, {}).value) self.assertEqual(value, [200, 600]) self.assertEqual( solve.solve(q.Query("5 * 5 * 5"), mocks.Process(1, None, None)).value, [125])
def testDifference(self): # Adding a constant to a list, adds it to each element. query = q.Query( "SELECT ages - 10 as diff FROM (bind('ages': [10, 20, 30]))") value = list(solve.solve(query, {}).value) diff = structured.resolve(value[0], "diff") self.assertEqual(diff, [0, 10, 20]) # Subtracting numbers from non numbers just gives None. query = q.Query('SELECT ages - 10 as diff FROM ' '(bind("ages": ["foo", "bar", "baz"]))') value = list(solve.solve(query, {}).value) diff = structured.resolve(value[0], "diff") self.assertEqual(diff, [None, None, None])
def solve_resolve(expr, vars): """Use IStructured.resolve to get member (rhs) from the object (lhs). This operation supports both scalars and repeated values on the LHS - resolving from a repeated value implies a map-like operation and returns a new repeated values. """ objs, _ = __solve_for_repeated(expr.lhs, vars) member = solve(expr.rhs, vars).value try: results = [ structured.resolve(o, member) for o in repeated.getvalues(objs) ] except (KeyError, AttributeError): # Raise a better exception for the non-existent member. raise errors.EfilterKeyError(root=expr.rhs, key=member, query=expr.source) except (TypeError, ValueError): # Is this a null object error? if vars.locals is None: raise errors.EfilterNoneError( root=expr, query=expr.source, message="Cannot resolve member %r from a null." % member) else: raise except NotImplementedError: raise errors.EfilterError( root=expr, query=expr.source, message="Cannot resolve members from a non-structured value.") return Result(repeated.meld(*results), ())
def solve_resolve(expr, vars): """Use IStructured.resolve to get member (rhs) from the object (lhs).""" objs = __within_lhs_as_repeated(expr.lhs, vars) member = solve(expr.rhs, vars).value try: results = [structured.resolve(o, member) for o in repeated.getvalues(objs)] except (KeyError, AttributeError): # Raise a better exception for the non-existent member. raise errors.EfilterKeyError(root=expr.rhs, key=member, query=expr.source) except (TypeError, ValueError): # Is this a null object error? if vars is None: raise errors.EfilterNoneError( root=expr, query=expr.source, message="Cannot resolve member %r from a null." % member) else: raise except NotImplementedError: raise errors.EfilterError( root=expr, query=expr.source, message="Cannot resolve members from a non-structured value.") return Result(repeated.meld(*results), ())
def solve_var(expr, vars): """Returns the value of the var named in the expression.""" try: return Result(structured.resolve(vars, expr.value), ()) except (KeyError, AttributeError) as e: # Raise a better exception for accessing a non-existent member. raise errors.EfilterKeyError(root=expr, key=expr.value, message=e, query=expr.source) except (TypeError, ValueError) as e: # Raise a better exception for what is probably a null pointer error. if vars.locals is None: raise errors.EfilterNoneError( root=expr, query=expr.source, message="Trying to access member %r of a null." % expr.value) else: raise errors.EfilterTypeError(root=expr, query=expr.source, message="%r (vars: %r)" % (e, vars)) except NotImplementedError as e: raise errors.EfilterError( root=expr, query=expr.source, message="Trying to access member %r of an instance of %r." % (expr.value, type(vars)))
def collect_members(self, item, depth): if depth > self.plugin_args.max_depth: return try: for member in sorted(structured.getmembers(item)): type_instance = structured.resolve(item, member) # If it was given as a type, we need an instance here. if isinstance(type_instance, type): type_instance = type_instance() try: object_type = type_instance.obj_type except AttributeError: object_type = type(type_instance).__name__ yield dict( Field=member, Type=object_type, depth=depth, ) for x in self.collect_members(type_instance, depth + 1): yield x except (TypeError, NotImplementedError): pass
def solve_resolve(expr, vars): """Use IStructured.resolve to get member (rhs) from the object (lhs). This operation supports both scalars and repeated values on the LHS - resolving from a repeated value implies a map-like operation and returns a new repeated values. """ objs = __solve_for_repeated(expr.lhs, vars) member = solve(expr.rhs, vars).value try: results = [structured.resolve(o, member) for o in repeated.getvalues(objs)] except (KeyError, AttributeError): # Raise a better exception for the non-existent member. raise errors.EfilterKeyError(root=expr.rhs, key=member, query=expr.source) except (TypeError, ValueError): # Is this a null object error? if vars.locals is None: raise errors.EfilterNoneError( root=expr, query=expr.source, message="Cannot resolve member %r from a null." % member) else: raise except NotImplementedError: raise errors.EfilterError( root=expr, query=expr.source, message="Cannot resolve members from a non-structured value.") return Result(repeated.meld(*results), ())
def testInterfaces(self): rt = row_tuple.RowTuple(values=dict(foo="Foo", bar="Bar", car="Car"), ordered_columns=["car", "bar", "foo"]) self.assertEqual(len(rt), 3) self.assertEqual(counted.count(rt), 3) self.assertEqual(list(rt), ["Car", "Bar", "Foo"]) self.assertEqual(associative.select(rt, 2), "Foo") with self.assertRaises(KeyError): associative.select(rt, "foo") with self.assertRaises(KeyError): structured.resolve(rt, 2) self.assertEqual(structured.resolve(rt, "foo"), "Foo")
def resolve_Pointer(ptr, member): """Delegate to target of the pointer, if any.""" target_obj = ptr.deref() if not target_obj: ptr.session.logging.warn( "Attempting to access member %r of a void pointer %r.", member, ptr) if target_obj: return structured.resolve(target_obj, member)
def resolve_Pointer(ptr, member): """Delegate to target of the pointer, if any.""" obj = ptr.deref() if not obj: ptr.session.logging.warn( "Attempting to access member %r of a void pointer %r.", member, ptr) if obj: return structured.resolve(obj, member)
def __init__(self, other): super(dict, self).__init__() if not protocol.implements(other, structured.IStructured): raise TypeError("Can only set scope from IStructured.") # Copy the scope locally. for key in structured.getmembers_runtime(other): self[key] = structured.resolve(other, key)
def resolve(self, name): """Call IStructured.resolve across all scopes and return first hit.""" for scope in reversed(self.scopes): try: return structured.resolve(scope, name) except (KeyError, AttributeError): continue raise AttributeError(name)
def solve_resolve(expr, vars): """Use IStructured.resolve to get member (rhs) from the object (lhs). This operation supports both scalars and repeated values on the LHS - resolving from a repeated value implies a map-like operation and returns a new repeated values. """ objs = solve(expr.lhs, vars).value member = solve(expr.rhs, vars).value results = [] if repeated.isrepeating(objs): for o in repeated.getvalues(objs): results.append(structured.resolve(o, member)) return Result(results, ()) return Result(structured.resolve(objs, member), ())
def collect_members(self, item, depth): if depth > self.plugin_args.max_depth: return try: for member in sorted(structured.getmembers(item)): type_instance = structured.resolve(item, member) # If it was given as a type, we need an instance here. yield dict( Field=member, Type=self._determine_type_name(type_instance), depth=depth, ) for x in self.collect_members(type_instance, depth + 1): yield x except (TypeError, NotImplementedError): pass
def __eq__(self, other): if isinstance(other, type(self)): return self.ordered_dict == other.ordered_dict elif isinstance(other, structured.IStructured): try: other_members = structured.getmembers(other) except NotImplementedError: return None members = sorted(self.ordered_dict.keys()) if members != sorted(other_members): return False vals = tuple([self.get(m) for m in members]) other_vals = tuple([structured.resolve(other, m) for m in members]) return vals == other_vals elif isinstance(other, (tuple, list)): return list(self) == list(other) else: return None
def convert_to_list(expr, repeated_list): if not repeated.isrepeating(repeated_list): return [repeated_list] result = [] for element in repeated_list: if element is not None: # The output from a select is a repeated structured # (dict). If it has a single member we just use that, # otherwise we raise because the query is probably bad # (it should only return a single column). if structured.isstructured(element): members = structured.getmembers(element) if len(members) != 1: raise errors.EfilterTypeError( message="Expecting a single column in subselect - " "got %s columns" % len(members), query=expr.source) element = structured.resolve(element, members[0]) result.append(element) return result
def solve_var(expr, vars): """Returns the value of the var named in the expression.""" try: return Result(structured.resolve(vars, expr.value), ()) except (KeyError, AttributeError) as e: # Raise a better exception for accessing a non-existent member. raise errors.EfilterKeyError(root=expr, key=expr.value, message=e, query=expr.source) except (TypeError, ValueError) as e: # Raise a better exception for what is probably a null pointer error. if vars.locals is None: raise errors.EfilterNoneError( root=expr, query=expr.source, message="Trying to access member %r of a null." % expr.value) else: raise errors.EfilterTypeError( root=expr, query=expr.source, message="%r (vars: %r)" % (e, vars)) except NotImplementedError as e: raise errors.EfilterError( root=expr, query=expr.source, message="Trying to access member %r of an instance of %r." % (expr.value, type(vars)))
def _materialize_repeated_kwarg(self, kwargs): """Materialize the result of the args. This is a shim between a repeated plugin arg and the efilter stream. We handle the following cases. 1. EFilter LazyRepetition with unstructured elements (e.g. dicts). 2. EFilter LazyRepetition with structured elements. These are usually returned from a subselect. In the special case where the arg name is present in the structure """ result = {} for k, v in six.iteritems(kwargs): if not repeated.isrepeating(v): result[k] = v else: expanded_value = [] for item in v: if structured.isstructured(item): members = structured.getmembers(item) if len(members) == 1 or k in members: # A single column in the subquery - just # use that as the arg value. If the name # emitted is the same as the expected arg # name we also just take that one. expanded_value.append( structured.resolve(item, members[0])) continue expanded_value.append(item) result[k] = expanded_value return result
(name for name, _ in s.getproperties())) }) # This lets us get indices out of Arrays. associative.IAssociative.implement(for_type=obj.Array, implementations={ associative.select: lambda obj, key: obj[key], }) # This lets us do some_array.some_member. structured.IStructured.implement(for_type=obj.Array, implementations={structured.resolve: getattr}) # Pointer[key] is implemented as Pointer.dereference()[key]. associative.IAssociative.implement( for_type=obj.Pointer, implementations={ associative.select: lambda ptr, key: associative.select(ptr.deref(), key), }) # Pointer.member is implemented as Pointer.dereference().member. structured.IStructured.implement( for_type=obj.Pointer, implementations={ structured.resolve: lambda ptr, key: structured.resolve(ptr.deref(), key) })
) # This lets us do some_array.some_member. structured.IStructured.implement( for_type=obj.Array, implementations={ structured.resolve: getattr } ) # Pointer[key] is implemented as Pointer.dereference()[key]. associative.IAssociative.implement( for_type=obj.Pointer, implementations={ associative.select: lambda ptr, key: associative.select(ptr.deref(), key), } ) # Pointer.member is implemented as Pointer.dereference().member. structured.IStructured.implement( for_type=obj.Pointer, implementations={ structured.resolve: lambda ptr, key: structured.resolve(ptr.deref(), key) } )