def testReverseLists(self):
"""Test that reverse supports both lists and IRepeated."""
# "lists" are actually Python tuples.
self.assertEqual(api.apply("reverse([1, 2, 3])"), (3, 2, 1))
self.assertEqual(api.apply("reverse((1, 2, 3))"),
repeated.meld(3, 2, 1))
def testReverseLists(self):
"""Test that reverse supports both lists and IRepeated."""
# "lists" are actually Python tuples.
self.assertEqual(api.apply("reverse([1, 2, 3])"), (3, 2, 1))
self.assertEqual(api.apply("reverse((1, 2, 3))"),
repeated.meld(3, 2, 1))
def testLet(self):
self.assertEqual(
api.apply("let(x = 5, y = 10 * 2) x + y"),
25)
self.assertEqual(
api.apply("let(x = 5, y = 10 * x, z = (x + y)) z - y"),
5)
def testDropAndTake(self):
"""Test that dropping and taking works properly."""
self.assertValuesEqual(api.apply("drop(2, (1, 2, 3, 4))"),
repeated.meld(3, 4))
self.assertValuesEqual(api.apply("drop(3, (1, 2, 3, 4))"), 4)
self.assertValuesEqual(api.apply("take(1, drop(2, (1, 2, 3, 4)))"), 3)
# Alternate syntax to do the same thing.
self.assertValuesEqual(
api.apply("SELECT * FROM (1, 2, 3, 4) LIMIT 1 OFFSET 2"), 3)
def testCountLists(self):
"""Test that count supports lists and IRepeated."""
self.assertEqual(api.apply("count((1, 2, 3))"), 3)
# Lists should work.
self.assertEqual(api.apply("count([1, 2, 3])"), 3)
# IRepeated are flat.
self.assertEqual(api.apply("count((1, (2, 3)))"), 3)
# Lists are not.
self.assertEqual(api.apply("count([1, [2, 3]])"), 2)
def testCountLists(self):
"""Test that count supports lists and IRepeated."""
self.assertEqual(api.apply("count((1, 2, 3))"), 3)
# Lists should work.
self.assertEqual(api.apply("count([1, 2, 3])"), 3)
# IRepeated are flat.
self.assertEqual(api.apply("count((1, (2, 3)))"), 3)
# Lists are not.
self.assertEqual(api.apply("count([1, [2, 3]])"), 2)
def testUserFunc(self):
with self.assertRaises(errors.EfilterKeyError):
api.apply("my_func(1, 5)")
def my_func(x, y):
return x + y
with self.assertRaises(NotImplementedError):
api.apply("my_func(1, 5)", vars={"my_func": my_func})
# Need to define it as a user_func!
result = api.apply("my_func(1, 5)",
vars={"my_func": api.user_func(my_func)})
self.assertEqual(result, 6)
def testScalarFunc(self):
# Define numeric addition.
def my_func(x, y):
return x + y
# Two scalars.
result = api.apply("my_func(1, 5)",
vars={
"my_func":
api.scalar_function(
my_func, (number.INumber, number.INumber))
})
self.assertEqual(result, [6])
# Scalar is repeated to length of vector.
result = api.apply("my_func(1, [1, 5])",
vars={
"my_func":
api.scalar_function(
my_func, (number.INumber, number.INumber))
})
self.assertEqual(result, [2, 6])
# If a type is non numeric then skip calling the function.
result = api.apply("my_func(1, [1, 'a'])",
vars={
"my_func":
api.scalar_function(
my_func, (number.INumber, number.INumber))
})
self.assertEqual(result, [2, None])
# Two vectors are added one element at a time.
result = api.apply("my_func([1, 2], [1, 5])",
vars={
"my_func":
api.scalar_function(
my_func, (number.INumber, number.INumber))
})
self.assertEqual(result, [2, 7])
# Short vectors are padded to the longest vector.
result = api.apply("my_func([1, 2], [1, 5, 7])",
vars={
"my_func":
api.scalar_function(
my_func, (number.INumber, number.INumber))
})
self.assertEqual(result, [2, 7, 7])
def testUserFunc(self):
with self.assertRaises(errors.EfilterKeyError):
api.apply("my_func(1, 5)")
def my_func(x, y):
return x + y
with self.assertRaises(NotImplementedError):
api.apply("my_func(1, 5)",
vars={"my_func": my_func})
# Need to define it as a user_func!
result = api.apply("my_func(1, 5)",
vars={"my_func": api.user_func(my_func)})
self.assertEqual(result, 6)
def testDropAndTake(self):
"""Test that dropping and taking works properly."""
self.assertValuesEqual(
api.apply("drop(2, (1, 2, 3, 4))"),
repeated.meld(3, 4))
self.assertValuesEqual(
api.apply("drop(3, (1, 2, 3, 4))"), 4)
self.assertValuesEqual(
api.apply("take(1, drop(2, (1, 2, 3, 4)))"), 3)
# Alternate syntax to do the same thing.
self.assertValuesEqual(
api.apply("SELECT * FROM (1, 2, 3, 4) LIMIT 1 OFFSET 2"), 3)
def testGroup(self):
result = api.apply(
query=q.Query(
(
"group",
# The input:
("apply", ("var", "csv"), ("param", 0), True),
# The grouper expression:
("var", "country"),
# The output reducers:
("reducer", ("var", "singleton"), ("var", "country")),
("reducer", ("var", "mean"), ("cast", ("var", "age"),
("var", "int"))),
("reducer", ("var", "sum"), ("cast", ("var", "age"),
("var", "int")))),
params=[testlib.get_fixture_path("fake_users.csv")]),
allow_io=True)
# Round the output means for comparison.
actual = []
for row in result:
row[1] = int(row[1])
actual.append(row)
expected = repeated.meld(['El Salvador', 55, 1287],
['Ethiopia', 55, 1210],
['French Guiana', 47, 381],
['Germany', 42, 299], ['Haiti', 46, 610],
['Mayotte', 50, 865], ['Portugal', 48, 485])
self.assertItemsEqual(expected, actual)
def ExamineEvent(self, mediator, event):
"""Analyzes an EventObject and tags it according to rules in the tag file.
Args:
mediator (AnalysisMediator): mediates interactions between analysis
plugins and other components, such as storage and dfvfs.
event (EventObject): event to examine.
"""
if self._tag_rules is None:
if self._autodetect_tag_file_attempt:
# There's nothing to tag with, and we've already tried to find a good
# tag file, so there's nothing we can do with this event (or any other).
return
if not self._AttemptAutoDetectTagFile(mediator):
logging.info(
u'No tag definition file specified, and plaso was not able to '
u'autoselect a tagging file. As no definitions were specified, '
u'no events will be tagged.')
return
try:
matched_labels = efilter_api.apply(self._tag_rules, vars=event)
except efilter_errors.EfilterTypeError as exception:
logging.warning(
u'Unable to apply efilter query with error: {0:s}'.format(
exception))
matched_labels = None
if not matched_labels:
return
labels = list(efilter_api.getvalues(matched_labels))
event_tag = self._CreateEventTag(event, self._EVENT_TAG_COMMENT,
labels)
mediator.ProduceEventTag(event_tag)
def ExamineEvent(self, analysis_mediator, event_object, **kwargs):
"""Analyzes an EventObject and tags it according to rules in the tag file.
Args:
analysis_mediator: The analysis mediator object (instance of
AnalysisMediator).
event_object: The event object (instance of EventObject) to examine.
"""
if self._tag_rules is None:
if self._autodetect_tag_file_attempt:
# There's nothing to tag with, and we've already tried to find a good
# tag file, so there's nothing we can do with this event (or any other).
return
if not self._AttemptAutoDetectTagFile(analysis_mediator):
logging.info(
u'No tag definition file specified, and plaso was not able to '
u'autoselect a tagging file. As no definitions were specified, '
u'no events will be tagged.')
return
matched_labels = efilter_api.apply(self._tag_rules, vars=event_object)
if not matched_labels:
return
event_uuid = getattr(event_object, u'uuid')
event_tag = events.EventTag(
comment=u'Tag applied by tagging analysis plugin.',
event_uuid=event_uuid)
for label in efilter_api.getvalues(matched_labels):
event_tag.AddLabel(label)
logging.debug(u'Tagging event: {0!s}'.format(event_uuid))
self._tags.append(event_tag)
def ExamineEvent(self, mediator, event):
"""Analyzes an EventObject and tags it according to rules in the tag file.
Args:
mediator (AnalysisMediator): mediates interactions between analysis
plugins and other components, such as storage and dfvfs.
event (EventObject): event to examine.
"""
if self._tag_rules is None:
if self._autodetect_tag_file_attempt:
# There's nothing to tag with, and we've already tried to find a good
# tag file, so there's nothing we can do with this event (or any other).
return
if not self._AttemptAutoDetectTagFile(mediator):
logging.info(
u'No tag definition file specified, and plaso was not able to '
u'autoselect a tagging file. As no definitions were specified, '
u'no events will be tagged.')
return
try:
matched_labels = efilter_api.apply(self._tag_rules, vars=event)
except efilter_errors.EfilterTypeError as exception:
logging.warning(u'Unable to apply efilter query with error: {0:s}'.format(
exception))
matched_labels = None
if not matched_labels:
return
labels = list(efilter_api.getvalues(matched_labels))
event_tag = self._CreateEventTag(event, self._EVENT_TAG_COMMENT, labels)
mediator.ProduceEventTag(event_tag)
def main():
for description, query in QUERIES:
print("# %s\n%s" % (description, query))
# We can find out what the EFILTER query will return by using the type
# inference system. If it is a repeated value, we can render it in
# multiple rows.
result_type = api.infer(query,
replacements=[CATALOG_PATH],
libs=("stdcore", "stdio"))
print("# Return type will be %s." % (result_type.__name__,))
# api.apply will give us the actual result of running the query, which
# should be of the type we got above.
results = api.apply(query,
replacements=[CATALOG_PATH],
allow_io=True,
# We provide the top level variables in a 'vars'
# argument. To bind 'parsec2ly' to the function of
# the same name, we have to also wrap it in the
# EFILTER user_func. This prevents EFILTER from
# accidentally calling regular Python functions.
vars={"parsec2ly": api.user_func(parsec2ly)})
# Because we don't know the cardinality of the query in 'query' we can
# use 'getvalues' to always receive an iterator of results. This is just
# a convenience function.
for n, result in enumerate(api.getvalues(results)):
print("%d - %r" % (n + 1, result))
print("\n\n")
def testReducer(self):
# This should return a reducer that computes the mean of the age
# property on a repeated object (tests let us use a dict as a stand-in).
r = api.apply(("reducer", ("var", "mean"), ("var", "age")))
self.assertIsInstance(r, reducer.IReducer)
users = repeated.meld({"name": "Mary", "age": 10},
{"name": "Bob", "age": 20})
average = reducer.reduce(r, users)
self.assertEqual(average, 15)
def testApply(self):
self.assertValuesEqual(
api.apply("select age from data where name == 'Peter'",
vars=dict(data=[dict(name="Peter", age=20),
dict(name="Paul", age=30)])),
dict(age=20))
self.assertValuesEqual(
api.apply("select * from data where name == 'Peter'",
vars=dict(data=[dict(name="Peter", age=20),
dict(name="Paul", age=30)])),
dict(age=20, name="Peter"))
self.assertValuesEqual(
api.apply("select * from data where name == ?",
vars=dict(data=[dict(name="Peter", age=20),
dict(name="Paul", age=30)]),
replacements=["Peter"]),
dict(age=20, name="Peter"))
def testApply(self):
self.assertValuesEqual(
api.apply("select age from data where name == 'Peter'",
vars=dict(data=[dict(name="Peter", age=20),
dict(name="Paul", age=30)])),
dict(age=20))
self.assertValuesEqual(
api.apply("select * from data where name == 'Peter'",
vars=dict(data=[dict(name="Peter", age=20),
dict(name="Paul", age=30)])),
dict(age=20, name="Peter"))
self.assertValuesEqual(
api.apply("select * from data where name == ?",
vars=dict(data=[dict(name="Peter", age=20),
dict(name="Paul", age=30)]),
replacements=["Peter"]),
dict(age=20, name="Peter"))
def run(self):
replacements = []
if self.fixture_name is not None:
replacements.append(testlib.get_fixture_path(self.fixture_name))
result = api.apply(self.query, replacements=replacements, allow_io=True)
# Force lazy results to be realized, but don't do anything with them.
for _ in repeated.getvalues(result):
pass
def testCountFilter(self):
self.assertEqual(
api.apply("count(select * from people where age > 20)",
vars=dict(people=[{
"age": 10
}, {
"age": 30
}, {
"age": 15
}, {
"age": 35
}])), 2)
def testIOReads(self):
"""Test that IO is properly hooked up when requested."""
self.assertValuesEqual(
api.apply("SELECT * FROM csv(?)",
replacements=[testlib.get_fixture_path("small.csv")],
allow_io=True), repeated.meld(*small_csv.EXPECTED))
# Keyword arguments to 'csv' should work.
result = api.apply(
"SELECT * FROM csv(?, decode_header: true)",
replacements=[testlib.get_fixture_path("small.csv")],
allow_io=True)
first_row = next(iter(result))
self.assertEqual(dict(Name="Alice", Age="25", City="Zurich"),
first_row)
# The FD closes, right?
fd = result.source.fd
result = None
self.assertTrue(fd.closed)
def testIOReads(self):
"""Test that IO is properly hooked up when requested."""
self.assertValuesEqual(
api.apply("SELECT * FROM csv(?)",
replacements=[testlib.get_fixture_path("small.csv")],
allow_io=True),
repeated.meld(*small_csv.EXPECTED))
# Keyword arguments to 'csv' should work.
result = api.apply(
"SELECT * FROM csv(?, decode_header: true)",
replacements=[testlib.get_fixture_path("small.csv")],
allow_io=True)
first_row = next(iter(result))
self.assertEqual(dict(Name="Alice", Age="25", City="Zurich"),
first_row)
# The FD closes, right?
fd = result.source.fd
result = None
self.assertTrue(fd.closed)
def testReducer(self):
# This should return a reducer that computes the mean of the age
# property on a repeated object (tests let us use a dict as a stand-in).
r = api.apply(("reducer", ("var", "mean"), ("var", "age")))
self.assertIsInstance(r, reducer.IReducer)
users = repeated.meld({
"name": "Mary",
"age": 10
}, {
"name": "Bob",
"age": 20
})
average = reducer.reduce(r, users)
self.assertEqual(average, 15)
def testGroup(self):
result = api.apply(
query=q.Query(
("group",
# The input:
("apply",
("var", "csv"),
("param", 0),
True),
# The grouper expression:
("var", "country"),
# The output reducers:
("reducer",
("var", "singleton"),
("var", "country")),
("reducer",
("var", "mean"),
("cast",
("var", "age"),
("var", "int"))),
("reducer",
("var", "sum"),
("cast",
("var", "age"),
("var", "int")))),
params=[testlib.get_fixture_path("fake_users.csv")]),
allow_io=True)
# Round the output means for comparison.
actual = []
for row in result:
row[1] = int(row[1])
actual.append(row)
expected = repeated.meld(['El Salvador', 55, 1287],
['Ethiopia', 55, 1210],
['French Guiana', 47, 381],
['Germany', 42, 299],
['Haiti', 46, 610],
['Mayotte', 50, 865],
['Portugal', 48, 485])
self.assertItemsEqual(expected, actual)
def main():
for description, query in QUERIES:
print("# %s\n%s" % (description, query))
# api.apply will give us the actual result of running the query, which
# should be of the type we got above.
results = api.apply(query,
replacements=[CATALOG_PATH],
allow_io=True,
# We provide the top level variables in a 'vars'
# argument. To bind 'parsec2ly' to the function of
# the same name, we have to also wrap it in the
# EFILTER user_func. This prevents EFILTER from
# accidentally calling regular Python functions.
vars={"parsec2ly": api.user_func(parsec2ly)})
# Because we don't know the cardinality of the query in 'query' we can
# use 'getvalues' to always receive an iterator of results. This is just
# a convenience function.
for n, result in enumerate(api.getvalues(results)):
print("%d - %r" % (n + 1, result))
print("\n\n")
def main():
parser = argparse.ArgumentParser(description="Convert a tafile to DottySQL")
parser.add_argument("path", type=str)
args = parser.parse_args()
with open(args.path, "r") as fd:
tag_rules = query.Query(fd, syntax="tagfile")
# What does the query look like as DottySQL?
dottysql = asdottysql.asdottysql(tag_rules)
print("# Tagfile %r converted:\n\n%s" % (args.path, dottysql))
# How will the query tag this event?
event = {
"data_type": "windows:evtx:record",
"timestamp_desc": "",
"strings": ("foo", "bar"),
"source_name": "Microsoft-Windows-Kernel-Power",
"event_identifier": 42
}
tags = api.apply(tag_rules, vars=event)
print("\n# Tagfile %r returned %r." % (args.path, list(tags)))
def main():
parser = argparse.ArgumentParser(
description="Convert a tafile to DottySQL")
parser.add_argument("path", type=str)
args = parser.parse_args()
with open(args.path, "r") as fd:
tag_rules = query.Query(fd, syntax="tagfile")
# What does the query look like as DottySQL?
dottysql = asdottysql.asdottysql(tag_rules)
print("# Tagfile %r converted:\n\n%s" % (args.path, dottysql))
# How will the query tag this event?
event = {
"data_type": "windows:evtx:record",
"timestamp_desc": "",
"strings": ("foo", "bar"),
"source_name": "Microsoft-Windows-Kernel-Power",
"event_identifier": 42
}
tags = api.apply(tag_rules, vars=event)
print("\n# Tagfile %r returned %r." % (args.path, list(tags)))
def testFuncCalls(self):
"""Test that function calls are completed."""
self.assertEqual(api.apply("count((1, 2, 3))"), 3)
def testIOMustBeRequested(self):
"""Test that IO isn't available unless requested."""
with self.assertRaises(errors.EfilterError):
api.apply("SELECT * FROM csv(?)",
replacements=[testlib.get_fixture_path("small.csv")])
def testIOMustBeRequested(self):
"""Test that IO isn't available unless requested."""
with self.assertRaises(errors.EfilterError):
api.apply("SELECT * FROM csv(?)",
replacements=[testlib.get_fixture_path("small.csv")])
def testFuncCalls(self):
"""Test that function calls are completed."""
self.assertEqual(api.apply("count((1, 2, 3))"), 3)
def testCountFilter(self):
self.assertEqual(
api.apply("count(select * from people where age > 20)",
vars=dict(people=[{"age": 10}, {"age": 30}, {"age": 15},
{"age": 35}])),
2)