def testTwoRegexMatchConditionsWithDifferentActions2(self):
expected_files = ["auth.log"]
non_expected_files = ["dpkg.log", "dpkg_false.log"]
regex_condition1 = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=(
rdf_file_finder.FileFinderContentsRegexMatchCondition(
mode="ALL_HITS",
bytes_before=10,
bytes_after=10,
regex=b"session opened for user .*?john")))
regex_condition2 = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=(
rdf_file_finder.FileFinderContentsRegexMatchCondition(
mode="FIRST_HIT", bytes_before=10, bytes_after=10,
regex=b".*")))
for action in self.CONDITION_TESTS_ACTIONS:
results = self.RunFlowAndCheckResults(
action=action,
conditions=[regex_condition1, regex_condition2],
expected_files=expected_files,
non_expected_files=non_expected_files)
self.assertLen(results, 1)
self.assertLen(results[0].matches, 2)
self.assertEqual(results[0].matches[0].offset, 350)
self.assertEqual(results[0].matches[0].data,
b"session): session opened for user dearjohn by (uid=0")
self.assertEqual(results[0].matches[1].offset, 0)
self.assertEqual(results[0].matches[1].length, 770)
def _ConditionsToFileFinderConditions(conditions):
"""Converts FileFinderSizeConditions to RegistryFinderConditions."""
ff_condition_type_cls = rdf_file_finder.FileFinderCondition.Type
result = []
for c in conditions:
if c.condition_type == RegistryFinderCondition.Type.MODIFICATION_TIME:
result.append(
rdf_file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.MODIFICATION_TIME,
modification_time=c.modification_time))
elif c.condition_type == RegistryFinderCondition.Type.VALUE_LITERAL_MATCH:
result.append(
rdf_file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.
CONTENTS_LITERAL_MATCH,
contents_literal_match=c.value_literal_match))
elif c.condition_type == RegistryFinderCondition.Type.VALUE_REGEX_MATCH:
result.append(
rdf_file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.CONTENTS_REGEX_MATCH,
contents_regex_match=c.value_regex_match))
elif c.condition_type == RegistryFinderCondition.Type.SIZE:
result.append(
rdf_file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.SIZE, size=c.size))
else:
raise ValueError("Unknown condition type: %s" % c.condition_type)
return result
def testRegexMatchConditionWithDifferentActions(self):
expected_files = ["auth.log"]
non_expected_files = ["dpkg.log", "dpkg_false.log"]
regex_condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=(
rdf_file_finder.FileFinderContentsRegexMatchCondition(
mode="ALL_HITS",
bytes_before=10,
bytes_after=10,
regex="session opened for user .*?john")))
for action in self.CONDITION_TESTS_ACTIONS:
self.RunFlowAndCheckResults(action=action,
conditions=[regex_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
fd = flow.GRRFlow.ResultCollectionForFID(self.last_session_id)
self.assertEqual(len(fd), 1)
self.assertEqual(len(fd[0].matches), 1)
self.assertEqual(fd[0].matches[0].offset, 350)
self.assertEqual(
fd[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
def testLiteralMatchConditionWithHexEncodedValue(self):
match = rdf_file_finder.FileFinderContentsLiteralMatchCondition(
mode=rdf_file_finder.FileFinderContentsLiteralMatchCondition.Mode.
FIRST_HIT,
bytes_before=10,
bytes_after=10,
literal="\x4D\x5A\x90")
literal_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=match)
paths = [os.path.join(os.path.dirname(self.fixture_path), "hello.exe")]
session_id = flow_test_lib.TestFlowHelper(
file_finder.FileFinder.__name__,
self.client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.OS,
conditions=[literal_condition],
token=self.token)
# Check that the results' matches fields are correctly filled. Expecting a
# match from hello.exe
fd = flow.GRRFlow.ResultCollectionForFID(session_id)
self.assertEqual(len(fd[0].matches), 1)
self.assertEqual(fd[0].matches[0].offset, 0)
self.assertEqual(fd[0].matches[0].data,
"MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff")
def testLiteralMatchConditionWithDifferentActions(self):
expected_files = ["auth.log"]
non_expected_files = ["dpkg.log", "dpkg_false.log"]
match = rdf_file_finder.FileFinderContentsLiteralMatchCondition(
mode=rdf_file_finder.FileFinderContentsLiteralMatchCondition.Mode.
ALL_HITS,
bytes_before=10,
bytes_after=10,
literal="session opened for user dearjohn")
literal_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=match)
for action in self.CONDITION_TESTS_ACTIONS:
results = self.RunFlowAndCheckResults(
action=action,
conditions=[literal_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
# Check that the results' matches fields are correctly filled.
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].matches), 1)
self.assertEqual(results[0].matches[0].offset, 350)
self.assertEqual(
results[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
def testLiteralMatchConditionWithHexEncodedValue(self):
match = rdf_file_finder.FileFinderContentsLiteralMatchCondition(
mode=rdf_file_finder.FileFinderContentsLiteralMatchCondition.Mode.
FIRST_HIT,
bytes_before=10,
bytes_after=10,
literal=b"\x4D\x5A\x90")
literal_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=match)
paths = [
os.path.join(os.path.dirname(self.fixture_path), "win_hello.exe")
]
results = self.RunFlow(paths=paths, conditions=[literal_condition])
# Check that the results' matches fields are correctly filled. Expecting a
# match from win_hello.exe
self.assertLen(results, 1)
self.assertLen(results[0].matches, 1)
self.assertEqual(results[0].matches[0].offset, 0)
self.assertEqual(results[0].matches[0].data,
b"MZ\x90\x00\x03\x00\x00\x00\x04\x00\x00\x00\xff")
def StartRequests(self):
"""Generate and send the Find requests."""
client = aff4.FACTORY.Open(self.client_id, token=self.token)
usernames = [
"%s\\%s" % (u.userdomain, u.username) for u in self.state.users
]
usernames = [u.lstrip("\\")
for u in usernames] # Strip \\ if no domain.
condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=rdf_file_finder.
FileFinderContentsRegexMatchCondition(
regex=self.args.data_regex,
mode=rdf_file_finder.FileFinderContentsRegexMatchCondition.
Mode.FIRST_HIT))
for path in self.state.all_paths:
full_paths = flow_utils.InterpolatePath(path,
client,
users=usernames)
for full_path in full_paths:
self.CallFlow(
file_finder.FileFinder.__name__,
paths=[os.path.join(full_path, "**5")],
pathtype=self.args.pathtype,
conditions=[condition],
action=rdf_file_finder.FileFinderAction.Download(),
next_state="HandleResults")
def testRange(self):
params = rdf_file_finder.FileFinderCondition()
params.size.min_file_size = 2
params.size.max_file_size = 6
condition = conditions.SizeCondition(params)
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"1")
self.assertFalse(condition.Check(self.Stat()))
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"12")
self.assertTrue(condition.Check(self.Stat()))
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"1234")
self.assertTrue(condition.Check(self.Stat()))
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"123456")
self.assertTrue(condition.Check(self.Stat()))
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"1234567")
self.assertFalse(condition.Check(self.Stat()))
def _CreateHuntFromFlow(self):
self.client_id = self.SetupClient(0)
flow_args = rdf_file_finder.FileFinderArgs(
paths=["a/*", "b/*"],
action=rdf_file_finder.FileFinderAction(action_type="STAT"))
flow_runner_args = rdf_flow_runner.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__)
flow_urn = flow.StartAFF4Flow(
client_id=self.client_id,
args=flow_args,
runner_args=flow_runner_args,
token=self.token)
ref = rdf_hunts.FlowLikeObjectReference.FromFlowIdAndClientId(
flow_urn.Basename(), self.client_id.Basename())
# Modify flow_args so that there are differences.
flow_args.paths = ["b/*", "c/*"]
flow_args.action.action_type = "DOWNLOAD"
flow_args.conditions = [
rdf_file_finder.FileFinderCondition(
condition_type="SIZE",
size=rdf_file_finder.FileFinderSizeCondition(min_file_size=42))
]
return self.CreateHunt(
flow_args=flow_args,
flow_runner_args=flow_runner_args,
original_object=ref), flow_urn
def Grep(self, source, pathtype):
"""Grep files in paths for any matches to content_regex_list.
Args:
source: artifact source
pathtype: pathspec path type
When multiple regexes are supplied, combine them into a single regex as an
OR match so that we check all regexes at once.
"""
path_list = self.InterpolateList(source.attributes.get("paths", []))
content_regex_list = self.InterpolateList(
source.attributes.get("content_regex_list", []))
regex_condition = rdf_file_finder.FileFinderContentsRegexMatchCondition(
regex=self._CombineRegex(content_regex_list),
bytes_before=0,
bytes_after=0,
mode="ALL_HITS")
file_finder_condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=regex_condition)
self.CallFlow(file_finder.FileFinder.__name__,
paths=path_list,
conditions=[file_finder_condition],
action=rdf_file_finder.FileFinderAction(),
pathtype=pathtype,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
def testNoMatchOsxBitsUnset(self):
params = rdf_file_finder.FileFinderCondition()
params.ext_flags.osx_bits_unset = self.UF_NODUMP | self.UF_HIDDEN
condition = conditions.ExtFlagsCondition(params)
self._Chflags(["hidden"])
self.assertFalse(condition.Check(self.Stat()))
def testNoMatchOsxBitsSet(self):
params = rdf_file_finder.FileFinderCondition()
params.ext_flags.osx_bits_set = self.UF_IMMUTABLE | self.UF_NODUMP
condition = conditions.ExtFlagsCondition(params)
self._Chflags(["nodump"])
self.assertFalse(condition.Check(self.Stat()))
def testMatchLinuxBitsUnset(self):
params = rdf_file_finder.FileFinderCondition()
params.ext_flags.linux_bits_unset = self.FS_IMMUTABLE_FL
condition = conditions.ExtFlagsCondition(params)
self._Chattr(["+c", "+d"])
self.assertTrue(condition.Check(self.Stat()))
def testDefault(self):
params = rdf_file_finder.FileFinderCondition()
condition = conditions.AccessTimeCondition(params)
self.Touch("-a", "241007151200") # 2410-07-15 12:00
self.assertTrue(condition.Check(self.Stat()))
self.Touch("-a", "201005160745") # 2010-05-16 7:45
self.assertTrue(condition.Check(self.Stat()))
def testDefault(self):
params = rdf_file_finder.FileFinderCondition()
condition = conditions.ModificationTimeCondition(params)
self.Touch("-m", "198309121200") # 1983-09-12 12:00
self.assertTrue(condition.Check(self.Stat()))
self.Touch("-m", "201710020815") # 2017-10-02 8:15
self.assertTrue(condition.Check(self.Stat()))
def Start(self):
"""Redirect to start on the workers and not in the UI."""
# Figure out which paths we are going to check.
if data_store.RelationalDBReadEnabled():
client = data_store.REL_DB.ReadClientSnapshot(self.client_id)
kb = client.knowledge_base
system = kb.os
else:
client = aff4.FACTORY.Open(self.client_id, token=self.token)
system = client.Get(client.Schema.SYSTEM)
kb = client.Get(client.Schema.KNOWLEDGE_BASE)
paths = BROWSER_PATHS.get(system)
self.state.all_paths = []
if self.args.check_chrome:
self.state.all_paths += paths.get("Chrome", [])
if self.args.check_ie:
self.state.all_paths += paths.get("IE", [])
if self.args.check_firefox:
self.state.all_paths += paths.get("Firefox", [])
if not self.state.all_paths:
raise flow.FlowError("Unsupported system %s for CacheGrep" %
system)
self.state.users = []
for user in self.args.grep_users:
user_info = flow_utils.GetUserInfo(kb, user)
if not user_info:
raise flow.FlowError("No such user %s" % user)
self.state.users.append(user_info)
usernames = [
"%s\\%s" % (u.userdomain, u.username) for u in self.state.users
]
usernames = [u.lstrip("\\")
for u in usernames] # Strip \\ if no domain.
condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=rdf_file_finder.
FileFinderContentsRegexMatchCondition(
regex=self.args.data_regex,
mode=rdf_file_finder.FileFinderContentsRegexMatchCondition.
Mode.FIRST_HIT))
for path in self.state.all_paths:
full_paths = flow_utils.InterpolatePath(path, kb, users=usernames)
for full_path in full_paths:
self.CallFlow(
file_finder.FileFinder.__name__,
paths=[os.path.join(full_path, "**5")],
pathtype=self.args.pathtype,
conditions=[condition],
action=rdf_file_finder.FileFinderAction.Download(),
next_state="HandleResults")
def testMatchLinuxBitsMixed(self):
params = rdf_file_finder.FileFinderCondition()
params.ext_flags.linux_bits_set = self.FS_NODUMP_FL
params.ext_flags.linux_bits_unset = self.FS_COMPR_FL
params.ext_flags.osx_bits_unset = self.UF_IMMUTABLE
condition = conditions.ExtFlagsCondition(params)
self._Chattr(["+d"])
self.assertTrue(condition.Check(self.Stat()))
def testNoHits(self):
with open(self.temp_filepath, "wb") as fd:
fd.write("foo bar quux")
params = rdf_file_finder.FileFinderCondition()
params.contents_literal_match.literal = b"baz"
params.contents_literal_match.mode = "ALL_HITS"
condition = conditions.LiteralMatchCondition(params)
results = list(condition.Search(self.temp_filepath))
self.assertFalse(results)
def testNoHits(self):
with open(self.temp_filepath, "wb") as fd:
fd.write("foo bar quux")
params = rdf_file_finder.FileFinderCondition()
params.contents_regex_match.regex = "\\d+"
params.contents_regex_match.mode = "FIRST_HIT"
condition = conditions.RegexMatchCondition(params)
results = list(condition.Search(self.temp_filepath))
self.assertFalse(results)
def testDefault(self):
params = rdf_file_finder.FileFinderCondition()
condition = conditions.SizeCondition(params)
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"1234567")
self.assertTrue(condition.Check(self.Stat()))
with io.open(self.temp_filepath, "wb") as fd:
fd.write(b"")
self.assertTrue(condition.Check(self.Stat()))
def testSizeAndRegexConditionsWithDifferentActions(self):
files_over_size_limit = ["auth.log"]
filtered_files = ["dpkg.log", "dpkg_false.log"]
expected_files = []
non_expected_files = files_over_size_limit + filtered_files
sizes = [
os.stat(os.path.join(self.fixture_path, f)).st_size
for f in files_over_size_limit
]
size_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE,
size=rdf_file_finder.FileFinderSizeCondition(
max_file_size=min(sizes) - 1))
regex_condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=rdf_file_finder.
FileFinderContentsRegexMatchCondition(
mode=(rdf_file_finder.FileFinderContentsRegexMatchCondition.
Mode.ALL_HITS),
bytes_before=10,
bytes_after=10,
regex="session opened for user .*?john"))
for action in self.CONDITION_TESTS_ACTIONS:
self.RunFlowAndCheckResults(
action=action,
conditions=[size_condition, regex_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
# Check that order of conditions doesn't influence results
for action in self.CONDITION_TESTS_ACTIONS:
self.RunFlowAndCheckResults(
action=action,
conditions=[regex_condition, size_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
def testMinTime(self):
time = rdfvalue.RDFDatetime.FromHumanReadable("2017-12-24 19:00:00")
params = rdf_file_finder.FileFinderCondition()
params.modification_time.min_last_modified_time = time
condition = conditions.ModificationTimeCondition(params)
self.Touch("-m", "201712240100") # 2017-12-24 1:30
self.assertFalse(condition.Check(self.Stat()))
self.Touch("-m", "201806141700") # 2018-06-14 17:00
self.assertTrue(condition.Check(self.Stat()))
def testMaxTime(self):
time = rdfvalue.RDFDatetime.FromHumanReadable("2125-12-28 18:45")
params = rdf_file_finder.FileFinderCondition()
params.modification_time.max_last_modified_time = time
condition = conditions.ModificationTimeCondition(params)
self.Touch("-m", "211811111200") # 2118-11-11 12:00
self.assertTrue(condition.Check(self.Stat()))
self.Touch("-m", "222510201500") # 2225-10-20 15:00
self.assertFalse(condition.Check(self.Stat()))
def testMatchOsxBitsUnset(self):
params = rdf_file_finder.FileFinderCondition()
params.ext_flags.osx_bits_unset = self.UF_NODUMP | self.UF_IMMUTABLE
condition = conditions.ExtFlagsCondition(params)
self._Chflags(["hidden", "uappend"])
try:
self.assertTrue(condition.Check(self.Stat()))
finally:
# Make the test file deletable.
self._Chflags(["nouappend"])
def testFirstHit(self):
with open(self.temp_filepath, "wb") as fd:
fd.write("bar foo baz foo")
params = rdf_file_finder.FileFinderCondition()
params.contents_literal_match.literal = b"foo"
params.contents_literal_match.mode = "FIRST_HIT"
condition = conditions.LiteralMatchCondition(params)
results = list(condition.Search(self.temp_filepath))
self.assertEqual(len(results), 1)
self.assertEqual(results[0].data, "foo")
self.assertEqual(results[0].offset, 4)
self.assertEqual(results[0].length, 3)
def testFirstHit(self):
with open(self.temp_filepath, "wb") as fd:
fd.write("4 8 15 16 23 42 foo 108 bar")
params = rdf_file_finder.FileFinderCondition()
params.contents_regex_match.regex = "[a-z]+"
params.contents_regex_match.mode = "FIRST_HIT"
condition = conditions.RegexMatchCondition(params)
results = list(condition.Search(self.temp_filepath))
self.assertEqual(len(results), 1)
self.assertEqual(results[0].data, "foo")
self.assertEqual(results[0].offset, 16)
self.assertEqual(results[0].length, 3)
def testMatchOsxBitsMixed(self):
params = rdf_file_finder.FileFinderCondition()
params.ext_flags.osx_bits_set = self.UF_NODUMP
params.ext_flags.osx_bits_unset = self.UF_HIDDEN
params.ext_flags.linux_bits_unset = self.FS_NODUMP_FL
condition = conditions.ExtFlagsCondition(params)
self._Chflags(["nodump", "uappend"])
try:
self.assertTrue(condition.Check(self.Stat()))
finally:
# Make the test file deletable.
self._Chflags(["nouappend"])
def Grep(self, source, pathtype, implementation_type):
"""Grep files in paths for any matches to content_regex_list.
When multiple regexes are supplied, combine
them into a single regex as an OR match so that we check all regexes at
once.
Args:
source: artifact source
pathtype: pathspec path typed
implementation_type: Pathspec implementation type to use.
"""
path_list = self.InterpolateList(source.attributes.get("paths", []))
# `content_regex_list` elements should be binary strings, but forcing
# artifact creators to use verbose YAML syntax for binary literals would
# be cruel. Therefore, we allow both kind of strings and we convert to bytes
# if required.
content_regex_list = []
for content_regex in source.attributes.get("content_regex_list", []):
if isinstance(content_regex, Text):
content_regex = content_regex.encode("utf-8")
content_regex_list.append(content_regex)
content_regex_list = self.InterpolateList(content_regex_list)
regex_condition = rdf_file_finder.FileFinderContentsRegexMatchCondition(
regex=self._CombineRegex(content_regex_list),
bytes_before=0,
bytes_after=0,
mode="ALL_HITS")
file_finder_condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=regex_condition)
self.CallFlow(
file_finder.FileFinder.__name__,
paths=path_list,
conditions=[file_finder_condition],
action=rdf_file_finder.FileFinderAction(),
pathtype=pathtype,
implementation_type=implementation_type,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state=compatibility.GetName(self.ProcessCollected))
def testStartOffset(self):
with open(self.temp_filepath, "wb") as fd:
fd.write("ooooooo")
params = rdf_file_finder.FileFinderCondition()
params.contents_regex_match.regex = "o+"
params.contents_regex_match.mode = "FIRST_HIT"
params.contents_regex_match.start_offset = 3
condition = conditions.RegexMatchCondition(params)
results = list(condition.Search(self.temp_filepath))
self.assertEqual(len(results), 1)
self.assertEqual(results[0].data, "oooo")
self.assertEqual(results[0].offset, 3)
self.assertEqual(results[0].length, 4)
def testAccessTimeConditionWithDifferentActions(self):
expected_files = ["dpkg.log", "dpkg_false.log"]
non_expected_files = ["auth.log"]
change_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1444444440)
access_time_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.
ACCESS_TIME,
access_time=rdf_file_finder.FileFinderAccessTimeCondition(
min_last_access_time=change_time))
for action in self.CONDITION_TESTS_ACTIONS:
self.RunFlowAndCheckResults(action=action,
conditions=[access_time_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
