def testTwoRegexMatchConditionsWithDifferentActions2(self):
expected_files = ["auth.log"]
non_expected_files = ["dpkg.log", "dpkg_false.log"]
regex_condition1 = file_finder.FileFinderCondition(
condition_type=(
file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=file_finder.
FileFinderContentsRegexMatchCondition(
mode=(file_finder.FileFinderContentsRegexMatchCondition.Mode.
ALL_HITS),
bytes_before=10,
bytes_after=10,
regex="session opened for user .*?john"))
regex_condition2 = file_finder.FileFinderCondition(
condition_type=(
file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=file_finder.
FileFinderContentsRegexMatchCondition(
mode=file_finder.FileFinderContentsRegexMatchCondition.Mode.
FIRST_HIT,
bytes_before=10,
bytes_after=10,
regex=".*"))
for action in self.CONDITION_TESTS_ACTIONS:
self.RunFlowAndCheckResults(
action=action,
conditions=[regex_condition1, regex_condition2],
expected_files=expected_files,
non_expected_files=non_expected_files)
# Check the output file is created
fd = aff4.FACTORY.Open(
self.last_session_id.Add(flow_runner.RESULTS_SUFFIX),
aff4_type=sequential_collection.GeneralIndexedCollection,
token=self.token)
self.assertEqual(len(fd), 1)
self.assertEqual(len(fd[0].matches), 2)
self.assertEqual(fd[0].matches[0].offset, 350)
self.assertEqual(
fd[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
self.assertEqual(fd[0].matches[1].offset, 0)
self.assertEqual(fd[0].matches[1].length, 770)
def testTwoRegexMatchConditionsWithDifferentActions2(self):
expected_files = ["auth.log"]
non_expected_files = ["dpkg.log", "dpkg_false.log"]
regex_condition1 = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_REGEX_MATCH,
contents_regex_match=file_finder.
FileFinderContentsRegexMatchCondition(
mode=file_finder.FileFinderContentsRegexMatchCondition.Mode.
ALL_HITS,
bytes_before=10,
bytes_after=10,
regex="session opened for user .*?john"))
regex_condition2 = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_REGEX_MATCH,
contents_regex_match=file_finder.
FileFinderContentsRegexMatchCondition(
mode=file_finder.FileFinderContentsRegexMatchCondition.Mode.
FIRST_HIT,
bytes_before=10,
bytes_after=10,
regex=".*"))
for action in sorted(
file_finder.FileFinderAction.Action.enum_dict.values()):
self.RunFlowAndCheckResults(
action=action,
conditions=[regex_condition1, regex_condition2],
expected_files=expected_files,
non_expected_files=non_expected_files)
# Check the output file is created
fd = aff4.FACTORY.Open(self.client_id.Add(self.output_path),
aff4_type="RDFValueCollection",
token=self.token)
self.assertEqual(len(fd), 1)
self.assertEqual(len(fd[0].matches), 2)
self.assertEqual(fd[0].matches[0].offset, 350)
self.assertEqual(
fd[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
self.assertEqual(fd[0].matches[1].offset, 0)
self.assertEqual(fd[0].matches[1].length, 770)
def testSizeAndRegexConditionsWithDifferentActions(self):
files_over_size_limit = ["auth.log"]
filtered_files = ["dpkg.log", "dpkg_false.log"]
expected_files = []
non_expected_files = files_over_size_limit + filtered_files
sizes = [
os.stat(os.path.join(self.fixture_path, f)).st_size
for f in files_over_size_limit
]
size_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=file_finder.FileFinderSizeCondition(max_file_size=min(sizes) -
1))
regex_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_REGEX_MATCH,
contents_regex_match=file_finder.
FileFinderContentsRegexMatchCondition(
mode=file_finder.FileFinderContentsRegexMatchCondition.Mode.
ALL_HITS,
bytes_before=10,
bytes_after=10,
regex="session opened for user .*?john"))
for action in sorted(
file_finder.FileFinderAction.Action.enum_dict.values()):
self.RunFlowAndCheckResults(
action=action,
conditions=[size_condition, regex_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
# Check that order of conditions doesn't influence results
for action in sorted(
file_finder.FileFinderAction.Action.enum_dict.values()):
self.RunFlowAndCheckResults(
action=action,
conditions=[regex_condition, size_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
def ConditionsToFileFinderConditions(self, conditions):
ff_condition_type_cls = file_finder.FileFinderCondition.Type
result = []
for c in conditions:
if c.condition_type == MemoryCollectorCondition.Type.LITERAL_MATCH:
result.append(
file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.
CONTENTS_LITERAL_MATCH,
contents_literal_match=c.literal_match))
elif c.condition_type == MemoryCollectorCondition.Type.REGEX_MATCH:
result.append(
file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.
CONTENTS_REGEX_MATCH,
contents_regex_match=c.regex_match))
else:
raise ValueError("Unknown condition type: %s",
c.condition_type)
return result
def testAppliesLiteralConditionWhenMemoryPathTypeIsUsed(self):
with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
test_lib.FakeTestDataVFSHandler):
with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.MEMORY,
test_lib.FakeTestDataVFSHandler):
paths = [
os.path.join(os.path.dirname(self.base_path), "auth.log"),
os.path.join(os.path.dirname(self.base_path), "dpkg.log")
]
literal_condition = file_finder.FileFinderContentsLiteralMatchCondition
all_hits = literal_condition.Mode.ALL_HITS
literal_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=literal_condition(
mode=all_hits,
literal="session opened for user dearjohn"))
# Check this condition with all the actions. This makes sense, as we may
# download memeory or send it to the socket.
for action in sorted(file_finder.FileFinderAction.Action.
enum_dict.values()):
for _ in test_lib.TestFlowHelper(
"FileFinder",
self.client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.MEMORY,
conditions=[literal_condition],
action=file_finder.FileFinderAction(
action_type=action),
token=self.token,
output=self.output_path):
pass
self.CheckFilesInCollection(["auth.log"])
fd = aff4.FACTORY.Open(self.client_id.Add(
self.output_path),
aff4_type="RDFValueCollection",
token=self.token)
self.assertEqual(fd[0].stat_entry.pathspec.CollapsePath(),
paths[0])
self.assertEqual(len(fd), 1)
self.assertEqual(len(fd[0].matches), 1)
self.assertEqual(fd[0].matches[0].offset, 350)
self.assertEqual(
fd[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
def testAppliesLiteralConditionWhenMemoryPathTypeIsUsed(self):
with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
test_lib.FakeTestDataVFSHandler):
with test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.MEMORY,
test_lib.FakeTestDataVFSHandler):
paths = ["/var/log/auth.log", "/etc/ssh/sshd_config"]
literal_condition = file_finder.FileFinderContentsLiteralMatchCondition
all_hits = literal_condition.Mode.ALL_HITS
literal_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=literal_condition(
mode=all_hits,
bytes_before=10,
bytes_after=10,
literal="session opened for user dearjohn"))
# Check this condition with all the actions. This makes sense, as we may
# download memeory or send it to the socket.
for action in self.CONDITION_TESTS_ACTIONS:
for s in test_lib.TestFlowHelper(
"FileFinder",
self.client_mock,
client_id=self.client_id,
paths=paths,
pathtype=rdf_paths.PathSpec.PathType.MEMORY,
conditions=[literal_condition],
action=file_finder.FileFinderAction(
action_type=action),
token=self.token):
session_id = s
self.CheckFilesInCollection(["auth.log"],
session_id=session_id)
fd = aff4.FACTORY.Open(session_id.Add(
flow_runner.RESULTS_SUFFIX),
aff4_type=sequential_collection.
GeneralIndexedCollection,
token=self.token)
self.assertEqual(fd[0].stat_entry.pathspec.CollapsePath(),
paths[0])
self.assertEqual(len(fd), 1)
self.assertEqual(len(fd[0].matches), 1)
self.assertEqual(fd[0].matches[0].offset, 350)
self.assertEqual(
fd[0].matches[0].data,
"session): session opened for user dearjohn by (uid=0")
def ConditionsToFileFinderConditions(self, conditions):
ff_condition_type_cls = file_finder.FileFinderCondition.Type
result = []
for c in conditions:
if c.condition_type == RegistryFinderCondition.Type.MODIFICATION_TIME:
result.append(file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.MODIFICATION_TIME,
modification_time=c.modification_time))
elif c.condition_type == RegistryFinderCondition.Type.VALUE_LITERAL_MATCH:
result.append(file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.CONTENTS_LITERAL_MATCH,
contents_literal_match=c.value_literal_match))
elif c.condition_type == RegistryFinderCondition.Type.VALUE_REGEX_MATCH:
result.append(file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.CONTENTS_REGEX_MATCH,
contents_regex_match=c.value_regex_match))
elif c.condition_type == RegistryFinderCondition.Type.SIZE:
result.append(file_finder.FileFinderCondition(
condition_type=ff_condition_type_cls.SIZE,
size=c.size))
else:
raise ValueError("Unknown condition type: %s", c.condition_type)
return result
def testAccessTimeConditionWithDifferentActions(self):
expected_files = ["dpkg.log", "dpkg_false.log"]
non_expected_files = ["auth.log"]
change_time = rdfvalue.RDFDatetime().FromSecondsFromEpoch(1444444440)
access_time_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.ACCESS_TIME,
access_time=file_finder.FileFinderAccessTimeCondition(
min_last_access_time=change_time))
for action in self.CONDITION_TESTS_ACTIONS:
self.RunFlowAndCheckResults(action=action,
conditions=[access_time_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
def testModificationTimeConditionWithDifferentActions(self):
expected_files = ["dpkg.log", "dpkg_false.log"]
non_expected_files = ["auth.log"]
change_time = rdfvalue.RDFDatetime().FromSecondsFromEpoch(1444444440)
modification_time_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.MODIFICATION_TIME,
modification_time=file_finder.FileFinderModificationTimeCondition(
min_last_modified_time=change_time))
for action in sorted(file_finder.FileFinderAction.Action.enum_dict.values(
)):
self.RunFlowAndCheckResults(action=action,
conditions=[modification_time_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
def testInodeChangeTimeConditionWithDifferentActions(self):
expected_files = ["dpkg.log", "dpkg_false.log"]
non_expected_files = ["auth.log"]
change_time = rdfvalue.RDFDatetime().FromSecondsFromEpoch(1444444440)
inode_change_time_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.INODE_CHANGE_TIME,
inode_change_time=file_finder.FileFinderInodeChangeTimeCondition(
min_last_inode_change_time=change_time))
for action in sorted(file_finder.FileFinderAction.Action.enum_dict.values(
)):
self.RunFlowAndCheckResults(action=action,
conditions=[inode_change_time_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
def testFileFinderListActionRaisesWithConditions(self):
match = file_finder.FileFinderContentsLiteralMatchCondition(
mode=file_finder.FileFinderContentsLiteralMatchCondition.Mode.
ALL_HITS,
bytes_before=10,
bytes_after=10,
literal="session opened for user dearjohn")
literal_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_LITERAL_MATCH,
contents_literal_match=match)
# RuntimeError: The LIST FileFinder action doesn't support conditions.
with self.assertRaises(RuntimeError):
self.RunFlow(action=file_finder.FileFinderAction.Action.LIST,
conditions=[literal_condition])
def testSizeConditionWithDifferentActions(self):
expected_files = ["dpkg.log", "dpkg_false.log"]
non_expected_files = ["auth.log"]
sizes = [os.stat(os.path.join(self.fixture_path, f)).st_size
for f in expected_files]
size_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=file_finder.FileFinderSizeCondition(max_file_size=max(sizes) + 1))
for action in sorted(file_finder.FileFinderAction.Action.enum_dict.values(
)):
self.RunFlowAndCheckResults(action=action,
conditions=[size_condition],
expected_files=expected_files,
non_expected_files=non_expected_files)
class TestFileFinderOSLinux(base.VFSPathContentIsELF):
"""Download a file with FileFinder."""
platforms = ["Linux"]
flow = "FileFinder"
test_output_path = "/fs/os/bin/ps"
sizecondition = file_finder.FileFinderSizeCondition(max_file_size=1000000)
filecondition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=sizecondition)
download = file_finder.FileFinderDownloadActionOptions()
action = file_finder.FileFinderAction(
action_type=file_finder.FileFinderAction.Action.DOWNLOAD,
download=download)
args = {"paths": ["/bin/ps"],
"conditions": filecondition,
"action": action}
class TestFileFinderOSLinuxProc(base.VFSPathContentExists):
"""Download a /proc/sys entry with FileFinder."""
platforms = ["Linux"]
flow = "FileFinder"
test_output_path = "/fs/os/proc/sys/net/ipv4/ip_forward"
client_min_version = 3007
sizecondition = file_finder.FileFinderSizeCondition(max_file_size=1000000)
filecondition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=sizecondition)
download = file_finder.FileFinderDownloadActionOptions()
action = file_finder.FileFinderAction(
action_type=file_finder.FileFinderAction.Action.DOWNLOAD,
download=download)
args = {"paths": ["/proc/sys/net/ipv4/ip_forward"],
"conditions": filecondition,
"action": action}
def Render(self, args, token=None):
client_urn = self.GetClientTarget(args, token=token)
size_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=file_finder.FileFinderSizeCondition(
max_file_size=args.max_file_size))
file_finder_args = file_finder.FileFinderArgs(
paths=args.paths,
action=file_finder.FileFinderAction(action_type=args.action),
conditions=[size_condition])
# Check our flow throttling limits, will raise if there are problems.
throttler = throttle.FlowThrottler()
throttler.EnforceLimits(client_urn,
token.username,
"FileFinder",
file_finder_args,
token=token)
# Limit the whole flow to 200MB so if a glob matches lots of small files we
# still don't have too much impact.
runner_args = flow_runner.FlowRunnerArgs(client_id=client_urn,
flow_name="FileFinder",
network_bytes_limit=200 *
1000 * 1000)
flow_id = flow.GRRFlow.StartFlow(runner_args=runner_args,
token=token,
args=file_finder_args)
# Provide a url where the caller can check on the flow status.
status_url = urlparse.urljoin(
config_lib.CONFIG["AdminUI.url"], "/api/flows/%s/%s/status" %
(client_urn.Basename(), flow_id.Basename()))
return dict(
flow_id=api_value_renderers.RenderValue(flow_id),
flow_args=api_value_renderers.RenderValue(file_finder_args),
runner_args=api_value_renderers.RenderValue(runner_args),
status_url=status_url)
class TestFileFinderOSWindows(transfer.TestGetFileOSWindows):
"""Download a file with FileFinder.
Exercise globbing, interpolation and filtering.
"""
flow = "FileFinder"
test_output_path = "/fs/os/.*/Windows/System32/notepad.exe"
sizecondition = file_finder.FileFinderSizeCondition(max_file_size=1000000)
filecondition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=sizecondition)
download = file_finder.FileFinderDownloadActionOptions()
action = file_finder.FileFinderAction(
action_type=file_finder.FileFinderAction.Action.DOWNLOAD,
download=download)
args = {"paths": ["%%environ_systemroot%%\\System32\\notepad.*"],
"conditions": filecondition,
"action": action}
def StartRequests(self):
"""Generate and send the Find requests."""
client = aff4.FACTORY.Open(self.client_id, token=self.token)
if self.runner.output is not None:
self.runner.output.Set(
self.runner.output.Schema.DESCRIPTION(
"CacheGrep for {0}".format(self.args.data_regex)))
usernames = [
"%s\\%s" % (u.userdomain, u.username) for u in self.state.users
]
usernames = [u.lstrip("\\")
for u in usernames] # Strip \\ if no domain.
condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.
CONTENTS_REGEX_MATCH,
contents_regex_match=file_finder.
FileFinderContentsRegexMatchCondition(
regex=self.args.data_regex,
mode=file_finder.FileFinderContentsRegexMatchCondition.Mode.
FIRST_HIT))
for path in self.state.all_paths:
full_paths = flow_utils.InterpolatePath(path,
client,
users=usernames)
for full_path in full_paths:
self.CallFlow("FileFinder",
paths=[os.path.join(full_path, "**5")],
pathtype=self.state.args.pathtype,
conditions=[condition],
action=file_finder.FileFinderAction(
action_type=file_finder.FileFinderAction.
Action.DOWNLOAD),
next_state="HandleResults")
def Handle(self, args, token=None):
client_urn = self.GetClientTarget(args, token=token)
size_condition = file_finder.FileFinderCondition(
condition_type=file_finder.FileFinderCondition.Type.SIZE,
size=file_finder.FileFinderSizeCondition(
max_file_size=args.max_file_size))
file_finder_args = file_finder.FileFinderArgs(
paths=args.paths,
action=file_finder.FileFinderAction(action_type=args.action),
conditions=[size_condition])
# Check our flow throttling limits, will raise if there are problems.
throttler = throttle.FlowThrottler(
daily_req_limit=config_lib.CONFIG.Get("API.DailyFlowRequestLimit"),
dup_interval=config_lib.CONFIG.Get("API.FlowDuplicateInterval"))
throttler.EnforceLimits(client_urn,
token.username,
file_finder.FileFinder.__name__,
file_finder_args,
token=token)
# Limit the whole flow to 200MB so if a glob matches lots of small files we
# still don't have too much impact.
runner_args = rdf_flows.FlowRunnerArgs(
client_id=client_urn,
flow_name=file_finder.FileFinder.__name__,
network_bytes_limit=200 * 1000 * 1000)
flow_id = flow.GRRFlow.StartFlow(runner_args=runner_args,
token=token,
args=file_finder_args)
return ApiStartRobotGetFilesOperationResult(
operation_id=utils.SmartUnicode(flow_id))
def testCopyHuntHandlesLiteralExpressionCorrectly(self):
"""Literals are raw bytes. Testing that raw bytes are processed right."""
literal_match = file_finder.FileFinderContentsLiteralMatchCondition(
literal="foo\x0d\xc8bar")
with self.ACLChecksDisabled():
hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
description="model hunt",
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=file_finder.FileFinderArgs(conditions=[
file_finder.FileFinderCondition(
condition_type="CONTENTS_LITERAL_MATCH",
contents_literal_match=literal_match)
],
paths=["/tmp/evil.txt"]),
token=self.token)
self.Open("/#main=ManageHunts")
self.Click("css=tr:contains('model hunt')")
self.Click("css=button[name=CopyHunt]:not([disabled])")
# Wait until dialog appears.
self.WaitUntil(self.IsTextPresent, "What to run?")
# Check that non-default values of sample hunt are prefilled.
self.WaitUntilEqual(
"foo\\x0d\\xc8bar", self.GetValue, "css=grr-new-hunt-wizard-form "
"label:contains('Literal') ~ * input:text")
# Click on "Next" button.
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Output Processing")
# Click on "Next" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Where to run?")
# Click on "Next" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Review")
# Check that the arguments summary is present.
self.WaitUntil(self.IsTextPresent, file_finder.FileFinder.__name__)
self.WaitUntil(self.IsTextPresent, "foo\\x0d\\xc8bar")
# Click on "Run" button
self.Click("css=grr-new-hunt-wizard-form button.Next")
self.WaitUntil(self.IsTextPresent, "Created Hunt")
# Close the window and check that the hunt was created.
self.Click("css=button.Next")
hunts_root = aff4.FACTORY.Open("aff4:/hunts", token=self.token)
hunts_list = sorted(list(hunts_root.ListChildren()),
key=lambda x: x.age)
self.assertEqual(len(hunts_list), 2)
last_hunt = aff4.FACTORY.Open(hunts_list[-1], token=self.token)
# Check that the hunt was created with a correct literal value.
self.assertEqual(last_hunt.args.flow_runner_args.flow_name,
file_finder.FileFinder.__name__)
self.assertEqual(
last_hunt.args.flow_args.conditions[0].contents_literal_match.
literal, "foo\x0d\xc8bar")
