def testFileFinderThrottlingByFlowCountWorks(self):
self.InitRouterConfig(self.__class__.FILE_FINDER_THROTTLED_ROUTER_CONFIG %
self.token.username)
args = []
for p in ["tests.plist", "numbers.txt", "numbers.txt.ver2"]:
args.append(
rdf_file_finder.FileFinderArgs(
action=rdf_file_finder.FileFinderAction(action_type="STAT"),
paths=[p]).AsPrimitiveProto())
client_ref = self.api.Client(client_id=self.client_id.Basename())
flow_obj = client_ref.CreateFlow(
name=file_finder.FileFinder.__name__, args=args[0])
self.assertEqual(flow_obj.data.state, flow_obj.data.RUNNING)
flow_obj = client_ref.CreateFlow(
name=file_finder.FileFinder.__name__, args=args[1])
self.assertEqual(flow_obj.data.state, flow_obj.data.RUNNING)
with self.assertRaisesRegexp(RuntimeError, "2 flows run since"):
client_ref.CreateFlow(name=file_finder.FileFinder.__name__, args=args[2])
def Start(self):
"""Determine the Firefox history directory."""
self.state.hist_count = 0
self.state.history_paths = []
if self.args.history_path:
self.state.history_paths.append(self.args.history_path)
else:
self.state.history_paths = self.GuessHistoryPaths(
self.args.username)
if not self.state.history_paths:
raise flow.FlowError("Could not find valid History paths.")
filename = "places.sqlite"
for path in self.state.history_paths:
self.CallFlow(file_finder.FileFinder.__name__,
paths=[os.path.join(path, "**2", filename)],
pathtype=self.args.pathtype,
action=rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.
Action.DOWNLOAD),
next_state="ParseFiles")
def testFileFinderDownloadMaxFileSizeCanBeOverriden(self):
router = self._CreateRouter(
file_finder_flow=rr.RobotRouterFileFinderFlowParams(
enabled=True, max_file_size=42))
da = rdf_file_finder.FileFinderDownloadActionOptions()
da.max_size = 80
da.oversized_file_policy = da.OversizedFilePolicy.DOWNLOAD_TRUNCATED
path = "/foo/bar"
handler = router.CreateFlow(api_flow.ApiCreateFlowArgs(
flow=api_flow.ApiFlow(name=file_finder.FileFinder.__name__,
args=rdf_file_finder.FileFinderArgs(
paths=[path],
action=rdf_file_finder.FileFinderAction(
action_type="DOWNLOAD",
download=da))),
client_id=self.client_id),
token=self.token)
da = handler.override_flow_args.action.download
self.assertEqual(da.oversized_file_policy, da.OversizedFilePolicy.SKIP)
self.assertEqual(da.max_size, 42)
class TestFileFinderOSLinuxProc(base.VFSPathContentExists):
"""Download a /proc/sys entry with FileFinder."""
platforms = ["Linux"]
flow = "FileFinder"
test_output_path = "/fs/os/proc/sys/net/ipv4/ip_forward"
client_min_version = 3007
sizecondition = rdf_file_finder.FileFinderSizeCondition(
max_file_size=1000000)
filecondition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE,
size=sizecondition)
download = rdf_file_finder.FileFinderDownloadActionOptions()
action = rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD,
download=download)
args = {
"paths": ["/proc/sys/net/ipv4/ip_forward"],
"conditions": filecondition,
"action": action
}
def setUp(self):
super(ApiGetHuntFileHandlerTest, self).setUp()
self.handler = hunt_plugin.ApiGetHuntFileHandler()
self.file_path = os.path.join(self.base_path, "test.plist")
self.hunt = hunts.GRRHunt.StartHunt(
hunt_name="GenericHunt",
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=rdf_file_finder.FileFinderArgs(
paths=[self.file_path],
action=rdf_file_finder.FileFinderAction(action_type="DOWNLOAD"),),
client_rate=0,
token=self.token)
self.hunt.Run()
self.aff4_file_path = "fs/os/%s" % self.file_path
self.client_id = self.SetupClients(1)[0]
self.AssignTasksToClients(client_ids=[self.client_id])
action_mock = action_mocks.FileFinderClientMock()
test_lib.TestHuntHelper(action_mock, [self.client_id], token=self.token)
def testListHuntClients(self):
hunt = implementation.GRRHunt.StartHunt(
hunt_name=standard.GenericHunt.__name__,
flow_runner_args=rdf_flows.FlowRunnerArgs(
flow_name=file_finder.FileFinder.__name__),
flow_args=rdf_file_finder.FileFinderArgs(
paths=[os.path.join(self.base_path, "test.plist")],
action=rdf_file_finder.FileFinderAction(
action_type="DOWNLOAD"),
),
client_rate=0,
token=self.token)
hunt.Run()
client_ids = self.SetupClients(5)
self.AssignTasksToClients(client_ids=client_ids)
action_mock = action_mocks.FileFinderClientMock()
hunt_test_lib.TestHuntHelper(action_mock,
client_ids,
iteration_limit=10,
token=self.token)
result = self.handler.Handle(
hunt_plugin.ApiListHuntClientsArgs(hunt_id=hunt.urn.Basename()),
token=self.token)
# TODO(user): This still uses data store internals and will fail on some
# data stores.
# This is not super deterministic, we start processing some
# clients, run the hunt for a bit but there is no order to all
# this. We should have some clients half finished though (i.e.,
# with pending requests) and five clients in total.
self.assertEqual(result.total_count, 5)
clients = list(result.items)
pending_requests = [client.pending_requests for client in clients]
self.assertTrue(any(r.next_state) for r in pending_requests)
def testAttributesOfFileFoundInHashFileStoreAreSetCorrectly(self):
client_ids = self.SetupClients(2)
filename = os.path.join(self.base_path, "tcpip.sig")
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=filename)
urn1 = pathspec.AFF4Path(client_ids[0])
urn2 = pathspec.AFF4Path(client_ids[1])
for client_id in client_ids:
client_mock = action_mocks.FileFinderClientMock()
for _ in flow_test_lib.TestFlowHelper(
file_finder.FileFinder.__name__,
client_mock,
token=self.token,
client_id=client_id,
paths=[filename],
action=rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.
DOWNLOAD)):
pass
# Running worker to make sure FileStore.AddFileToStore event is processed
# by the worker.
worker = worker_test_lib.MockWorker(token=self.token)
worker.Simulate()
fd1 = aff4.FACTORY.Open(urn1, token=self.token)
self.assertTrue(isinstance(fd1, aff4_grr.VFSBlobImage))
fd2 = aff4.FACTORY.Open(urn2, token=self.token)
self.assertTrue(isinstance(fd2, aff4_grr.VFSBlobImage))
self.assertTrue(fd1.Get(fd1.Schema.STAT))
self.assertTrue(fd2.Get(fd2.Schema.STAT))
self.assertEqual(fd1.Get(fd1.Schema.SIZE), fd2.Get(fd2.Schema.SIZE))
self.assertEqual(fd1.Get(fd1.Schema.CONTENT_LAST),
fd2.Get(fd2.Schema.CONTENT_LAST))
def Grep(self, source, pathtype):
"""Grep files in paths for any matches to content_regex_list.
Args:
source: artifact source
pathtype: pathspec path type
When multiple regexes are supplied, combine them into a single regex as an
OR match so that we check all regexes at once.
"""
path_list = self.InterpolateList(source.attributes.get("paths", []))
content_regex_list = self.InterpolateList(
source.attributes.get("content_regex_list", []))
regex_condition = rdf_file_finder.FileFinderContentsRegexMatchCondition(
regex=self._CombineRegex(content_regex_list),
bytes_before=0,
bytes_after=0,
mode="ALL_HITS")
file_finder_condition = rdf_file_finder.FileFinderCondition(
condition_type=(
rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH),
contents_regex_match=regex_condition)
self.CallFlow(
file_finder.FileFinder.__name__,
paths=path_list,
conditions=[file_finder_condition],
action=rdf_file_finder.FileFinderAction(),
pathtype=pathtype,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
class TestFileFinderOSWindows(base.VFSPathContentIsPE):
"""Download a file with FileFinder.
Exercise globbing, interpolation and filtering.
"""
platforms = ["Windows"]
flow = file_finder.FileFinder.__name__
test_output_path = "/fs/os/C:/Windows/System32/notepad.exe"
sizecondition = rdf_file_finder.FileFinderSizeCondition(max_file_size=1000000)
filecondition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE,
size=sizecondition)
download = rdf_file_finder.FileFinderDownloadActionOptions()
action = rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD,
download=download)
args = {
"paths": ["%%environ_systemroot%%\\System32\\notepad.*"],
"conditions": filecondition,
"action": action
}
def Handle(self, args, token=None):
client_urn = self.GetClientTarget(args, token=token)
size_condition = rdf_file_finder.FileFinderCondition(
condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE,
size=rdf_file_finder.FileFinderSizeCondition(
max_file_size=args.max_file_size))
file_finder_args = rdf_file_finder.FileFinderArgs(
paths=args.paths,
action=rdf_file_finder.FileFinderAction(action_type=args.action),
conditions=[size_condition])
# Check our flow throttling limits, will raise if there are problems.
throttler = throttle.FlowThrottler(
daily_req_limit=config_lib.CONFIG.Get("API.DailyFlowRequestLimit"),
dup_interval=config_lib.CONFIG.Get("API.FlowDuplicateInterval"))
throttler.EnforceLimits(client_urn,
token.username,
file_finder.FileFinder.__name__,
file_finder_args,
token=token)
# Limit the whole flow to 200MB so if a glob matches lots of small files we
# still don't have too much impact.
runner_args = rdf_flows.FlowRunnerArgs(
client_id=client_urn,
flow_name=file_finder.FileFinder.__name__,
network_bytes_limit=200 * 1000 * 1000)
flow_id = flow.GRRFlow.StartFlow(runner_args=runner_args,
token=token,
args=file_finder_args)
return ApiStartRobotGetFilesOperationResult(
operation_id=utils.SmartUnicode(flow_id))
def testLinkStat(self):
"""Tests resolving symlinks when getting stat entries."""
test_dir = os.path.join(self.temp_dir, "lnk_stat_test")
lnk = os.path.join(test_dir, "lnk")
lnk_target = os.path.join(test_dir, "lnk_target")
os.mkdir(test_dir)
with open(lnk_target, "wb") as fd:
fd.write("sometext")
os.symlink(lnk_target, lnk)
paths = [lnk]
link_size = os.lstat(lnk).st_size
target_size = os.stat(lnk).st_size
for expected_size, resolve_links in [(link_size, False),
(target_size, True)]:
stat_action = rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.STAT,
stat=rdf_file_finder.FileFinderStatActionOptions(
resolve_links=resolve_links))
results = self._RunFileFinder(paths, stat_action)
self.assertEqual(len(results), 1)
res = results[0]
self.assertEqual(res.stat_entry.st_size, expected_size)
def testClientFileFinderUploadSkip(self):
paths = [os.path.join(self.base_path, "**/*.plist")]
action_type = rdf_file_finder.FileFinderAction.Action.DOWNLOAD
download_action = rdf_file_finder.FileFinderDownloadActionOptions(
oversized_file_policy="SKIP", max_size=300)
action = rdf_file_finder.FileFinderAction(
action_type=action_type, download=download_action)
session_id = self._RunClientFileFinder(paths, action)
collection = aff4.FACTORY.Open(session_id.Add("Results"), token=self.token)
results = list(collection)
# Only two instead of the usual four results.
self.assertEqual(len(results), 2)
relpaths = [
os.path.relpath(p.stat_entry.pathspec.path, self.base_path)
for p in results
]
self.assertItemsEqual(relpaths, ["History.plist", "test.plist"])
for r in results:
aff4_obj = aff4.FACTORY.Open(
r.stat_entry.pathspec.AFF4Path(self.client_id), token=self.token)
self.assertEqual(
aff4_obj.Read(100), open(r.stat_entry.pathspec.path, "rb").read(100))
def RunFlowAndCheckResults(
self,
conditions=None,
action=rdf_file_finder.FileFinderAction.Action.STAT,
expected_files=None,
non_expected_files=None,
paths=None):
if not isinstance(action, rdf_file_finder.FileFinderAction):
action = rdf_file_finder.FileFinderAction(action_type=action)
action_type = action.action_type
conditions = conditions or []
expected_files = expected_files or []
non_expected_files = non_expected_files or []
for fname in expected_files + non_expected_files:
aff4.FACTORY.Delete(self.FileNameToURN(fname), token=self.token)
results = self.RunFlow(paths=paths, conditions=conditions, action=action)
self.CheckReplies(results, action_type, expected_files)
self.CheckFilesInCollection(expected_files)
if action_type == rdf_file_finder.FileFinderAction.Action.STAT:
self.CheckFilesNotDownloaded(expected_files + non_expected_files)
self.CheckFilesNotHashed(expected_files + non_expected_files)
elif action_type == rdf_file_finder.FileFinderAction.Action.DOWNLOAD:
self.CheckFilesHashed(expected_files)
self.CheckFilesNotHashed(non_expected_files)
self.CheckFilesDownloaded(expected_files)
self.CheckFilesNotDownloaded(non_expected_files)
# Downloaded files are hashed to allow for deduping.
elif action_type == rdf_file_finder.FileFinderAction.Action.HASH:
self.CheckFilesNotDownloaded(expected_files + non_expected_files)
self.CheckFilesHashed(expected_files)
self.CheckFilesNotHashed(non_expected_files)
def testFileFinderWorkflowWorks(self):
self.InitRouterConfig(
self.__class__.FILE_FINDER_ROUTER_CONFIG % self.token.username)
client_ref = self.api.Client(client_id=self.client_id.Basename())
args = rdf_file_finder.FileFinderArgs(
paths=[
os.path.join(self.base_path, "test.plist"),
os.path.join(self.base_path, "numbers.txt"),
os.path.join(self.base_path, "numbers.txt.ver2")
],
action=rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD)
).AsPrimitiveProto()
flow_obj = client_ref.CreateFlow(
name=file_finder.FileFinder.__name__, args=args)
self.assertEqual(flow_obj.data.state, flow_obj.data.RUNNING)
# Now run the flow we just started.
client_id = rdf_client.ClientURN(flow_obj.client_id)
flow_urn = client_id.Add("flows").Add(flow_obj.flow_id)
for _ in flow_test_lib.TestFlowHelper(
flow_urn,
client_id=client_id,
client_mock=action_mocks.FileFinderClientMock(),
token=self.token):
pass
# Refresh flow.
flow_obj = client_ref.Flow(flow_obj.flow_id).Get()
self.assertEqual(flow_obj.data.state, flow_obj.data.TERMINATED)
# Check that we got 3 results (we downloaded 3 files).
results = list(flow_obj.ListResults())
self.assertEqual(len(results), 3)
# We expect results to be FileFinderResult.
self.assertItemsEqual(
[os.path.basename(r.payload.stat_entry.pathspec.path)
for r in results], ["test.plist", "numbers.txt", "numbers.txt.ver2"])
# Now downloads the files archive.
zip_stream = StringIO.StringIO()
flow_obj.GetFilesArchive().WriteToStream(zip_stream)
zip_fd = zipfile.ZipFile(zip_stream)
# Now check that the archive has only "test.plist" file, as it's the
# only file that matches the whitelist (see FILE_FINDER_ROUTER_CONFIG).
# There should be 3 items in the archive: the hash of the "test.plist"
# file, the symlink to this hash and the MANIFEST file.
namelist = zip_fd.namelist()
self.assertEqual(len(namelist), 3)
# First component of every path in the archive is the containing folder,
# we should strip it.
namelist = [os.path.join(*n.split(os.sep)[1:]) for n in namelist]
with open(os.path.join(self.base_path, "test.plist")) as test_plist_fd:
test_plist_hash = hashlib.sha256(test_plist_fd.read()).hexdigest()
self.assertEqual(
sorted([
# pyformat: disable
os.path.join(self.client_id.Basename(), "fs", "os",
self.base_path.strip("/"), "test.plist"),
os.path.join("hashes", test_plist_hash),
"MANIFEST"
# pyformat: enable
]),
sorted(namelist))
def testFlowDuplicateLimit(self):
# Disable the request limit checking by setting it to 0.
throttler = throttle.FlowThrottler(
daily_req_limit=0, dup_interval=rdfvalue.Duration("1200s"))
# Running the same flow immediately should fail
with test_lib.FakeTime(self.BASE_TIME):
throttler.EnforceLimits(self.client_id,
self.token.username,
"DummyLogFlow",
None,
token=self.token)
flow.GRRFlow.StartFlow(client_id=self.client_id,
flow_name="DummyLogFlow",
token=self.token)
with self.assertRaises(throttle.ErrorFlowDuplicate):
throttler.EnforceLimits(self.client_id,
self.token.username,
"DummyLogFlow",
None,
token=self.token)
# Doing the same outside the window should work
with test_lib.FakeTime(self.BASE_TIME + 1200 + 1):
throttler.EnforceLimits(self.client_id,
self.token.username,
"DummyLogFlow",
None,
token=self.token)
flow.GRRFlow.StartFlow(client_id=self.client_id,
flow_name="DummyLogFlow",
token=self.token)
with self.assertRaises(throttle.ErrorFlowDuplicate):
throttler.EnforceLimits(self.client_id,
self.token.username,
"DummyLogFlow",
None,
token=self.token)
# Now try a flow with more complicated args
args = rdf_file_finder.FileFinderArgs(
paths=["/tmp/1", "/tmp/2"],
action=rdf_file_finder.FileFinderAction(action_type="STAT"))
with test_lib.FakeTime(self.BASE_TIME):
throttler.EnforceLimits(self.client_id,
self.token.username,
file_finder.FileFinder.__name__,
args,
token=self.token)
flow.GRRFlow.StartFlow(
client_id=self.client_id,
flow_name=file_finder.FileFinder.__name__,
token=self.token,
paths=["/tmp/1", "/tmp/2"],
action=rdf_file_finder.FileFinderAction(action_type="STAT"))
with self.assertRaises(throttle.ErrorFlowDuplicate):
throttler.EnforceLimits(self.client_id,
self.token.username,
file_finder.FileFinder.__name__,
args,
token=self.token)
# Different args should succeed.
args = rdf_file_finder.FileFinderArgs(
paths=["/tmp/1", "/tmp/3"],
action=rdf_file_finder.FileFinderAction(action_type="STAT"))
throttler.EnforceLimits(self.client_id,
self.token.username,
file_finder.FileFinder.__name__,
args,
token=self.token)
def setUp(self):
super(FileFinderTest, self).setUp()
self.stat_action = rdf_file_finder.FileFinderAction(
action_type=rdf_file_finder.FileFinderAction.Action.STAT)
def _StatAction(**kwargs):
action_type = rdf_file_finder.FileFinderAction.Action.STAT
opts = rdf_file_finder.FileFinderStatActionOptions(**kwargs)
return rdf_file_finder.FileFinderAction(action_type=action_type,
stat=opts)