def Start(self):
"""Issue the find request."""
super(FileFinder, self).Start()
if not self.args.paths:
# Nothing to do.
return
self.state.Register("files_found", 0)
self.state.Register("sorted_conditions",
sorted(self.args.conditions, key=self._ConditionWeight))
if self.args.pathtype in (rdfvalue.PathSpec.PathType.MEMORY,
rdfvalue.PathSpec.PathType.REGISTRY):
# Memory and Registry StatEntries won't pass the file type check.
self.args.no_file_type_check = True
if self.args.pathtype == rdfvalue.PathSpec.PathType.MEMORY:
# If pathtype is MEMORY, we're treating provided paths not as globs,
# but as paths to memory devices.
for path in self.args.paths:
pathspec = rdfvalue.PathSpec(
path=utils.SmartUnicode(path),
pathtype=rdfvalue.PathSpec.PathType.MEMORY)
aff4path = aff4.AFF4Object.VFSGRRClient.PathspecToURN(
pathspec, self.client_id)
stat_entry = rdfvalue.StatEntry(aff4path=aff4path, pathspec=pathspec)
self.ApplyCondition(rdfvalue.FileFinderResult(stat_entry=stat_entry),
condition_index=0)
else:
self.GlobForPaths(self.args.paths, pathtype=self.args.pathtype,
no_file_type_check=self.args.no_file_type_check)
def testStartsZipGenerationWhenGenerateZipButtonIsClicked(self):
stat_entry = rdfvalue.StatEntry(aff4path="aff4:/foo/bar")
values = [rdfvalue.FileFinderResult(stat_entry=stat_entry)]
with self.ACLChecksDisabled():
hunt_urn = self.CreateGenericHuntWithCollection(values=values)
self.GrantHuntApproval(hunt_urn)
self.Open("/")
self.Click("css=a[grrtarget=ManageHunts]")
self.Click("css=td:contains('GenericHunt')")
self.Click("css=li[heading=Results]")
# Using :visible selector as we don't know which button (ZIP or TAR.GZ) will
# be shown - it depends on the platform.
self.Click("css=button.DownloadButton:visible")
self.WaitUntil(self.IsTextPresent, "Generation has started")
with self.ACLChecksDisabled():
flows_dir = aff4.FACTORY.Open("aff4:/flows")
flows = list(flows_dir.OpenChildren())
export_flows = [
f for f in flows
if f.__class__.__name__ == "ExportHuntResultFilesAsArchive"]
self.assertEqual(len(export_flows), 1)
self.assertEqual(export_flows[0].args.hunt_urn, hunt_urn)
def testDownloadCollection(self):
"""Check we can export a file without errors."""
# Create a collection with URNs to some files.
fd = aff4.FACTORY.Create("aff4:/testcoll", "RDFValueCollection",
token=self.token)
fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1")))
fd.Add(rdfvalue.StatEntry(aff4path=self.out.Add("testfile2")))
fd.Add(rdfvalue.FileFinderResult(
stat_entry=rdfvalue.StatEntry(aff4path=self.out.Add("testfile5"))))
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True,
dump_client_info=True, token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile1" in os.listdir(expected_outdir))
self.assertTrue("testfile2" in os.listdir(expected_outdir))
self.assertTrue("testfile5" in os.listdir(expected_outdir))
# Check we dumped a YAML file to the root of the client.
expected_rootdir = os.path.join(tmpdir, self.client_id.Basename())
self.assertTrue("client_info.yaml" in os.listdir(expected_rootdir))
def testDownloadCollectionWithFlattenOption(self):
"""Check we can download files references in RDFValueCollection."""
# Create a collection with URNs to some files.
fd = aff4.FACTORY.Create("aff4:/testcoll", "RDFValueCollection",
token=self.token)
fd.Add(rdfvalue.RDFURN(self.out.Add("testfile1")))
fd.Add(rdfvalue.StatEntry(aff4path=self.out.Add("testfile2")))
fd.Add(rdfvalue.FileFinderResult(
stat_entry=rdfvalue.StatEntry(aff4path=self.out.Add("testfile5"))))
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True,
dump_client_info=True, flatten=True,
token=self.token, max_threads=2)
# Check that "files" folder is filled with symlinks to downloaded files.
symlinks = os.listdir(os.path.join(tmpdir, "files"))
self.assertEqual(len(symlinks), 3)
self.assertListEqual(sorted(symlinks),
["C.1000000000000000_fs_os_testfile1",
"C.1000000000000000_fs_os_testfile2",
"C.1000000000000000_fs_os_testfile5"])
self.assertEqual(os.readlink(
os.path.join(tmpdir, "files", "C.1000000000000000_fs_os_testfile1")),
os.path.join(tmpdir, "C.1000000000000000", "fs", "os",
"testfile1"))
def testDownloadCollectionWithFoldersEntries(self):
"""Check we can download RDFValueCollection that also references folders."""
fd = aff4.FACTORY.Create("aff4:/testcoll", "RDFValueCollection",
token=self.token)
fd.Add(rdfvalue.FileFinderResult(
stat_entry=rdfvalue.StatEntry(aff4path=self.out.Add("testfile5"))))
fd.Add(rdfvalue.FileFinderResult(
stat_entry=rdfvalue.StatEntry(aff4path=self.out.Add("testdir1"),
st_mode=stat.S_IFDIR)))
fd.Close()
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection("aff4:/testcoll", tmpdir, overwrite=True,
dump_client_info=True, token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile5" in os.listdir(expected_outdir))
self.assertTrue("testdir1" in os.listdir(expected_outdir))
def testFileViewHasExportTabWhenCollectionHasFileFinderResults(self):
collection_urn = "aff4:/C.0000000000000001/analysis/SomeFlow/results"
with self.ACLChecksDisabled():
with aff4.FACTORY.Create(collection_urn,
"RDFValueCollection",
token=self.token) as fd:
fd.Add(
rdfvalue.FileFinderResult(stat_entry=rdfvalue.StatEntry(
aff4path="aff4:/some/unique/path")))
self.GrantClientApproval("C.0000000000000001")
self.CheckExportTabIsPresent()
def testNetgroupBufferParser(self):
"""Ensure we can extract users from a netgroup file."""
parser = linux_file_parser.NetgroupBufferParser()
buf1 = rdfvalue.BufferReference(data="group1 (-,user1,) (-,user2,) "
"(-,user3,)\n")
buf2 = rdfvalue.BufferReference(data="super_group3 (-,user5,) (-,user6,)"
" group1 group2\n")
ff_result = rdfvalue.FileFinderResult(matches=[buf1, buf2])
config_lib.CONFIG.Set("Artifacts.netgroup_user_blacklist", ["user2",
"user3"])
out = list(parser.Parse(ff_result, None))
self.assertItemsEqual([x.username for x in out],
[u"user1", u"user5", u"user6"])
def testShowsGenerateArchiveButtonForFileFinderHunt(self):
stat_entry = rdfvalue.StatEntry(aff4path="aff4:/foo/bar")
values = [rdfvalue.FileFinderResult(stat_entry=stat_entry)]
with self.ACLChecksDisabled():
self.CreateGenericHuntWithCollection(values=values)
self.Open("/")
self.Click("css=a[grrtarget=ManageHunts]")
self.Click("css=td:contains('GenericHunt')")
self.Click("css=li[heading=Results]")
self.WaitUntil(self.IsTextPresent,
"Results of this hunt can be downloaded as an archive")
def ProcessFilters(self, responses):
"""Iterate through glob responses, and filter each hit."""
if not responses.success:
# Glob failing is fatal here.
return self.Error("Failed Glob: %s", responses.status)
results = []
for response in responses:
# Only process regular files.
if self.args.no_file_type_check or stat.S_ISREG(response.st_mode):
results.append(rdfvalue.FileFinderResult(stat_entry=response))
self.CallStateInline(messages=results,
next_state="ApplyFilter",
request_data=dict(filter_index=0))
def testHuntAuthorizationIsRequiredToGenerateResultsArchive(self):
stat_entry = rdfvalue.StatEntry(aff4path="aff4:/foo/bar")
values = [rdfvalue.FileFinderResult(stat_entry=stat_entry)]
with self.ACLChecksDisabled():
self.CreateGenericHuntWithCollection(values=values)
self.Open("/")
self.Click("css=a[grrtarget=ManageHunts]")
self.Click("css=td:contains('GenericHunt')")
self.Click("css=li[heading=Results]")
# Using :visible selector as we don't know which button (ZIP or TAR.GZ) will
# be shown - it depends on the platform.
self.Click("css=button.DownloadButton:visible")
self.WaitUntil(self.IsElementPresent, "acl_dialog")
def testPasswdBufferParser(self):
"""Ensure we can extract users from a passwd file."""
parser = linux_file_parser.PasswdBufferParser()
buf1 = rdfvalue.BufferReference(data="user1:x:1000:1000:User1"
" Name,,,:/home/user1:/bin/bash\n")
buf2 = rdfvalue.BufferReference(data="user2:x:1000:1000:User2"
" Name,,,:/home/user2:/bin/bash\n")
ff_result = rdfvalue.FileFinderResult(matches=[buf1, buf2])
out = list(parser.Parse(ff_result, None))
self.assertEquals(len(out), 2)
self.assertTrue(isinstance(out[1], rdfvalue.KnowledgeBaseUser))
self.assertTrue(isinstance(out[1], rdfvalue.KnowledgeBaseUser))
self.assertTrue(out[0].username, "user1")
self.assertTrue(out[0].full_name, "User1 Name")
def testGenerateZipButtonGetsDisabledAfterClick(self):
stat_entry = rdfvalue.StatEntry(aff4path="aff4:/foo/bar")
values = [rdfvalue.FileFinderResult(stat_entry=stat_entry)]
with self.ACLChecksDisabled():
hunt_urn = self.CreateGenericHuntWithCollection(values=values)
self.GrantHuntApproval(hunt_urn)
self.Open("/")
self.Click("css=a[grrtarget=ManageHunts]")
self.Click("css=td:contains('GenericHunt')")
self.Click("css=li[heading=Results]")
# Using :visible selector as we don't know which button (ZIP or TAR.GZ) will
# be shown - it depends on the platform.
self.Click("css=button.DownloadButton:visible")
self.WaitUntil(self.IsElementPresent,
"css=button[name*=generate_][disabled]:visible")
self.WaitUntil(self.IsTextPresent, "Generation has started")
def GlobReportMatch(self, response):
"""This method is called by the glob mixin when there is a match."""
super(FileFinder, self).GlobReportMatch(response)
self.ApplyCondition(rdfvalue.FileFinderResult(stat_entry=response),
condition_index=0)
def testFileFinderResultExportConverter(self):
pathspec = rdfvalue.PathSpec(path="/some/path",
pathtype=rdfvalue.PathSpec.PathType.OS)
match1 = rdfvalue.BufferReference(
offset=42, length=43, data="somedata1", pathspec=pathspec)
match2 = rdfvalue.BufferReference(
offset=44, length=45, data="somedata2", pathspec=pathspec)
stat_entry = rdfvalue.StatEntry(
aff4path=rdfvalue.RDFURN("aff4:/C.00000000000001/fs/os/some/path"),
pathspec=pathspec,
st_mode=33184,
st_ino=1063090,
st_atime=1336469177,
st_mtime=1336129892,
st_ctime=1336129892)
file_finder_result = rdfvalue.FileFinderResult(stat_entry=stat_entry,
matches=[match1, match2])
metadata = rdfvalue.ExportedMetadata(client_urn="C.0000000000000001")
converter = export.FileFinderResultConverter()
results = list(converter.Convert(metadata, file_finder_result,
token=self.token))
# We expect 1 ExportedFile instances in the results
exported_files = [result for result in results
if isinstance(result, rdfvalue.ExportedFile)]
self.assertEqual(len(exported_files), 1)
self.assertEqual(exported_files[0].basename, "path")
self.assertEqual(exported_files[0].urn,
rdfvalue.RDFURN("aff4:/C.00000000000001/fs/os/some/path"))
self.assertEqual(exported_files[0].st_mode, 33184)
self.assertEqual(exported_files[0].st_ino, 1063090)
self.assertEqual(exported_files[0].st_atime, 1336469177)
self.assertEqual(exported_files[0].st_mtime, 1336129892)
self.assertEqual(exported_files[0].st_ctime, 1336129892)
self.assertFalse(exported_files[0].HasField("content"))
self.assertFalse(exported_files[0].HasField("content_sha256"))
self.assertFalse(exported_files[0].HasField("hash_md5"))
self.assertFalse(exported_files[0].HasField("hash_sha1"))
self.assertFalse(exported_files[0].HasField("hash_sha256"))
# We expect 2 ExportedMatch instances in the results
exported_matches = [result for result in results
if isinstance(result, rdfvalue.ExportedMatch)]
exported_matches = sorted(exported_matches, key=lambda x: x.offset)
self.assertEqual(len(exported_matches), 2)
self.assertEqual(exported_matches[0].offset, 42)
self.assertEqual(exported_matches[0].length, 43)
self.assertEqual(exported_matches[0].data, "somedata1")
self.assertEqual(
exported_matches[0].urn,
rdfvalue.RDFURN("aff4:/C.0000000000000001/fs/os/some/path"))
self.assertEqual(exported_matches[1].offset, 44)
self.assertEqual(exported_matches[1].length, 45)
self.assertEqual(exported_matches[1].data, "somedata2")
self.assertEqual(
exported_matches[1].urn,
rdfvalue.RDFURN("aff4:/C.0000000000000001/fs/os/some/path"))
