def testMultiGetFileSetsFileHashAttributeWhenMultipleChunksDownloaded(
self):
client_mock = action_mocks.MultiGetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
args = transfer.MultiGetFileArgs(pathspecs=[pathspec])
for _ in test_lib.TestFlowHelper("MultiGetFile",
client_mock,
token=self.token,
client_id=self.client_id,
args=args):
pass
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = pathspec.AFF4Path(self.client_id)
fd = aff4.FACTORY.Open(urn, token=self.token)
fd_hash = fd.Get(fd.Schema.HASH)
self.assertTrue(fd_hash)
h = hashlib.sha256()
with open(os.path.join(self.base_path, "test_img.dd"),
"rb") as model_fd:
h.update(model_fd.read())
self.assertEqual(fd_hash.sha256, h.digest())
def testMultiGetFileDeduplication(self):
client_mock = action_mocks.MultiGetFileClientMock()
pathspecs = []
# Make 10 files to download.
for i in xrange(10):
path = os.path.join(self.temp_dir, "test_%s.txt" % i)
with open(path, "wb") as fd:
fd.write("Hello")
pathspecs.append(
rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=path))
# All those files are the same so the individual chunks should
# only be downloaded once. By forcing maximum_pending_files=1,
# there should only be a single TransferBuffer call.
args = transfer.MultiGetFileArgs(pathspecs=pathspecs,
maximum_pending_files=1)
for _ in test_lib.TestFlowHelper("MultiGetFile",
client_mock,
token=self.token,
client_id=self.client_id,
args=args):
pass
self.assertEqual(client_mock.action_counts["TransferBuffer"], 1)
def testMultiGetFile(self):
"""Test MultiGetFile."""
client_mock = action_mocks.MultiGetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
args = transfer.MultiGetFileArgs(pathspecs=[pathspec, pathspec])
with test_lib.Instrument(transfer.MultiGetFile,
"StoreStat") as storestat_instrument:
for _ in test_lib.TestFlowHelper("MultiGetFile",
client_mock,
token=self.token,
client_id=self.client_id,
args=args):
pass
# We should only have called StoreStat once because the two paths
# requested were identical.
self.assertEqual(len(storestat_instrument.args), 1)
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
fd2 = open(pathspec.path, "rb")
fd2.seek(0, 2)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def testMultiGetFileOfSpecialFiles(self):
"""Test that special /proc/ files are handled correctly.
/proc/ files have the property that they are non seekable from their end
(i.e. seeking them relative to the end is not supported). They also return
an st_size of 0. For example:
$ stat /proc/self/maps
File: '/proc/self/maps'
Size: 0 Blocks: 0 IO Block: 1024 regular empty file
$ head /proc/self/maps
00400000-00409000 r-xp 00000000 fc:01 9180740 /usr/bin/head
00608000-00609000 r--p 00008000 fc:01 9180740 /usr/bin/head
...
When we try to use the MultiGetFile flow, it deduplicates the files and
since it thinks the file has a zero size, the flow will not download the
file, and instead copy the zero size file into it.
"""
client_mock = action_mocks.MultiGetFileClientMock()
# # Create a zero sized file.
zero_sized_filename = os.path.join(self.temp_dir, "zero_size")
with open(zero_sized_filename, "wb") as fd:
pass
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=zero_sized_filename)
for _ in test_lib.TestFlowHelper("MultiGetFile",
client_mock,
token=self.token,
file_size="1MiB",
client_id=self.client_id,
pathspecs=[pathspec]):
pass
# Now if we try to fetch a real /proc/ filename this will fail because the
# filestore already contains the zero length file
# aff4:/files/nsrl/da39a3ee5e6b4b0d3255bfef95601890afd80709.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path="/proc/self/environ")
for _ in test_lib.TestFlowHelper("MultiGetFile",
client_mock,
token=self.token,
file_size=1024 * 1024,
client_id=self.client_id,
pathspecs=[pathspec]):
pass
data = open(pathspec.last.path, "rb").read()
# Test the AFF4 file that was created - it should be empty since by default
# we judge the file size based on its stat.st_size.
urn = pathspec.AFF4Path(self.client_id)
fd = aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual(fd.size, len(data))
self.assertMultiLineEqual(fd.read(len(data)), data)
def testMultiGetFileMultiFiles(self):
"""Test MultiGetFile downloading many files at once."""
client_mock = action_mocks.MultiGetFileClientMock()
pathspecs = []
# Make 30 files to download.
for i in xrange(30):
path = os.path.join(self.temp_dir, "test_%s.txt" % i)
with open(path, "wb") as fd:
fd.write("Hello")
pathspecs.append(
rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=path))
args = transfer.MultiGetFileArgs(pathspecs=pathspecs,
maximum_pending_files=10)
for session_id in test_lib.TestFlowHelper("MultiGetFile",
client_mock,
token=self.token,
client_id=self.client_id,
args=args):
# Check up on the internal flow state.
flow_obj = aff4.FACTORY.Open(session_id,
mode="r",
token=self.token)
flow_state = flow_obj.state
# All the pathspecs should be in this list.
self.assertEqual(len(flow_state.indexed_pathspecs), 30)
# At any one time, there should not be more than 10 files or hashes
# pending.
self.assertLessEqual(len(flow_state.pending_files), 10)
self.assertLessEqual(len(flow_state.pending_hashes), 10)
# When we finish there should be no pathspecs stored in the flow state.
for flow_pathspec in flow_state.indexed_pathspecs:
self.assertIsNone(flow_pathspec)
for flow_request_data in flow_state.request_data_list:
self.assertIsNone(flow_request_data)
# Now open each file and make sure the data is there.
for pathspec in pathspecs:
urn = pathspec.AFF4Path(self.client_id)
fd = aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual("Hello", fd.Read(100000))
def testMultiGetFileSizeLimit(self):
client_mock = action_mocks.MultiGetFileClientMock()
image_path = os.path.join(self.base_path, "test_img.dd")
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS, path=image_path)
# Read a bit more than one chunk (600 * 1024).
expected_size = 750 * 1024
args = transfer.MultiGetFileArgs(
pathspecs=[pathspec], file_size=expected_size)
for _ in flow_test_lib.TestFlowHelper(
transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args):
pass
urn = pathspec.AFF4Path(self.client_id)
blobimage = aff4.FACTORY.Open(urn, token=self.token)
# Make sure a VFSBlobImage got written.
self.assertTrue(isinstance(blobimage, aff4_grr.VFSBlobImage))
self.assertEqual(len(blobimage), expected_size)
data = blobimage.read(100 * expected_size)
self.assertEqual(len(data), expected_size)
expected_data = open(image_path, "rb").read(expected_size)
self.assertEqual(data, expected_data)
hash_obj = blobimage.Get(blobimage.Schema.HASH)
d = hashlib.sha1()
d.update(expected_data)
expected_hash = d.hexdigest()
self.assertEqual(hash_obj.sha1, expected_hash)
def testRefreshFileStartsFlow(self):
self.Open("/")
self.Type("client_query", "C.0000000000000001")
self.Click("client_query_submit")
self.WaitUntilEqual(u"C.0000000000000001", self.GetText,
"css=span[type=subject]")
# Choose client 1.
self.Click("css=td:contains('0001')")
# Go to Browse VFS.
self.Click("css=a:contains('Browse Virtual Filesystem')")
self.Click("css=#_fs i.jstree-icon")
self.Click("css=#_fs-os i.jstree-icon")
self.Click("css=#_fs-os-c i.jstree-icon")
# Test file versioning.
self.WaitUntil(self.IsElementPresent, "css=#_fs-os-c-Downloads")
self.Click("link=Downloads")
# Select a file and start a flow by requesting a newer version.
self.Click("css=tr:contains(\"a.txt\")")
self.Click("css=li[heading=Download]")
self.Click("css=button:contains(\"Collect from the client\")")
# Create a new file version (that would have been created by the flow
# otherwise) and finish the flow.
with self.ACLChecksDisabled():
client_id = rdf_client.ClientURN("C.0000000000000001")
fd = aff4.FACTORY.Open(client_id.Add("flows"), token=self.token)
# Make sure that the flow has started (when button is clicked, the HTTP
# API request is sent asynchronously).
def MultiGetFileStarted():
return "MultiGetFile" in list(x.__class__.__name__
for x in fd.OpenChildren())
self.WaitUntil(MultiGetFileStarted)
flows = list(fd.ListChildren())
client_mock = action_mocks.MultiGetFileClientMock()
for flow_urn in flows:
for _ in test_lib.TestFlowHelper(flow_urn,
client_mock,
client_id=client_id,
check_flow_errors=False,
token=self.token):
pass
time_in_future = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration(
"1h")
# We have to make sure that the new version will not be within a second
# from the current one, otherwise the previous one and the new one will
# be indistinguishable in the UI (as it has a 1s precision when
# displaying versions).
with test_lib.FakeTime(time_in_future):
with aff4.FACTORY.Open(
"aff4:/C.0000000000000001/fs/os/c/Downloads/a.txt",
aff4_type=aff4_grr.VFSFile,
mode="rw",
token=self.token) as fd:
fd.Write("The newest version!")
# Once the flow has finished, the file view should update and add the
# newly created, latest version of the file to the list. The selected
# option should still be "HEAD".
self.WaitUntilContains("HEAD", self.GetText,
"css=.version-dropdown > option[selected]")
self.WaitUntilContains(gui_test_lib.DateTimeString(time_in_future),
self.GetText,
"css=.version-dropdown > option:nth(1)")
# The file table should also update and display the new timestamp.
self.WaitUntil(
self.IsElementPresent,
"css=grr-file-table tbody > tr td:contains(\"%s\")" %
(gui_test_lib.DateTimeString(time_in_future)))
# Make sure the file content has changed.
self.Click("css=li[heading=TextView]")
self.WaitUntilContains("The newest version!", self.GetText,
"css=div.monospace pre")
# Go to the flow management screen and check that there was a new flow.
self.Click("css=a:contains('Manage launched flows')")
self.Click("css=grr-flows-list tr:contains('MultiGetFile')")
self.WaitUntilContains("MultiGetFile", self.GetText,
"css=#main_bottomPane")
self.WaitUntilContains(
"c/Downloads/a.txt", self.GetText,
"css=#main_bottomPane table > tbody td.proto_key:contains(\"Path\") "
"~ td.proto_value")
