def testVFSVirtualRoot(self):
# Let's open a file in the virtual root.
os_root = "os:%s" % self.base_path
with test_lib.ConfigOverrider({"Client.vfs_virtualroots": [os_root]}):
# We need to reset the vfs.VFS_VIRTUALROOTS too.
vfs.VFSInit().Run()
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path="/morenumbers.txt",
pathtype=rdf_paths.PathSpec.PathType.OS))
data = fd.read(10)
self.assertEqual(data, "1\n2\n3\n4\n5\n")
# This should also work with TSK.
tsk_root = "tsk:%s" % os.path.join(self.base_path, "test_img.dd")
with test_lib.ConfigOverrider({"Client.vfs_virtualroots": [tsk_root]}):
vfs.VFSInit().Run()
image_file_ps = rdf_paths.PathSpec(
path=u"איןד ןד ש אקדא/איןד.txt",
pathtype=rdf_paths.PathSpec.PathType.TSK)
fd = vfs.VFSOpen(image_file_ps)
data = fd.read(10)
self.assertEqual(data, "1\n2\n3\n4\n5\n")
# This should not influence vfs handlers other than OS and TSK.
reg_type = rdf_paths.PathSpec.PathType.REGISTRY
os_handler = vfs.VFS_HANDLERS[rdf_paths.PathSpec.PathType.OS]
with vfs_test_lib.VFSOverrider(reg_type, os_handler):
with self.assertRaises(IOError):
image_file_ps.pathtype = reg_type
vfs.VFSOpen(image_file_ps)
def testDownloadCollectionWithFoldersEntries(self):
"""Check we can download a collection that also references folders."""
fd = sequential_collection.GeneralIndexedCollection(
self.collection_urn)
with data_store.DB.GetMutationPool() as pool:
fd.Add(rdf_file_finder.FileFinderResult(
stat_entry=rdf_client.StatEntry(pathspec=rdf_paths.PathSpec(
path="testfile5", pathtype="OS"))),
mutation_pool=pool)
fd.Add(rdf_file_finder.FileFinderResult(
stat_entry=rdf_client.StatEntry(pathspec=rdf_paths.PathSpec(
path="testdir1", pathtype="OS"),
st_mode=stat.S_IFDIR)),
mutation_pool=pool)
with utils.TempDirectory() as tmpdir:
export_utils.DownloadCollection(self.collection_urn,
tmpdir,
overwrite=True,
dump_client_info=True,
token=self.token,
max_threads=2)
expected_outdir = os.path.join(tmpdir, self.out.Path()[1:])
# Check we found both files.
self.assertTrue("testfile5" in os.listdir(expected_outdir))
self.assertTrue("testdir1" in os.listdir(expected_outdir))
def testOpenFilehandlesExpire(self):
"""Test that file handles expire from cache."""
files.FILE_HANDLE_CACHE = utils.FastStore(max_size=10)
current_process = psutil.Process(os.getpid())
num_open_files = len(current_process.open_files())
path = os.path.join(self.base_path, "morenumbers.txt")
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS))
fds = []
for filename in fd.ListNames():
child_fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=os.path.join(path, filename),
pathtype=rdf_paths.PathSpec.PathType.OS))
fd.read(20)
fds.append(child_fd)
# This should not create any new file handles.
self.assertTrue(len(current_process.open_files()) - num_open_files < 5)
# Make sure we exceeded the size of the cache.
self.assertGreater(fds, 20)
def testIteratedDirectoryListing(self):
"""Test that the client iterator works."""
with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS,
MockVFSHandler):
path = "/"
# Run the flow in the simulated way
client_mock = action_mocks.ActionMock(
standard.IteratedListDirectory)
flow_test_lib.TestFlowHelper(
filesystem.IteratedListDirectory.__name__,
client_mock,
client_id=self.client_id,
pathspec=rdf_paths.PathSpec(
path="/", pathtype=rdf_paths.PathSpec.PathType.OS),
token=self.token)
fd = aff4.FACTORY.Open(self.client_id.Add("fs/os").Add(path),
token=self.token)
directory = [ch for ch in fd.OpenChildren()]
pb = rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS)
directory2 = list(vfs.VFSOpen(pb).ListFiles())
directory.sort()
result = [x.Get(x.Schema.STAT) for x in directory]
# Make sure that the resulting directory is what it should be
for x, y in zip(result, directory2):
self.assertEqual(x.st_mode, y.st_mode)
self.assertRDFValuesEqual(x, y)
def testTSKInodeHandling(self):
"""Test that we can open files by inode."""
path = os.path.join(self.base_path, "ntfs_img.dd")
ps2 = rdf_paths.PathSpec(inode=65,
ntfs_type=128,
ntfs_id=0,
path="/this/will/be/ignored",
pathtype=rdf_paths.PathSpec.PathType.TSK)
ps = rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS,
offset=63 * 512)
ps.Append(ps2)
fd = vfs.VFSOpen(ps)
self.assertEqual(fd.Read(100), "Hello world\n")
ps2 = rdf_paths.PathSpec(inode=65,
ntfs_type=128,
ntfs_id=4,
pathtype=rdf_paths.PathSpec.PathType.TSK)
ps = rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS,
offset=63 * 512)
ps.Append(ps2)
fd = vfs.VFSOpen(ps)
self.assertEqual(fd.read(100), "I am a real ADS\n")
# Make sure the size is correct:
self.assertEqual(fd.Stat().st_size, len("I am a real ADS\n"))
def testAFF4Path(self):
"""Test the pathspec to URN conversion function."""
pathspec = rdf_paths.PathSpec(
path="\\\\.\\Volume{1234}\\",
pathtype=rdf_paths.PathSpec.PathType.OS,
mount_point="/c:/").Append(
path="/windows", pathtype=rdf_paths.PathSpec.PathType.TSK)
urn = pathspec.AFF4Path(rdf_client.ClientURN("C.1234567812345678"))
self.assertEqual(
urn,
rdfvalue.RDFURN(
r"aff4:/C.1234567812345678/fs/tsk/\\.\Volume{1234}\/windows"))
# Test an ADS
pathspec = rdf_paths.PathSpec(
path="\\\\.\\Volume{1234}\\",
pathtype=rdf_paths.PathSpec.PathType.OS,
mount_point="/c:/").Append(
pathtype=rdf_paths.PathSpec.PathType.TSK,
path="/Test Directory/notes.txt:ads",
inode=66,
ntfs_type=128,
ntfs_id=2)
urn = pathspec.AFF4Path(rdf_client.ClientURN("C.1234567812345678"))
self.assertEqual(
urn,
rdfvalue.RDFURN(r"aff4:/C.1234567812345678/fs/tsk/\\.\Volume{1234}\/"
"Test Directory/notes.txt:ads"))
def ConvertGlobIntoPathComponents(self, pattern):
r"""Converts a glob pattern into a list of pathspec components.
Wildcards are also converted to regular expressions. The pathspec components
do not span directories, and are marked as a regex or a literal component.
We also support recursion into directories using the ** notation. For
example, /home/**2/foo.txt will find all files named foo.txt recursed 2
directories deep. If the directory depth is omitted, it defaults to 3.
Example:
/home/test/* -> ['home', 'test', '.*\\Z(?ms)']
Args:
pattern: A glob expression with wildcards.
Returns:
A list of PathSpec instances for each component.
Raises:
ValueError: If the glob is invalid.
"""
components = []
for path_component in pattern.split("/"):
# A ** in the path component means recurse into directories that match the
# pattern.
m = rdf_paths.GlobExpression.RECURSION_REGEX.search(path_component)
if m:
path_component = path_component.replace(m.group(0), "*")
component = rdf_paths.PathSpec(
path=fnmatch.translate(path_component),
pathtype=self.state.pathtype,
path_options=rdf_paths.PathSpec.Options.RECURSIVE)
# Allow the user to override the recursion depth.
if m.group(1):
component.recursion_depth = int(m.group(1))
elif self.GLOB_MAGIC_CHECK.search(path_component):
component = rdf_paths.PathSpec(
path=fnmatch.translate(path_component),
pathtype=self.state.pathtype,
path_options=rdf_paths.PathSpec.Options.REGEX)
else:
pathtype = self.state.pathtype
# TODO(amoser): This is a backwards compatibility hack. Remove when
# all clients reach 3.0.0.2.
if (pathtype == rdf_paths.PathSpec.PathType.TSK
and re.match("^.:$", path_component)):
path_component = "%s\\" % path_component
component = rdf_paths.PathSpec(
path=path_component,
pathtype=pathtype,
path_options=rdf_paths.PathSpec.Options.CASE_INSENSITIVE)
components.append(component)
return components
def testFindPaths(self):
# TODO(user): Deal with cases where multiple vars are exported.
# export TERM PERLLIB=.:shouldntbeignored
bashrc_data = StringIO.StringIO("""
IGNORE='bad' PATH=${HOME}/bin:$PATH
{ PYTHONPATH=/path1:/path2 }
export TERM=screen-256color
export http_proxy="http://proxy.example.org:3128/"
export HTTP_PROXY=$http_proxy
if [[ "$some_condition" ]]; then
export PATH=:$PATH; LD_LIBRARY_PATH=foo:bar:$LD_LIBRARY_PATH
PYTHONPATH=$PATH:"${PYTHONPATH}"
CLASSPATH=
fi
echo PATH=/should/be/ignored
# Ignore PATH=foo:bar
TERM=vt100 PS=" Foo" PERL5LIB=:shouldntbeignored
""")
cshrc_data = StringIO.StringIO("""
setenv PATH ${HOME}/bin:$PATH
setenv PYTHONPATH /path1:/path2
set term = (screen-256color)
setenv http_proxy "http://proxy.example.org:3128/"
setenv HTTP_PROXY $http_proxy
if ( -e "$some_condition" ) then
set path = (. $path); setenv LD_LIBRARY_PATH foo:bar:$LD_LIBRARY_PATH
setenv PYTHONPATH $PATH:"${PYTHONPATH}"
setenv CLASSPATH
endif
echo PATH=/should/be/ignored
setenv PERL5LIB :shouldntbeignored
""")
parser = linux_file_parser.PathParser()
bashrc_stat = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec(
path="/home/user1/.bashrc", pathtype="OS"))
cshrc_stat = rdf_client.StatEntry(pathspec=rdf_paths.PathSpec(
path="/home/user1/.cshrc", pathtype="OS"))
bashrc = {
r.name: r.vals
for r in parser.Parse(bashrc_stat, bashrc_data, None)
}
cshrc = {
r.name: r.vals
for r in parser.Parse(cshrc_stat, cshrc_data, None)
}
expected = {
"PATH": [".", "${HOME}/bin", "$PATH"],
"PYTHONPATH": [".", "${HOME}/bin", "$PATH", "/path1", "/path2"],
"LD_LIBRARY_PATH": ["foo", "bar", "$LD_LIBRARY_PATH"],
"CLASSPATH": [],
"PERL5LIB": [".", "shouldntbeignored"]
}
# Got the same environment variables for bash and cshrc files.
self.assertItemsEqual(expected, bashrc)
self.assertItemsEqual(expected, cshrc)
# The path values are expanded correctly.
for var_name in ("PATH", "PYTHONPATH", "LD_LIBRARY_PATH"):
self.assertEqual(expected[var_name], bashrc[var_name])
self.assertEqual(expected[var_name], cshrc[var_name])
def testMultiGetFileOfSpecialFiles(self):
"""Test that special /proc/ files are handled correctly.
/proc/ files have the property that they are non seekable from their end
(i.e. seeking them relative to the end is not supported). They also return
an st_size of 0. For example:
$ stat /proc/self/maps
File: '/proc/self/maps'
Size: 0 Blocks: 0 IO Block: 1024 regular empty file
$ head /proc/self/maps
00400000-00409000 r-xp 00000000 fc:01 9180740 /usr/bin/head
00608000-00609000 r--p 00008000 fc:01 9180740 /usr/bin/head
...
When we try to use the MultiGetFile flow, it deduplicates the files and
since it thinks the file has a zero size, the flow will not download the
file, and instead copy the zero size file into it.
"""
client_mock = action_mocks.MultiGetFileClientMock()
# # Create a zero sized file.
zero_sized_filename = os.path.join(self.temp_dir, "zero_size")
with open(zero_sized_filename, "wb") as fd:
pass
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=zero_sized_filename)
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
file_size="1MiB",
client_id=self.client_id,
pathspecs=[pathspec])
# Now if we try to fetch a real /proc/ filename this will fail because the
# filestore already contains the zero length file
# aff4:/files/nsrl/da39a3ee5e6b4b0d3255bfef95601890afd80709.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path="/proc/self/environ")
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
file_size=1024 * 1024,
client_id=self.client_id,
pathspecs=[pathspec])
data = open(pathspec.last.path, "rb").read()
# Test the AFF4 file that was created - it should be empty since by default
# we judge the file size based on its stat.st_size.
urn = pathspec.AFF4Path(self.client_id)
fd = aff4.FACTORY.Open(urn, token=self.token)
self.assertEqual(fd.size, len(data))
self.assertMultiLineEqual(fd.read(len(data)), data)
def testTSKListDirectory(self):
"""Test directory listing in sleuthkit."""
path = os.path.join(self.base_path, u"test_img.dd")
ps2 = rdf_paths.PathSpec(path=u"入乡随俗 海外春节别样过法",
pathtype=rdf_paths.PathSpec.PathType.TSK)
ps = rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS)
ps.Append(ps2)
directory = vfs.VFSOpen(ps)
self.CheckDirectoryListing(directory, u"入乡随俗.txt")
def testFileCasing(self):
"""Test our ability to read the correct casing from filesystem."""
try:
os.lstat(os.path.join(self.base_path, "nUmBeRs.txt"))
os.lstat(os.path.join(self.base_path, "nuMbErs.txt"))
# If we reached this point we are on a case insensitive file system
# and the tests below do not make any sense.
logging.warning(
"Case insensitive file system detected. Skipping test.")
return
except (IOError, OSError):
pass
# Create 2 files with names that differ only in casing.
with utils.TempDirectory() as temp_dir:
path1 = os.path.join(temp_dir, "numbers.txt")
shutil.copy(os.path.join(self.base_path, "numbers.txt"), path1)
path2 = os.path.join(temp_dir, "numbers.TXT")
shutil.copy(os.path.join(self.base_path, "numbers.txt.ver2"),
path2)
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path1,
pathtype=rdf_paths.PathSpec.PathType.OS))
self.assertEqual(fd.pathspec.Basename(), "numbers.txt")
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path2,
pathtype=rdf_paths.PathSpec.PathType.OS))
self.assertEqual(fd.pathspec.Basename(), "numbers.TXT")
path = os.path.join(self.base_path, "Numbers.txt")
fd = vfs.VFSOpen(
rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS))
read_path = fd.pathspec.Basename()
# The exact file now is non deterministic but should be either of the two:
if read_path != "numbers.txt" and read_path != "numbers.TXT":
raise RuntimeError("read path is %s" % read_path)
# Ensure that the produced pathspec specified no case folding:
s = fd.Stat()
self.assertEqual(s.pathspec.path_options,
rdf_paths.PathSpec.Options.CASE_LITERAL)
# Case folding will only occur when requested - this should raise because
# we have the CASE_LITERAL option:
pathspec = rdf_paths.PathSpec(
path=path,
pathtype=rdf_paths.PathSpec.PathType.OS,
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL)
self.assertRaises(IOError, vfs.VFSOpen, pathspec)
def testTSKFile(self):
"""Test our ability to read from image files."""
path = os.path.join(self.base_path, "test_img.dd")
path2 = "Test Directory/numbers.txt"
p2 = rdf_paths.PathSpec(path=path2,
pathtype=rdf_paths.PathSpec.PathType.TSK)
p1 = rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS)
p1.Append(p2)
fd = vfs.VFSOpen(p1)
self.TestFileHandling(fd)
def testUnicodeFile(self):
"""Test ability to read unicode files from images."""
path = os.path.join(self.base_path, "test_img.dd")
path2 = os.path.join(u"איןד ןד ש אקדא", u"איןד.txt")
ps2 = rdf_paths.PathSpec(path=path2,
pathtype=rdf_paths.PathSpec.PathType.TSK)
ps = rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS)
ps.Append(ps2)
fd = vfs.VFSOpen(ps)
self.TestFileHandling(fd)
def testRecursiveImages(self):
"""Test directory listing in sleuthkit."""
p3 = rdf_paths.PathSpec(path="/home/a.txt",
pathtype=rdf_paths.PathSpec.PathType.TSK)
p2 = rdf_paths.PathSpec(path="/home/image2.img",
pathtype=rdf_paths.PathSpec.PathType.TSK)
p1 = rdf_paths.PathSpec(path=os.path.join(self.base_path,
"test_img.dd"),
pathtype=rdf_paths.PathSpec.PathType.OS)
p2.Append(p3)
p1.Append(p2)
f = vfs.VFSOpen(p1)
self.assertEqual(f.read(3), "yay")
def Parse(self, persistence, knowledge_base, download_pathtype):
"""Convert persistence collector output to downloadable rdfvalues."""
pathspecs = []
if isinstance(persistence, rdf_client.OSXServiceInformation):
if persistence.program:
pathspecs = rdf_paths.PathSpec(
path=persistence.program, pathtype=download_pathtype)
elif persistence.args:
pathspecs = rdf_paths.PathSpec(
path=persistence.args[0], pathtype=download_pathtype)
for pathspec in pathspecs:
yield rdf_standard.PersistenceFile(pathspec=pathspec)
def testFindFilesWithGlob(self):
"""Test that the Find flow works with glob."""
client_mock = action_mocks.ActionMock(searching.Find)
# Prepare a findspec.
findspec = rdf_client.FindSpec(
path_glob="bash*",
pathspec=rdf_paths.PathSpec(
path="/", pathtype=rdf_paths.PathSpec.PathType.OS))
session_id = flow_test_lib.TestFlowHelper(find.FindFiles.__name__,
client_mock,
client_id=self.client_id,
token=self.token,
findspec=findspec)
# Check the results collection.
fd = flow.GRRFlow.ResultCollectionForFID(session_id)
# Make sure that bash is a file.
matches = set([x.AFF4Path(self.client_id).Basename() for x in fd])
self.assertEqual(sorted(matches), ["bash"])
self.assertEqual(len(fd), 2)
for child in fd:
path = utils.SmartStr(child.AFF4Path(self.client_id))
self.assertTrue(path.endswith("bash"))
self.assertEqual(child.__class__.__name__, "StatEntry")
def testFindDirectories(self):
"""Test that the Find flow works with directories."""
client_mock = action_mocks.ActionMock(searching.Find)
# Prepare a findspec.
findspec = rdf_client.FindSpec(
path_regex="bin",
pathspec=rdf_paths.PathSpec(
path="/", pathtype=rdf_paths.PathSpec.PathType.OS))
session_id = flow_test_lib.TestFlowHelper(find.FindFiles.__name__,
client_mock,
client_id=self.client_id,
token=self.token,
findspec=findspec)
# Check the results collection.
fd = flow.GRRFlow.ResultCollectionForFID(session_id)
# Make sure that bin is a directory
self.assertEqual(len(fd), 2)
for child in fd:
path = utils.SmartStr(child.AFF4Path(self.client_id))
self.assertTrue("bin" in path)
self.assertEqual(child.__class__.__name__, "StatEntry")
def testMultiGetFileSetsFileHashAttributeWhenMultipleChunksDownloaded(
self):
client_mock = action_mocks.MultiGetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
args = transfer.MultiGetFileArgs(pathspecs=[pathspec])
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args)
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = pathspec.AFF4Path(self.client_id)
fd_hash = data_store_utils.GetUrnHashEntry(urn)
self.assertTrue(fd_hash)
h = hashlib.sha256()
with open(os.path.join(self.base_path, "test_img.dd"),
"rb") as model_fd:
h.update(model_fd.read())
self.assertEqual(fd_hash.sha256, h.digest())
def testMultiGetFileSizeLimit(self):
client_mock = action_mocks.MultiGetFileClientMock()
image_path = os.path.join(self.base_path, "test_img.dd")
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=image_path)
# Read a bit more than one chunk (600 * 1024).
expected_size = 750 * 1024
args = transfer.MultiGetFileArgs(pathspecs=[pathspec],
file_size=expected_size)
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args)
urn = pathspec.AFF4Path(self.client_id)
blobimage = aff4.FACTORY.Open(urn, token=self.token)
# Make sure a VFSBlobImage got written.
self.assertTrue(isinstance(blobimage, aff4_grr.VFSBlobImage))
self.assertEqual(len(blobimage), expected_size)
data = blobimage.read(100 * expected_size)
self.assertEqual(len(data), expected_size)
expected_data = open(image_path, "rb").read(expected_size)
self.assertEqual(data, expected_data)
hash_obj = data_store_utils.GetFileHashEntry(blobimage)
d = hashlib.sha1()
d.update(expected_data)
expected_hash = d.hexdigest()
self.assertEqual(hash_obj.sha1, expected_hash)
def testMultiGetFile(self):
"""Test MultiGetFile."""
client_mock = action_mocks.MultiGetFileClientMock()
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "test_img.dd"))
args = transfer.MultiGetFileArgs(pathspecs=[pathspec, pathspec])
with test_lib.Instrument(transfer.MultiGetFile,
"StoreStat") as storestat_instrument:
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args)
# We should only have called StoreStat once because the two paths
# requested were identical.
self.assertEqual(len(storestat_instrument.args), 1)
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
fd2 = open(pathspec.path, "rb")
fd2.seek(0, 2)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def testMultiGetFileDeduplication(self):
client_mock = action_mocks.MultiGetFileClientMock()
pathspecs = []
# Make 10 files to download.
for i in xrange(10):
path = os.path.join(self.temp_dir, "test_%s.txt" % i)
with open(path, "wb") as fd:
fd.write("Hello")
pathspecs.append(
rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=path))
# All those files are the same so the individual chunks should
# only be downloaded once. By forcing maximum_pending_files=1,
# there should only be a single TransferBuffer call.
args = transfer.MultiGetFileArgs(pathspecs=pathspecs,
maximum_pending_files=1)
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args)
self.assertEqual(client_mock.action_counts["TransferBuffer"], 1)
def testGetFilePathCorrection(self):
"""Tests that the pathspec returned is used for the aff4path."""
client_mock = action_mocks.GetFileClientMock()
# Deliberately using the wrong casing.
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(
self.base_path, "TEST_IMG.dd"))
session_id = flow_test_lib.TestFlowHelper(transfer.GetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
pathspec=pathspec)
results = flow.GRRFlow.ResultCollectionForFID(session_id)
self.assertEqual(len(results), 1)
res_pathspec = results[0].pathspec
# Fix path for Windows testing.
pathspec.path = pathspec.path.replace("\\", "/")
# Test the AFF4 file that was created.
urn = res_pathspec.AFF4Path(self.client_id)
fd1 = aff4.FACTORY.Open(urn, token=self.token)
fd2 = open(res_pathspec.path, "rb")
fd2.seek(0, 2)
self.assertEqual(fd2.tell(), int(fd1.Get(fd1.Schema.SIZE)))
self.CompareFDs(fd1, fd2)
def Run(self, args):
if platform.system() == "Windows":
raise RuntimeError("os.statvfs not available on Windows")
for path in args.path_list:
try:
fd = vfs.VFSOpen(rdf_paths.PathSpec(path=path,
pathtype=args.pathtype),
progress_callback=self.Progress)
st = fd.StatFS()
mount_point = fd.GetMountPoint()
except (IOError, OSError) as e:
self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.IOERROR, e)
continue
unix = rdf_client.UnixVolume(mount_point=mount_point)
# On linux pre 2.6 kernels don't have frsize, so we fall back to bsize.
# The actual_available_allocation_units attribute is set to blocks
# available to the unprivileged user, root may have some additional
# reserved space.
self.SendReply(
rdf_client.Volume(
bytes_per_sector=(st.f_frsize or st.f_bsize),
sectors_per_allocation_unit=1,
total_allocation_units=st.f_blocks,
actual_available_allocation_units=st.f_bavail,
unixvolume=unix))
def PopulateCache(self):
"""Parse the paths from the fixture."""
if self.paths:
return
# The cache is attached to the class so it can be shared by all instance.
self.paths = self.__class__.cache[self.prefix] = {}
for path, (vfs_type, attributes) in client_fixture.VFS:
if not path.startswith(self.prefix):
continue
path = utils.NormalizePath(path[len(self.prefix):])
if path == "/":
continue
stat = rdf_client.StatEntry()
args = {"client_id": "C.1234"}
attrs = attributes.get("aff4:stat")
if attrs:
attrs %= args # Remove any %% and interpolate client_id.
stat = rdf_client.StatEntry.FromTextFormat(
utils.SmartStr(attrs))
stat.pathspec = rdf_paths.PathSpec(
pathtype=self.supported_pathtype, path=path)
# TODO(user): Once we add tests around not crossing device boundaries,
# we need to be smarter here, especially for the root entry.
stat.st_dev = 1
path = self._NormalizeCaseForPath(path, vfs_type)
self.paths[path] = (vfs_type, stat)
self.BuildIntermediateDirectories()
def testCronJobPreservesFlowNameAndArguments(self):
"""Testing initialization of a ConfigManager."""
pathspec = rdf_paths.PathSpec(path="/foo",
pathtype=rdf_paths.PathSpec.PathType.TSK)
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity="1d",
allow_overruns=False)
cron_args.flow_runner_args.flow_name = transfer.GetFile.__name__
cron_args.flow_args.pathspec = pathspec
job_id = cron_manager.CreateJob(cron_args=cron_args)
# Check that CronJob definition is saved properly
jobs = cron_manager.ListJobs(token=self.token)
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0], job_id)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertEqual(cron_job.cron_args.flow_runner_args.flow_name,
transfer.GetFile.__name__)
self.assertEqual(cron_job.cron_args.flow_args.pathspec, pathspec)
self.assertEqual(cron_job.cron_args.periodicity,
rdfvalue.Duration("1d"))
self.assertEqual(cron_job.cron_args.allow_overruns, False)
def GetRegistryValue(self, source):
"""Retrieve directly specified registry values, returning Stat objects."""
new_paths = set()
has_glob = False
for kvdict in source.attributes["key_value_pairs"]:
if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search(
kvdict["key"]):
has_glob = True
if kvdict["value"]:
# This currently only supports key value pairs specified using forward
# slash.
path = "\\".join((kvdict["key"], kvdict["value"]))
else:
# If value is not set, we want to get the default value. In
# GRR this is done by specifying the key only, so this is what
# we do here.
path = kvdict["key"]
expanded_paths = artifact_utils.InterpolateKbAttributes(
path,
self.state.knowledge_base,
ignore_errors=self.args.ignore_interpolation_errors)
new_paths.update(expanded_paths)
if has_glob:
self.CallFlow(filesystem.Glob.__name__,
paths=new_paths,
pathtype=rdf_paths.PathSpec.PathType.REGISTRY,
request_data={
"artifact_name": self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollected")
else:
# We call statfile directly for keys that don't include globs because it
# is faster and some artifacts rely on getting an IOError to trigger
# fallback processing.
for new_path in new_paths:
pathspec = rdf_paths.PathSpec(
path=new_path,
pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
# TODO(hanuszczak): Support for old clients ends on 2021-01-01.
# This conditional should be removed after that date.
if self.client_version >= 3221:
stub = server_stubs.GetFileStat
request = rdf_client.GetFileStatRequest(pathspec=pathspec)
else:
stub = server_stubs.StatFile
request = rdf_client.ListDirRequest(pathspec=pathspec)
self.CallClient(stub,
request,
request_data={
"artifact_name":
self.current_artifact_name,
"source": source.ToPrimitiveDict()
},
next_state="ProcessCollectedRegistryStatEntry")
def testListDirectory(self):
"""Test our ability to list a directory."""
directory = vfs.VFSOpen(
rdf_paths.PathSpec(path=self.base_path,
pathtype=rdf_paths.PathSpec.PathType.OS))
self.CheckDirectoryListing(directory, "morenumbers.txt")
def Start(self):
"""Schedules the ReadBuffer client action."""
pathspec = rdf_paths.PathSpec(
path="\\\\.\\PhysicalDrive0\\",
pathtype=rdf_paths.PathSpec.PathType.OS,
path_options=rdf_paths.PathSpec.Options.CASE_LITERAL)
self.state.bytes_downloaded = 0
# An array to collect buffers. This is not very efficient, MBR
# data should be kept short though so this is not a big deal.
self.state.buffers = []
buffer_size = constants.CLIENT_MAX_BUFFER_SIZE
buffers_we_need = self.args.length / buffer_size
if self.args.length % buffer_size:
buffers_we_need += 1
bytes_we_need = self.args.length
for i in xrange(buffers_we_need):
request = rdf_client.BufferReference(
pathspec=pathspec,
offset=i * buffer_size,
length=min(bytes_we_need, buffer_size))
self.CallClient(server_stubs.ReadBuffer, request, next_state="StoreMBR")
bytes_we_need -= buffer_size
def testFileSizeDistributionReportPlugin(self):
filename = "winexec_img.dd"
client_id = self.SetupClient(0)
# Add a file to be reported.
filestore_test_lib.AddFileToFileStore(rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS,
path=os.path.join(self.base_path, filename)),
client_id=client_id,
token=self.token)
# Scan for files to be reported (the one we just added).
flow_test_lib.TestFlowHelper(
filestore_stats.FilestoreStatsCronFlow.__name__, token=self.token)
report = report_plugins.GetReportByName(
filestore_report_plugins.FileSizeDistributionReportPlugin.__name__)
api_report_data = report.GetReportData(
stats_api.ApiGetReportArgs(name=report.__class__.__name__),
token=self.token)
self.checkStaticData(api_report_data)
for series in api_report_data.stack_chart.data:
if series.label == "976.6 KiB - 4.8 MiB":
self.assertEqual([p.y for p in series.points], [1])
else:
self.assertEqual([p.y for p in series.points], [0])
def testRecursiveListNames(self):
"""Test our ability to walk over a directory tree."""
path = os.path.join(self.base_path, "a")
directory = vfs.VFSOpen(
rdf_paths.PathSpec(path=path,
pathtype=rdf_paths.PathSpec.PathType.OS))
# Test the helper method
self.assertEqual(directory._GetDepth("/"), 0)
self.assertEqual(directory._GetDepth("/foo/bar/baz"), 3)
# Relative paths aren't supported
with self.assertRaises(RuntimeError):
directory._GetDepth("foo/bar")
# Multiple separators are redundant
self.assertEqual(directory._GetDepth("/////foo///bar//////baz//"), 3)
# Test the whole thing
walk_tups_0 = list(directory.RecursiveListNames())
walk_tups_1 = list(directory.RecursiveListNames(depth=1))
walk_tups_2 = list(directory.RecursiveListNames(depth=2))
walk_tups_inf = list(directory.RecursiveListNames(depth=float("inf")))
self.assertEqual(walk_tups_0, [(path, ["b"], [])])
self.assertEqual(walk_tups_1, [(path, ["b"], []),
("%s/b" % path, ["c", "d"], [])])
self.assertEqual(walk_tups_2, [(path, ["b"], []),
("%s/b" % path, ["c", "d"], []),
("%s/b/c" % path, [], ["helloc.txt"]),
("%s/b/d" % path, [], ["hellod.txt"])])
self.assertEqual(walk_tups_inf,
[(path, ["b"], []), ("%s/b" % path, ["c", "d"], []),
("%s/b/c" % path, [], ["helloc.txt"]),
("%s/b/d" % path, [], ["hellod.txt"])])
