def testMultiGetFileSizeLimit(self):
client_mock = action_mocks.MultiGetFileClientMock()
image_path = os.path.join(self.base_path, "test_img.dd")
pathspec = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS,
path=image_path)
# Read a bit more than one chunk (600 * 1024).
expected_size = 750 * 1024
args = transfer.MultiGetFileArgs(pathspecs=[pathspec],
file_size=expected_size)
flow_test_lib.TestFlowHelper(transfer.MultiGetFile.__name__,
client_mock,
token=self.token,
client_id=self.client_id,
args=args)
urn = pathspec.AFF4Path(self.client_id)
blobimage = aff4.FACTORY.Open(urn, token=self.token)
# Make sure a VFSBlobImage got written.
self.assertTrue(isinstance(blobimage, aff4_grr.VFSBlobImage))
self.assertEqual(len(blobimage), expected_size)
data = blobimage.read(100 * expected_size)
self.assertEqual(len(data), expected_size)
expected_data = open(image_path, "rb").read(expected_size)
self.assertEqual(data, expected_data)
hash_obj = data_store_utils.GetFileHashEntry(blobimage)
d = hashlib.sha1()
d.update(expected_data)
expected_hash = d.hexdigest()
self.assertEqual(hash_obj.sha1, expected_hash)
def testClientFileFinderUpload(self):
paths = [os.path.join(self.base_path, "{**,.}/*.plist")]
action = rdf_file_finder.FileFinderAction.Download()
session_id = self._RunClientFileFinder(paths, action)
collection = flow.GRRFlow.ResultCollectionForFID(session_id)
results = list(collection)
self.assertEqual(len(results), 4)
relpaths = [
os.path.relpath(p.stat_entry.pathspec.path, self.base_path)
for p in results
]
self.assertItemsEqual(relpaths, [
"History.plist", "History.xml.plist", "test.plist",
"parser_test/com.google.code.grr.plist"
])
for r in results:
aff4_obj = aff4.FACTORY.Open(
r.stat_entry.pathspec.AFF4Path(self.client_id), token=self.token)
data = open(r.stat_entry.pathspec.path, "rb").read()
self.assertEqual(aff4_obj.Read(100), data[:100])
hash_obj = data_store_utils.GetFileHashEntry(aff4_obj)
self.assertEqual(hash_obj.md5, hashlib.md5(data).hexdigest())
self.assertEqual(hash_obj.sha1, hashlib.sha1(data).hexdigest())
self.assertEqual(hash_obj.sha256, hashlib.sha256(data).hexdigest())
def testDownloadActionSizeLimitWithDownloadTruncatedPolicy(self):
image_path = os.path.join(self.base_path, "test_img.dd")
# Read a bit more than a typical chunk (600 * 1024).
expected_size = 750 * 1024
action = rdf_file_finder.FileFinderAction.Download(
max_size=expected_size, oversized_file_policy="DOWNLOAD_TRUNCATED")
results = self.RunFlow(paths=[image_path], action=action)
urn = rdfvalue.RDFURN(self.client_id).Add("/fs/os").Add(image_path)
blobimage = aff4.FACTORY.Open(urn, token=self.token)
# Make sure a VFSBlobImage got written.
self.assertTrue(isinstance(blobimage, aff4_grr.VFSBlobImage))
self.assertEqual(len(blobimage), expected_size)
data = blobimage.read(100 * expected_size)
self.assertEqual(len(data), expected_size)
expected_data = open(image_path, "rb").read(expected_size)
self.assertEqual(data, expected_data)
hash_obj = data_store_utils.GetFileHashEntry(blobimage)
d = hashlib.sha1()
d.update(expected_data)
expected_hash = d.hexdigest()
self.assertEqual(hash_obj.sha1, expected_hash)
unused_flow, flow_reply = results[0]
self.assertEqual(flow_reply.hash_entry.sha1, expected_hash)
def testDownloadAndHashActionSizeLimitWithHashTruncatedPolicy(self):
image_path = os.path.join(self.base_path, "test_img.dd")
# Read a bit more than a typical chunk (600 * 1024).
expected_size = 750 * 1024
hash_action = rdf_file_finder.FileFinderAction.Hash(
max_size=expected_size, oversized_file_policy="HASH_TRUNCATED")
download_action = rdf_file_finder.FileFinderAction.Download(
max_size=expected_size, oversized_file_policy="HASH_TRUNCATED")
for action in [hash_action, download_action]:
results = self.RunFlow(paths=[image_path], action=action)
urn = rdfvalue.RDFURN(self.client_id).Add("/fs/os").Add(image_path)
vfs_file = aff4.FACTORY.Open(urn, token=self.token)
# Make sure just a VFSFile got written.
self.assertTrue(isinstance(vfs_file, aff4_grr.VFSFile))
expected_data = open(image_path, "rb").read(expected_size)
d = hashlib.sha1()
d.update(expected_data)
expected_hash = d.hexdigest()
hash_entry = data_store_utils.GetFileHashEntry(vfs_file)
self.assertEqual(hash_entry.sha1, expected_hash)
unused_flow, flow_reply = results[0]
self.assertEqual(flow_reply.hash_entry.sha1, expected_hash)
def testClientFileFinderFilestoreIntegration(self):
paths = [os.path.join(self.base_path, "{**,.}/*.plist")]
action = rdf_file_finder.FileFinderAction.Download()
client_ids = self.SetupClients(2)
session_ids = {
c: self._RunClientFileFinder(paths, action, client_id=c)
for c in client_ids
}
collections = {
c: flow.GRRFlow.ResultCollectionForFID(session_id)
for c, session_id in session_ids.iteritems()
}
for client_id, collection in collections.iteritems():
results = list(collection)
self.assertEqual(len(results), 4)
relpaths = [
os.path.relpath(p.stat_entry.pathspec.path, self.base_path)
for p in results
]
self.assertItemsEqual(relpaths, [
"History.plist", "History.xml.plist", "test.plist",
"parser_test/com.google.code.grr.plist"
])
for r in results:
aff4_obj = aff4.FACTORY.Open(
r.stat_entry.pathspec.AFF4Path(client_id),
token=self.token)
# When files are uploaded to the server they are stored as VFSBlobImage.
self.assertIsInstance(aff4_obj, aff4_grr.VFSBlobImage)
# There is a STAT entry.
self.assertTrue(aff4_obj.Get(aff4_obj.Schema.STAT))
# Make sure the HashFileStore has references to this file for
# all hashes.
hashes = data_store_utils.GetFileHashEntry(aff4_obj)
fs = filestore.HashFileStore
md5_refs = list(
fs.GetReferencesMD5(hashes.md5, token=self.token))
self.assertIn(aff4_obj.urn, md5_refs)
sha1_refs = list(
fs.GetReferencesSHA1(hashes.sha1, token=self.token))
self.assertIn(aff4_obj.urn, sha1_refs)
sha256_refs = list(
fs.GetReferencesSHA256(hashes.sha256, token=self.token))
self.assertIn(aff4_obj.urn, sha256_refs)
# Open the file inside the file store.
urn, _ = fs(None, token=self.token).CheckHashes(hashes).next()
filestore_fd = aff4.FACTORY.Open(urn, token=self.token)
# This is a VFSBlobImage too.
self.assertIsInstance(filestore_fd, aff4_grr.VFSBlobImage)
# No STAT object attached.
self.assertFalse(filestore_fd.Get(filestore_fd.Schema.STAT))
def _HashFile(self, fd):
"""Look for the required hashes in the file."""
hashes = data_store_utils.GetFileHashEntry(fd)
if hashes:
found_all = True
for fingerprint_type, hash_types in self.HASH_TYPES.iteritems():
for hash_type in hash_types:
if fingerprint_type == "pecoff":
hash_type = "pecoff_%s" % hash_type
if not hashes.HasField(hash_type):
found_all = False
break
if not found_all:
break
if found_all:
return hashes
fingerprinter = fingerprint.Fingerprinter(fd)
if "generic" in self.HASH_TYPES:
hashers = self._GetHashers(self.HASH_TYPES["generic"])
fingerprinter.EvalGeneric(hashers=hashers)
if "pecoff" in self.HASH_TYPES:
hashers = self._GetHashers(self.HASH_TYPES["pecoff"])
if hashers:
fingerprinter.EvalPecoff(hashers=hashers)
if not hashes:
hashes = fd.Schema.HASH()
for result in fingerprinter.HashIt():
fingerprint_type = result["name"]
for hash_type in self.HASH_TYPES[fingerprint_type]:
if hash_type not in result:
continue
if hash_type == "SignedData":
# There can be several certs in the same file.
for signed_data in result[hash_type]:
hashes.signed_data.Append(
revision=signed_data[0],
cert_type=signed_data[1],
certificate=signed_data[2])
continue
# Set the hashes in the original object
if fingerprint_type == "generic":
hashes.Set(hash_type, result[hash_type])
elif fingerprint_type == "pecoff":
hashes.Set("pecoff_%s" % hash_type, result[hash_type])
else:
logging.error("Unknown fingerprint_type %s.", fingerprint_type)
return hashes
