def testWithFiles(self):
foo = test_lib.AutoTempFilePath(suffix="foo")
bar = test_lib.AutoTempFilePath(suffix="bar")
baz = test_lib.AutoTempFilePath(suffix="baz")
with utils.MultiContext([foo, bar, baz]) as filepaths:
self.assertEqual(len(filepaths), 3)
self.assertTrue(filepaths[0].endswith("foo"))
self.assertTrue(filepaths[1].endswith("bar"))
self.assertTrue(filepaths[2].endswith("baz"))
wbopen = functools.partial(io.open, mode="wb")
with utils.MultiContext(map(wbopen, filepaths)) as filedescs:
self.assertEqual(len(filedescs), 3)
filedescs[0].write(b"FOO")
filedescs[1].write(b"BAR")
filedescs[2].write(b"BAZ")
# At this point all three files should be correctly written, closed and
# ready for reading.
rbopen = functools.partial(io.open, mode="rb")
with utils.MultiContext(map(rbopen, filepaths)) as filedescs:
self.assertEqual(len(filedescs), 3)
self.assertEqual(filedescs[0].read(), b"FOO")
self.assertEqual(filedescs[1].read(), b"BAR")
self.assertEqual(filedescs[2].read(), b"BAZ")
def testGetSize(self):
with test_lib.AutoTempFilePath() as temp_filepath:
with open(temp_filepath, "wb") as fd:
fd.write("foobarbaz")
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertEqual(stat.GetSize(), 9)
def testRegular(self):
with test_lib.AutoTempFilePath() as temp_filepath:
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertFalse(stat.IsDirectory())
self.assertTrue(stat.IsRegular())
self.assertFalse(stat.IsSocket())
self.assertFalse(stat.IsSymlink())
def testManyChunks(self):
action = FakeAction()
uploader = uploading.TransferStoreUploader(action, chunk_size=3)
with test_lib.AutoTempFilePath() as temp_filepath:
with open(temp_filepath, "w") as temp_file:
temp_file.write("1234567890")
blobdesc = uploader.UploadFilePath(temp_filepath)
self.assertEqual(action.charged_bytes, 10)
self.assertEqual(len(action.messages), 4)
self.assertEqual(action.messages[0].item.data,
zlib.compress("123"))
self.assertEqual(action.messages[1].item.data,
zlib.compress("456"))
self.assertEqual(action.messages[2].item.data,
zlib.compress("789"))
self.assertEqual(action.messages[3].item.data, zlib.compress("0"))
self.assertEqual(len(blobdesc.chunks), 4)
self.assertEqual(blobdesc.chunk_size, 3)
self.assertEqual(blobdesc.chunks[0].offset, 0)
self.assertEqual(blobdesc.chunks[0].length, 3)
self.assertEqual(blobdesc.chunks[0].digest, Sha256("123"))
self.assertEqual(blobdesc.chunks[1].offset, 3)
self.assertEqual(blobdesc.chunks[1].length, 3)
self.assertEqual(blobdesc.chunks[1].digest, Sha256("456"))
self.assertEqual(blobdesc.chunks[2].offset, 6)
self.assertEqual(blobdesc.chunks[2].length, 3)
self.assertEqual(blobdesc.chunks[2].digest, Sha256("789"))
self.assertEqual(blobdesc.chunks[3].offset, 9)
self.assertEqual(blobdesc.chunks[3].length, 1)
self.assertEqual(blobdesc.chunks[3].digest, Sha256("0"))
def testCustomOffset(self):
action = FakeAction()
uploader = uploading.TransferStoreUploader(action, chunk_size=2)
with test_lib.AutoTempFilePath() as temp_filepath:
with open(temp_filepath, "w") as temp_file:
temp_file.write("0123456")
blobdesc = uploader.UploadFilePath(temp_filepath, offset=2)
self.assertEqual(action.charged_bytes, 5)
self.assertEqual(len(action.messages), 3)
self.assertEqual(action.messages[0].item.data, zlib.compress("23"))
self.assertEqual(action.messages[1].item.data, zlib.compress("45"))
self.assertEqual(action.messages[2].item.data, zlib.compress("6"))
self.assertEqual(len(blobdesc.chunks), 3)
self.assertEqual(blobdesc.chunk_size, 2)
self.assertEqual(blobdesc.chunks[0].offset, 2)
self.assertEqual(blobdesc.chunks[0].length, 2)
self.assertEqual(blobdesc.chunks[0].digest, Sha256("23"))
self.assertEqual(blobdesc.chunks[1].offset, 4)
self.assertEqual(blobdesc.chunks[1].length, 2)
self.assertEqual(blobdesc.chunks[1].digest, Sha256("45"))
self.assertEqual(blobdesc.chunks[2].offset, 6)
self.assertEqual(blobdesc.chunks[2].length, 1)
self.assertEqual(blobdesc.chunks[2].digest, Sha256("6"))
def testStatExtAttrs(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.SetExtAttr(temp_filepath,
name="user.foo",
value="bar")
client_test_lib.SetExtAttr(temp_filepath,
name="user.quux",
value="norf")
action = rdf_file_finder.FileFinderAction.Stat()
results = self._RunFileFinder([temp_filepath], action)
self.assertEqual(len(results), 1)
ext_attrs = results[0].stat_entry.ext_attrs
self.assertEqual(ext_attrs[0].name, "user.foo")
self.assertEqual(ext_attrs[0].value, "bar")
self.assertEqual(ext_attrs[1].name, "user.quux")
self.assertEqual(ext_attrs[1].value, "norf")
action = rdf_file_finder.FileFinderAction.Stat(
collect_ext_attrs=False)
results = self._RunFileFinder([temp_filepath], action)
self.assertEqual(len(results), 1)
ext_attrs = results[0].stat_entry.ext_attrs
self.assertFalse(ext_attrs)
def testGetOsxFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
subprocess.check_call(["chflags", "nodump hidden", temp_filepath])
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertTrue(stat.IsRegular())
self.assertTrue(stat.GetOsxFlags() & self.UF_NODUMP)
self.assertTrue(stat.GetOsxFlags() & self.UF_HIDDEN)
self.assertFalse(stat.GetOsxFlags() & self.UF_IMMUTABLE)
self.assertEqual(stat.GetLinuxFlags(), 0)
def testGetLinuxFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.Chattr(temp_filepath, attrs=["+c", "+d"])
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertTrue(stat.IsRegular())
self.assertTrue(stat.GetLinuxFlags() & self.FS_COMPR_FL)
self.assertTrue(stat.GetLinuxFlags() & self.FS_NODUMP_FL)
self.assertFalse(stat.GetLinuxFlags() & self.FS_IMMUTABLE_FL)
self.assertEqual(stat.GetOsxFlags(), 0)
def testGetOsxFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.Chflags(temp_filepath, flags=["nodump", "hidden"])
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertTrue(stat.IsRegular())
self.assertTrue(stat.GetOsxFlags() & self.UF_NODUMP)
self.assertTrue(stat.GetOsxFlags() & self.UF_HIDDEN)
self.assertFalse(stat.GetOsxFlags() & self.UF_IMMUTABLE)
self.assertEqual(stat.GetLinuxFlags(), 0)
def testGetFlagsSymlink(self):
with test_lib.AutoTempDirPath(remove_non_empty=True) as temp_dirpath,\
test_lib.AutoTempFilePath() as temp_filepath:
temp_linkpath = os.path.join(temp_dirpath, "foo")
os.symlink(temp_filepath, temp_linkpath)
stat = utils.Stat(temp_linkpath, follow_symlink=False)
self.assertTrue(stat.IsSymlink())
self.assertEqual(stat.GetLinuxFlags(), 0)
self.assertEqual(stat.GetOsxFlags(), 0)
def testAttrChangeAfterListing(self, listxattr):
with test_lib.AutoTempFilePath() as temp_filepath:
self._SetAttr(temp_filepath, "user.bar", "baz")
attrs = list(client_utils_linux.GetExtAttrs(temp_filepath))
self.assertTrue(listxattr.called)
self.assertEqual(len(attrs), 1)
self.assertEqual(attrs[0].name, "user.bar")
self.assertEqual(attrs[0].value, "baz")
def testStatExtFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.Chattr(temp_filepath, attrs=["+c"])
action = rdf_file_finder.FileFinderAction.Stat()
results = self._RunFileFinder([temp_filepath], action)
self.assertEqual(len(results), 1)
stat_entry = results[0].stat_entry
self.assertTrue(stat_entry.st_flags_linux & self.EXT2_COMPR_FL)
self.assertFalse(stat_entry.st_flags_linux & self.EXT2_IMMUTABLE_FL)
def testEmpty(self):
action = FakeAction()
uploader = uploading.TransferStoreUploader(action, chunk_size=3)
with test_lib.AutoTempFilePath() as temp_filepath:
blobdesc = uploader.UploadFilePath(temp_filepath)
self.assertEqual(action.charged_bytes, 0)
self.assertEqual(len(action.messages), 0)
self.assertEqual(len(blobdesc.chunks), 0)
def testGetTime(self):
adate = datetime.datetime(2017, 10, 2, 8, 45)
mdate = datetime.datetime(2001, 5, 3, 10, 30)
with test_lib.AutoTempFilePath() as temp_filepath:
self._Touch(temp_filepath, "-a", adate)
self._Touch(temp_filepath, "-m", mdate)
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertEqual(stat.GetAccessTime(), self._EpochMillis(adate))
self.assertEqual(stat.GetModificationTime(), self._EpochMillis(mdate))
def testFileFinderStatExtFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.Chattr(temp_filepath, attrs=["+d"])
action = rdf_file_finder.FileFinderAction.Stat()
results = self.RunFlow(action=action, paths=[temp_filepath])
self.assertEqual(len(results), 1)
stat_entry = results[0].stat_entry
self.assertTrue(stat_entry.st_flags_linux & self.FS_NODUMP_FL)
self.assertFalse(stat_entry.st_flags_linux & self.FS_UNRM_FL)
def testMany(self):
with test_lib.AutoTempFilePath() as temp_filepath:
self._SetAttr(temp_filepath, "user.foo", "bar")
self._SetAttr(temp_filepath, "user.quux", "norf")
attrs = list(client_utils_linux.GetExtAttrs(temp_filepath))
self.assertEqual(len(attrs), 2)
self.assertEqual(attrs[0].name, "user.foo")
self.assertEqual(attrs[0].value, "bar")
self.assertEqual(attrs[1].name, "user.quux")
self.assertEqual(attrs[1].value, "norf")
def testStatExtAttrsDisabled(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.SetExtAttr(temp_filepath, name="user.foo", value="bar")
pathspec = rdf_paths.PathSpec(
path=temp_filepath, pathtype=rdf_paths.PathSpec.PathType.OS)
request = rdf_client_action.GetFileStatRequest(
pathspec=pathspec, collect_ext_attrs=False)
results = self.RunAction(standard.GetFileStat, request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].ext_attrs), 0)
def testStatSize(self):
with test_lib.AutoTempFilePath() as temp_filepath:
with open(temp_filepath, "wb") as temp_file:
temp_file.write("123456")
pathspec = rdf_paths.PathSpec(
path=temp_filepath, pathtype=rdf_paths.PathSpec.PathType.OS)
request = rdf_client_action.GetFileStatRequest(pathspec=pathspec)
results = self.RunAction(standard.GetFileStat, request)
self.assertEqual(len(results), 1)
self.assertEqual(results[0].st_size, 6)
def testHashFilePart(self):
with test_lib.AutoTempFilePath() as tmp_path:
with open(tmp_path, "wb") as tmp_file:
tmp_file.write("foobar")
hasher = client_utils_common.MultiHasher(["md5", "sha1"])
hasher.HashFilePath(tmp_path, len("foo"))
hash_object = hasher.GetHashObject()
self.assertEqual(hash_object.num_bytes, len("foo"))
self.assertEqual(hash_object.md5, self._GetHash(hashlib.md5, "foo"))
self.assertEqual(hash_object.sha1, self._GetHash(hashlib.sha1, "foo"))
self.assertFalse(hash_object.sha256)
def testFileFinderStatExtAttrs(self):
with test_lib.AutoTempFilePath() as temp_filepath:
client_test_lib.SetExtAttr(temp_filepath, name="user.bar", value="baz")
client_test_lib.SetExtAttr(temp_filepath, name="user.quux", value="norf")
action = rdf_file_finder.FileFinderAction.Stat()
results = self.RunFlow(action=action, paths=[temp_filepath])
self.assertEqual(len(results), 1)
stat_entry = results[0][1].stat_entry
self.assertItemsEqual(stat_entry.ext_attrs, [
rdf_client.ExtAttr(name="user.bar", value="baz"),
rdf_client.ExtAttr(name="user.quux", value="norf"),
])
def testGetLinuxFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
if subprocess.call(["which", "chattr"]) != 0:
raise unittest.SkipTest("`chattr` command is not available")
if subprocess.call(["chattr", "+c", "+d", temp_filepath]):
reason = "extended attributes not supported by filesystem"
raise unittest.SkipTest(reason)
stat = utils.Stat(temp_filepath, follow_symlink=False)
self.assertTrue(stat.IsRegular())
self.assertTrue(stat.GetLinuxFlags() & self.FS_COMPR_FL)
self.assertTrue(stat.GetLinuxFlags() & self.FS_NODUMP_FL)
self.assertFalse(stat.GetLinuxFlags() & self.FS_IMMUTABLE_FL)
self.assertEqual(stat.GetOsxFlags(), 0)
def testFileFinderStatExtFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
if subprocess.call(["which", "chattr"]) != 0:
raise unittest.SkipTest("`chattr` command is not available")
if subprocess.call(["chattr", "+d", temp_filepath]) != 0:
reason = "extended attributes not supported by filesystem"
raise unittest.SkipTest(reason)
action = rdf_file_finder.FileFinderAction.Stat()
results = self.RunFlow(action=action, paths=[temp_filepath])
self.assertEqual(len(results), 1)
stat_entry = results[0][1].stat_entry
self.assertTrue(stat_entry.st_flags_linux & self.FS_NODUMP_FL)
self.assertFalse(stat_entry.st_flags_linux & self.FS_UNRM_FL)
def testStatExtFlags(self):
with test_lib.AutoTempFilePath() as temp_filepath:
if subprocess.call(["which", "chattr"]) != 0:
raise unittest.SkipTest("`chattr` command is not available")
if subprocess.call(["chattr", "+c", temp_filepath]) != 0:
reason = "extended attributes not supported by filesystem"
raise unittest.SkipTest(reason)
action = self._StatAction()
results = self._RunFileFinder([temp_filepath], action)
self.assertEqual(len(results), 1)
stat_entry = results[0].stat_entry
self.assertTrue(stat_entry.st_flags_linux & self.EXT2_COMPR_FL)
self.assertFalse(stat_entry.st_flags_linux & self.EXT2_IMMUTABLE_FL)
def testProgressCallback(self):
with test_lib.AutoTempFilePath() as temppath:
self._Touch(temppath, b"QUUX")
pathspec = rdf_paths.PathSpec(
pathtype=rdf_paths.PathSpec.PathType.OS, path=temppath)
func = mock.MagicMock()
with vfs.VFSMultiOpen([pathspec],
progress_callback=func) as filedescs:
self.assertEqual(len(filedescs), 1)
self.assertEqual(filedescs[0].Read(), b"QUUX")
self.assertTrue(func.called)
def testSingleChunk(self):
action = FakeAction()
uploader = uploading.TransferStoreUploader(action, chunk_size=6)
with test_lib.AutoTempFilePath() as temp_filepath:
with open(temp_filepath, "w") as temp_file:
temp_file.write("foobar")
blobdesc = uploader.UploadFilePath(temp_filepath)
self.assertEqual(action.charged_bytes, 6)
self.assertEqual(len(action.messages), 1)
self.assertEqual(action.messages[0].item.data, zlib.compress("foobar"))
self.assertEqual(len(blobdesc.chunks), 1)
self.assertEqual(blobdesc.chunks[0].offset, 0)
self.assertEqual(blobdesc.chunks[0].length, 6)
self.assertEqual(blobdesc.chunks[0].digest, Sha256("foobar"))
def testFileFinderStatExtAttrs(self):
with test_lib.AutoTempFilePath() as temp_filepath:
if subprocess.call(["which", "setfattr"]) != 0:
raise unittest.SkipTest("`setfattr` command is not available")
subprocess.check_call(
["setfattr", temp_filepath, "-n", "user.bar", "-v", "baz"])
subprocess.check_call(
["setfattr", temp_filepath, "-n", "user.quux", "-v", "norf"])
action = rdf_file_finder.FileFinderAction.Stat()
results = self.RunFlow(action=action, paths=[temp_filepath])
self.assertEqual(len(results), 1)
stat_entry = results[0][1].stat_entry
self.assertItemsEqual(stat_entry.ext_attrs, [
rdf_client.ExtAttr(name="user.bar", value="baz"),
rdf_client.ExtAttr(name="user.quux", value="norf"),
])
def testStatExtAttrsDisabled(self):
with test_lib.AutoTempFilePath() as temp_filepath:
# TODO(hanuszczak): See a TODO comment above.
if subprocess.call(["which", "setfattr"]) != 0:
raise unittest.SkipTest("`setfattr` command is not available")
if subprocess.call([
"setfattr", temp_filepath, "-n", "user.foo", "-v", "bar"
]) != 0:
raise unittest.SkipTest("extended attributes not supported")
pathspec = rdf_paths.PathSpec(
path=temp_filepath, pathtype=rdf_paths.PathSpec.PathType.OS)
request = rdf_client.GetFileStatRequest(pathspec=pathspec,
collect_ext_attrs=False)
results = self.RunAction(standard.GetFileStat, request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].ext_attrs), 0)
def testStatExtAttrs(self):
with test_lib.AutoTempFilePath() as temp_filepath:
self._SetExtAttr(temp_filepath, "user.foo", "bar")
self._SetExtAttr(temp_filepath, "user.quux", "norf")
action = self._StatAction()
results = self._RunFileFinder([temp_filepath], action)
self.assertEqual(len(results), 1)
ext_attrs = results[0].stat_entry.ext_attrs
self.assertEqual(ext_attrs[0].name, "user.foo")
self.assertEqual(ext_attrs[0].value, "bar")
self.assertEqual(ext_attrs[1].name, "user.quux")
self.assertEqual(ext_attrs[1].value, "norf")
action = self._StatAction(ext_attrs=False)
results = self._RunFileFinder([temp_filepath], action)
self.assertEqual(len(results), 1)
ext_attrs = results[0].stat_entry.ext_attrs
self.assertFalse(ext_attrs)
def testSymlink(self):
with test_lib.AutoTempDirPath(remove_non_empty=True) as temp_dirpath,\
test_lib.AutoTempFilePath() as temp_filepath:
with open(temp_filepath, "wb") as fd:
fd.write("foobar")
temp_linkpath = os.path.join(temp_dirpath, "foo")
os.symlink(temp_filepath, temp_linkpath)
stat = utils.Stat(temp_linkpath, follow_symlink=False)
self.assertFalse(stat.IsDirectory())
self.assertFalse(stat.IsRegular())
self.assertFalse(stat.IsSocket())
self.assertTrue(stat.IsSymlink())
stat = utils.Stat(temp_linkpath, follow_symlink=True)
self.assertFalse(stat.IsDirectory())
self.assertTrue(stat.IsRegular())
self.assertFalse(stat.IsSocket())
self.assertFalse(stat.IsSymlink())
self.assertEqual(stat.GetSize(), 6)
def testStatExtAttrsEnabled(self):
with test_lib.AutoTempFilePath() as temp_filepath:
# TODO(hanuszczak): This call is repeated in many tests and should be
# refactored to some utility method in testing library.
if subprocess.call(["which", "setfattr"]) != 0:
raise unittest.SkipTest("`setfattr` command is not available")
if subprocess.call([
"setfattr", temp_filepath, "-n", "user.foo", "-v", "bar"
]) != 0:
raise unittest.SkipTest("extended attributes not supported")
pathspec = rdf_paths.PathSpec(
path=temp_filepath, pathtype=rdf_paths.PathSpec.PathType.OS)
request = rdf_client.GetFileStatRequest(pathspec=pathspec,
collect_ext_attrs=True)
results = self.RunAction(standard.GetFileStat, request)
self.assertEqual(len(results), 1)
self.assertEqual(len(results[0].ext_attrs), 1)
self.assertEqual(results[0].ext_attrs[0].name, "user.foo")
self.assertEqual(results[0].ext_attrs[0].value, "bar")