def test_bad_symlinks(tmpdir):
cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir,
log_directory=os.path.join(str(tmpdir), "logs"))
try:
d = CasBasedDirectory(cas_cache)
test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [("a", "D", ""), ("a/l", "S", "../target"),
("target", "F", "You got me")]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
exp_reason = "not-a-directory"
with pytest.raises(VirtualDirectoryError) as error:
d.descend("a", "l", follow_symlinks=True)
assert error.reason == exp_reason
with pytest.raises(VirtualDirectoryError) as error:
d.descend("a", "l")
assert error.reason == exp_reason
with pytest.raises(VirtualDirectoryError) as error:
d.descend("a", "f")
assert error.reason == exp_reason
finally:
cas_cache.release_resources()
def test_abs_symlink(tmpdir):
cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir,
log_directory=os.path.join(str(tmpdir), "logs"))
try:
d = CasBasedDirectory(cas_cache)
Content_to_check = "two step file"
test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [
("a", "D", ""),
("a/l", "S", "/target"),
("target", "D", ""),
("target/file", "F", Content_to_check),
]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
digest = d.descend("a", "l",
follow_symlinks=True).index["file"].get_digest()
with open(cas_cache.objpath(digest)) as fp:
content = fp.read()
assert Content_to_check == content
finally:
cas_cache.release_resources()
def __init__(self, directory, *, quota=None, casd=False, index_only=False):
# The working directory for the artifact share (in case it
# needs to do something outside of its backend's storage folder).
#
self.directory = os.path.abspath(directory)
# The directory the actual repo will be stored in.
#
# Unless this gets more complicated, just use this directly
# in tests as a remote artifact push/pull configuration
#
self.repodir = os.path.join(self.directory, "repo")
os.makedirs(self.repodir)
self.artifactdir = os.path.join(self.repodir, "artifacts", "refs")
os.makedirs(self.artifactdir)
self.sourcedir = os.path.join(self.repodir, "source_protos")
os.makedirs(self.sourcedir)
logdir = os.path.join(self.directory, "logs") if casd else None
self.cas = CASCache(self.repodir, casd=casd, log_directory=logdir)
self.quota = quota
self.index_only = index_only
super().__init__()
def _extract_subdirectory(self, tmpdir, digest):
with tempfile.TemporaryDirectory() as extractdir:
try:
cas = CASCache(str(tmpdir), casd=False)
cas.checkout(extractdir, digest)
yield extractdir
except FileNotFoundError:
yield None
def setup_backend(backend_class, tmpdir):
if backend_class == FileBasedDirectory:
path = os.path.join(tmpdir, "vdir")
os.mkdir(path)
yield backend_class(path)
else:
cas_cache = CASCache(os.path.join(tmpdir, "cas"),
log_directory=os.path.join(tmpdir, "logs"))
try:
yield backend_class(cas_cache)
finally:
cas_cache.release_resources()
def get_cache_usage(directory):
cas_cache = CASCache(directory, log_directory=os.path.dirname(directory))
try:
wait = 0.1
for _ in range(0, int(5 / wait)):
used_size = cas_cache.get_cache_usage().used_size
if used_size is not None:
return used_size
time.sleep(wait)
assert False, "Unable to retrieve cache usage"
return None
finally:
cas_cache.release_resources()
def _listing_test(tmpdir, root, generator_function):
cas_cache = CASCache(tmpdir, log_directory=os.path.join(tmpdir, "logs"))
try:
# Create some fake content
generator_function(root, tmpdir)
d = create_new_filedir(root, tmpdir)
filelist = list(d.list_relative_paths())
d2 = create_new_casdir(root, cas_cache, tmpdir)
filelist2 = list(d2.list_relative_paths())
assert filelist == filelist2
finally:
cas_cache.release_resources()
def test_descend(tmpdir):
cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir,
log_directory=os.path.join(str(tmpdir), "logs"))
try:
d = CasBasedDirectory(cas_cache)
Content_to_check = "You got me"
test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [("a", "D", ""), ("a/l", "D", ""),
("a/l/g", "F", Content_to_check)]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
digest = d.descend("a", "l").index["g"].get_digest()
with open(cas_cache.objpath(digest)) as fp:
content = fp.read()
assert Content_to_check == content
finally:
cas_cache.release_resources()
def test_bad_sym_escape(tmpdir):
cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir,
log_directory=os.path.join(str(tmpdir), "logs"))
try:
d = CasBasedDirectory(cas_cache)
test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [
("jail", "D", ""),
("jail/a", "D", ""),
("jail/a/l", "S", "../../target"),
("target", "D", ""),
("target/file", "F", "two step file"),
]
generate_import_root(test_dir, filesys_discription)
d.import_files(os.path.join(test_dir, "jail"))
with pytest.raises(VirtualDirectoryError) as error:
d.descend("a", "l", follow_symlinks=True)
assert error.reason == "directory-not-found"
finally:
cas_cache.release_resources()
def _import_test(tmpdir,
original,
overlay,
generator_function,
verify_contents=False):
# Skip this test if we do not have support for subsecond precision mtimes
#
if not have_subsecond_mtime(str(tmpdir)):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
str(tmpdir)))
cas_cache = CASCache(tmpdir, log_directory=os.path.join(tmpdir, "logs"))
try:
# Create some fake content
generator_function(original, tmpdir)
if original != overlay:
generator_function(overlay, tmpdir)
d = create_new_casdir(original, cas_cache, tmpdir)
duplicate_cas = create_new_casdir(original, cas_cache, tmpdir)
assert duplicate_cas._get_digest().hash == d._get_digest().hash
d2 = create_new_casdir(overlay, cas_cache, tmpdir)
d.import_files(d2, properties=["mtime"])
export_dir = os.path.join(tmpdir,
"output-{}-{}".format(original, overlay))
roundtrip_dir = os.path.join(
tmpdir, "roundtrip-{}-{}".format(original, overlay))
d2.export_files(roundtrip_dir)
d.export_files(export_dir)
if verify_contents:
for item in root_filesets[overlay - 1]:
(path, typename, content) = item
realpath = resolve_symlinks(path, export_dir)
if typename == "F":
if os.path.isdir(realpath) and directory_not_empty(
realpath):
# The file should not have overwritten the directory in this case.
pass
else:
assert os.path.isfile(
realpath
), "{} did not exist in the combined virtual directory".format(
path)
assert file_contents_are(realpath, content)
roundtrip = os.path.join(roundtrip_dir, path)
assert os.path.getmtime(roundtrip) == MTIME
assert os.path.getmtime(realpath) == MTIME
elif typename == "S":
if os.path.isdir(realpath) and directory_not_empty(
realpath):
# The symlink should not have overwritten the directory in this case.
pass
else:
assert os.path.islink(realpath)
assert os.readlink(realpath) == content
elif typename == "D":
# We can't do any more tests than this because it
# depends on things present in the original. Blank
# directories here will be ignored and the original
# left in place.
assert os.path.lexists(realpath)
# Now do the same thing with filebaseddirectories and check the contents match
duplicate_cas.import_files(roundtrip_dir, properties=["mtime"])
assert duplicate_cas._get_digest().hash == d._get_digest().hash
finally:
cas_cache.release_resources()