def test_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
assert cli.get_element_state(project, "target.bst") == "no reference"
def test_unique_key(cli, tmpdir, datafiles):
"""This test confirms that the 'filename' parameter is honoured when it comes
to generating a cache key for the source.
"""
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
states = cli.get_element_states(
project,
["target.bst", "target-custom.bst", "target-custom-executable.bst"])
assert states["target.bst"] == "fetch needed"
assert states["target-custom.bst"] == "fetch needed"
assert states["target-custom-executable.bst"] == "fetch needed"
# Try to fetch it
cli.run(project=project, args=["source", "fetch", "target.bst"])
# We should download_yaml the file only once
states = cli.get_element_states(
project,
["target.bst", "target-custom.bst", "target-custom-executable.bst"])
assert states["target.bst"] == "buildable"
assert states["target-custom.bst"] == "buildable"
assert states["target-custom-executable.bst"] == "buildable"
# But the cache key is different because the 'filename' is different.
assert (cli.get_element_key(project, "target.bst") != cli.get_element_key(
project, "target-custom.bst") != cli.get_element_key(
project, "target-custom-executable.bst"))
def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create a local tar
src_tar = os.path.join(str(tmpdir), "a.tar.lz")
_assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
# Track, fetch, build, checkout
result = cli.run(project=project,
args=["source", "track", "target-lz.bst"])
result.assert_success()
result = cli.run(project=project,
args=["source", "fetch", "target-lz.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target-lz.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target-lz.bst",
"--directory", checkoutdir
])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
original_dir = os.path.join(str(datafiles), "content", "a")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
file_server_files = os.path.join(str(tmpdir), "file_server")
fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(file_server_files, exist_ok=True)
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
os.environ["HOME"] = fake_home
with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
f.write(b"machine 127.0.0.1\n")
f.write(b"login testuser\n")
f.write(b"password 12345\n")
with create_file_server(server_type) as server:
server.add_user("otheruser", "12345", file_server_files)
parts = urllib.parse.urlsplit(server.base_url())
base_url = urllib.parse.urlunsplit(
[parts[0], "otheruser@{}".format(parts[1]), *parts[2:]])
generate_project(project, config={"aliases": {"tmpdir": base_url}})
src_tar = os.path.join(file_server_files, "a.tar.gz")
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
server.start()
result = cli.run(project=project,
args=["source", "track", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
def test_use_netrc(cli, datafiles, server_type, tmpdir):
fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
os.environ["HOME"] = fake_home
with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
f.write(b"machine 127.0.0.1\n")
f.write(b"login testuser\n")
f.write(b"password 12345\n")
with create_file_server(server_type) as server:
server.add_user("testuser", "12345", project)
generate_project(project, {"aliases": {"tmpdir": server.base_url()}})
server.start()
result = cli.run(project=project,
args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst",
"--directory", checkoutdir
])
result.assert_success()
checkout_file = os.path.join(checkoutdir, "file")
assert os.path.exists(checkout_file)
def test_custom_transform_source(cli, datafiles):
project = str(datafiles)
# Set the project_dir alias in project.conf to the path to the tested project
project_config_path = os.path.join(project, "project.conf")
project_config = load_yaml(project_config_path)
aliases = project_config.get_mapping("aliases")
aliases["project_dir"] = "file://{}".format(project)
generate_project(project, project_config)
# Ensure we can track
result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
# Ensure we can fetch
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
# Ensure we get correct output from foo_transform
cli.run(project=project, args=["build", "target.bst"])
destpath = os.path.join(cli.directory, "checkout")
result = cli.run(
project=project,
args=["artifact", "checkout", "target.bst", "--directory", destpath])
result.assert_success()
# Assert that files from both sources exist, and that they have
# the same content
assert os.path.exists(os.path.join(destpath, "file"))
assert os.path.exists(os.path.join(destpath, "filetransform"))
with open(os.path.join(destpath, "file"), encoding="utf-8") as file1:
with open(os.path.join(destpath, "filetransform"),
encoding="utf-8") as file2:
assert file1.read() == file2.read()
def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
try:
project = str(datafiles)
generate_project(
project, config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
tar_file = "{}.tar.gz".format(tar_name)
generate_element(
project,
"target.bst",
{
"kind":
"import",
"sources": [{
"kind": "tar",
"url": "tmpdir:/{}".format(tar_file),
"ref": "foo",
"base-dir": base_dir
}],
},
)
# Get the tarball in tests/sources/tar/read-only/content
#
# NOTE that we need to do this because tarfile.open and tar.add()
# are packing the tar up with writeable files and dirs
tarball = os.path.join(str(datafiles), "content", tar_file)
if not os.path.exists(tarball):
raise FileNotFoundError("{} does not exist".format(tarball))
copyfile(tarball, os.path.join(str(tmpdir), tar_file))
# Because this test can potentially leave directories behind
# which are difficult to remove, ask buildstream to use
# our temp directory, so we can clean up.
tmpdir_str = str(tmpdir)
if not tmpdir_str.endswith(os.path.sep):
tmpdir_str += os.path.sep
env = {"TMP": tmpdir_str}
# Track, fetch, build, checkout
result = cli.run(project=project,
args=["source", "track", "target.bst"],
env=env)
result.assert_success()
result = cli.run(project=project,
args=["source", "fetch", "target.bst"],
env=env)
result.assert_success()
result = cli.run(project=project,
args=["build", "target.bst"],
env=env)
result.assert_success()
finally:
utils._force_rmtree(str(tmpdir))
def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
def ensure_link(member):
# By default, python will simply duplicate files - we want
# hardlinks!
if member.path == "contents/to_extract/a":
member.type = tarfile.LNKTYPE
member.linkname = "contents/elsewhere/a"
return member
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create a tarball with an odd hardlink
src_tar = os.path.join(str(tmpdir), "contents.tar.gz")
old_dir = os.getcwd()
os.chdir(str(tmpdir))
with tarfile.open(src_tar, "w:gz") as tar:
# Don't recursively add `contents` as the order is not guaranteed.
# We need to add `elsewhere` before `to_extract` as the latter
# references the former in `linkname`.
tar.add("contents", recursive=False)
tar.add("contents/elsewhere")
tar.add("contents/to_extract", filter=ensure_link)
os.chdir(old_dir)
# Make sure our tarfile is actually created with the desired
# attributes set
with tarfile.open(src_tar, "r:gz") as tar:
assert any(member.islnk() and member.path == "contents/to_extract/a"
and member.linkname == "contents/elsewhere/a"
for member in tar.getmembers())
# Assert that we will actually create a singular copy of the file
result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst", "--directory",
checkoutdir
])
result.assert_success()
original_dir = os.path.join(str(datafiles), "contents", "to_extract")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
def test_missing_file(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
def test_fetch_bad_url(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
assert "FAILURE Try #" in result.stderr
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
def test_path_in_filename(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
# The bst file has a / in the filename param
result.assert_main_error(ErrorDomain.SOURCE, "filename-contains-directory")
def test_track_warning(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Create a local tar
src_tar = os.path.join(str(tmpdir), "a.tar.gz")
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
# Track it
result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
assert "Potential man-in-the-middle attack!" in result.stderr
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Create a local tar
src_tar = os.path.join(str(tmpdir), "a.tar.gz")
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
# Try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
def test_homeless_environment(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Create a local tar
src_tar = os.path.join(str(tmpdir), "a.tar.gz")
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
# Use a track, make sure the plugin tries to find a ~/.netrc
result = cli.run(project=project,
args=["source", "track", "target.bst"],
env={"HOME": None})
result.assert_success()
def test_use_netrc(cli, datafiles, server_type, tmpdir):
file_server_files = os.path.join(str(tmpdir), "file_server")
fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(file_server_files, exist_ok=True)
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
os.environ["HOME"] = fake_home
with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
f.write(b"machine 127.0.0.1\n")
f.write(b"login testuser\n")
f.write(b"password 12345\n")
with create_file_server(server_type) as server:
server.add_user("testuser", "12345", file_server_files)
generate_project(project,
config={"aliases": {
"tmpdir": server.base_url()
}})
src_tar = os.path.join(file_server_files, "a.tar.gz")
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
server.start()
result = cli.run(project=project,
args=["source", "track", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst",
"--directory", checkoutdir
])
result.assert_success()
original_dir = os.path.join(str(datafiles), "content", "a")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
def test_stage_contains_links(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create a local tar
src_tar = os.path.join(str(tmpdir), "a.tar.gz")
# Create a hardlink, we wont trust git to store that info for us
os.makedirs(os.path.join(str(datafiles), "content", "base-directory",
"subdir2"),
exist_ok=True)
file1 = os.path.join(str(datafiles), "content", "base-directory",
"subdir1", "file.txt")
file2 = os.path.join(str(datafiles), "content", "base-directory",
"subdir2", "file.txt")
os.link(file1, file2)
_assemble_tar(os.path.join(str(datafiles), "content"), "base-directory",
src_tar)
# Track, fetch, build, checkout
result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst", "--directory",
checkoutdir
])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
original_dir = os.path.join(str(datafiles), "content", "base-directory")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
def test_track_with_comments(cli, datafiles):
project = str(datafiles)
generate_project(project,
{"aliases": {
"project-root": "file:///" + project
}})
target = "comments.bst"
# Assert that it needs to be tracked
assert cli.get_element_state(project, target) == "no reference"
# Track and fetch the sources
result = cli.run(project=project, args=["source", "track", target])
result.assert_success()
result = cli.run(project=project, args=["source", "fetch", target])
result.assert_success()
# Assert that the sources are cached
assert cli.get_element_state(project, target) == "buildable"
def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
# Create a maliciously-hardlinked tarball
def ensure_link(member):
# By default, python will simply duplicate files - we want
# hardlinks!
if member.path == "contents/elsewhere/malicious":
member.type = tarfile.LNKTYPE
# This should not be allowed
member.linkname = "../../../malicious_target.bst"
return member
src_tar = os.path.join(str(tmpdir), "contents.tar.gz")
old_dir = os.getcwd()
os.chdir(str(tmpdir))
with tarfile.open(src_tar, "w:gz") as tar:
tar.add("contents", filter=ensure_link)
os.chdir(old_dir)
# Make sure our tarfile is actually created with the desired
# attributes set
with tarfile.open(src_tar, "r:gz") as tar:
assert any(
member.islnk() and member.path == "contents/elsewhere/malicious"
and member.linkname == "../../../malicious_target.bst"
for member in tar.getmembers())
# Try to execute the exploit
result = cli.run(project=project,
args=["source", "track", "malicious_target.bst"])
result.assert_success()
result = cli.run(project=project,
args=["source", "fetch", "malicious_target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
def test_executable(cli, tmpdir, datafiles):
"""This test confirms that the 'ecxecutable' parameter is honoured."""
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
assert cli.get_element_state(
project, "target-custom-executable.bst") == "fetch needed"
# Try to fetch it
cli.run(project=project, args=["build", "target-custom-executable.bst"])
cli.run(project=project,
args=[
"artifact", "checkout", "target-custom-executable.bst",
"--directory", checkoutdir
])
mode = os.stat(os.path.join(checkoutdir, "some-custom-file")).st_mode
assert mode & stat.S_IEXEC
# Assert executable by anyone
assert mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
def test_track_deps(cli, datafiles, deps, expected_states):
project = str(datafiles)
generate_project(project,
{"aliases": {
"project-root": "file:///" + project
}})
target = "bananas.bst"
build_dep = "apples.bst"
runtime_dep = "oranges.bst"
# Assert that none of the sources have a reference
states = cli.get_element_states(project, [target, build_dep, runtime_dep])
assert all(state == "no reference" for state in states.values())
# Now track the specified sources
result = cli.run(project=project,
args=["source", "track", "--deps", deps, target])
result.assert_success()
# Finally assert that we have tracked _only_ the desired sources
states = cli.get_element_states(project, [target, build_dep, runtime_dep])
states_flattened = (states[target], states[build_dep], states[runtime_dep])
assert states_flattened == expected_states
def test_simple_file_build(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst", "--directory",
checkoutdir
])
result.assert_success()
# Note that the url of the file in target.bst is actually /dir/file
# but this tests confirms we take the basename
checkout_file = os.path.join(checkoutdir, "file")
assert os.path.exists(checkout_file)
mode = os.stat(checkout_file).st_mode
# Assert not executable by anyone
assert not mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
# Assert not writeable by anyone other than me (unless umask allows it)
if utils.get_umask() & stat.S_IWGRP:
assert not mode & stat.S_IWGRP
if utils.get_umask() & stat.S_IWOTH:
assert not mode & stat.S_IWOTH
def test_simple_file_custom_name_build(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst", "--directory",
checkoutdir
])
result.assert_success()
assert not os.path.exists(os.path.join(checkoutdir, "file"))
assert os.path.exists(os.path.join(checkoutdir, "custom-file"))