def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) assert cli.get_element_state(project, "target.bst") == "no reference"
def test_unique_key(cli, tmpdir, datafiles): """This test confirms that the 'filename' parameter is honoured when it comes to generating a cache key for the source. """ project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) states = cli.get_element_states( project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) assert states["target.bst"] == "fetch needed" assert states["target-custom.bst"] == "fetch needed" assert states["target-custom-executable.bst"] == "fetch needed" # Try to fetch it cli.run(project=project, args=["source", "fetch", "target.bst"]) # We should download_yaml the file only once states = cli.get_element_states( project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) assert states["target.bst"] == "buildable" assert states["target-custom.bst"] == "buildable" assert states["target-custom-executable.bst"] == "buildable" # But the cache key is different because the 'filename' is different. assert (cli.get_element_key(project, "target.bst") != cli.get_element_key( project, "target-custom.bst") != cli.get_element_key( project, "target-custom-executable.bst"))
def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.lz") _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target-lz.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target-lz.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target-lz.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target-lz.bst", "--directory", checkoutdir ]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents
def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir): file_server_files = os.path.join(str(tmpdir), "file_server") fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"login testuser\n") f.write(b"password 12345\n") with create_file_server(server_type) as server: server.add_user("otheruser", "12345", file_server_files) parts = urllib.parse.urlsplit(server.base_url()) base_url = urllib.parse.urlunsplit( [parts[0], "otheruser@{}".format(parts[1]), *parts[2:]]) generate_project(project, config={"aliases": {"tmpdir": base_url}}) src_tar = os.path.join(file_server_files, "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None)
def test_use_netrc(cli, datafiles, server_type, tmpdir): fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(fake_home, exist_ok=True) project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"login testuser\n") f.write(b"password 12345\n") with create_file_server(server_type) as server: server.add_user("testuser", "12345", project) generate_project(project, {"aliases": {"tmpdir": server.base_url()}}) server.start() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target.bst", "--directory", checkoutdir ]) result.assert_success() checkout_file = os.path.join(checkoutdir, "file") assert os.path.exists(checkout_file)
def test_custom_transform_source(cli, datafiles): project = str(datafiles) # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = load_yaml(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) generate_project(project, project_config) # Ensure we can track result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() # Ensure we can fetch result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() # Ensure we get correct output from foo_transform cli.run(project=project, args=["build", "target.bst"]) destpath = os.path.join(cli.directory, "checkout") result = cli.run( project=project, args=["artifact", "checkout", "target.bst", "--directory", destpath]) result.assert_success() # Assert that files from both sources exist, and that they have # the same content assert os.path.exists(os.path.join(destpath, "file")) assert os.path.exists(os.path.join(destpath, "filetransform")) with open(os.path.join(destpath, "file"), encoding="utf-8") as file1: with open(os.path.join(destpath, "filetransform"), encoding="utf-8") as file2: assert file1.read() == file2.read()
def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): try: project = str(datafiles) generate_project( project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) tar_file = "{}.tar.gz".format(tar_name) generate_element( project, "target.bst", { "kind": "import", "sources": [{ "kind": "tar", "url": "tmpdir:/{}".format(tar_file), "ref": "foo", "base-dir": base_dir }], }, ) # Get the tarball in tests/sources/tar/read-only/content # # NOTE that we need to do this because tarfile.open and tar.add() # are packing the tar up with writeable files and dirs tarball = os.path.join(str(datafiles), "content", tar_file) if not os.path.exists(tarball): raise FileNotFoundError("{} does not exist".format(tarball)) copyfile(tarball, os.path.join(str(tmpdir), tar_file)) # Because this test can potentially leave directories behind # which are difficult to remove, ask buildstream to use # our temp directory, so we can clean up. tmpdir_str = str(tmpdir) if not tmpdir_str.endswith(os.path.sep): tmpdir_str += os.path.sep env = {"TMP": tmpdir_str} # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"], env=env) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"], env=env) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"], env=env) result.assert_success() finally: utils._force_rmtree(str(tmpdir))
def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles): def ensure_link(member): # By default, python will simply duplicate files - we want # hardlinks! if member.path == "contents/to_extract/a": member.type = tarfile.LNKTYPE member.linkname = "contents/elsewhere/a" return member project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a tarball with an odd hardlink src_tar = os.path.join(str(tmpdir), "contents.tar.gz") old_dir = os.getcwd() os.chdir(str(tmpdir)) with tarfile.open(src_tar, "w:gz") as tar: # Don't recursively add `contents` as the order is not guaranteed. # We need to add `elsewhere` before `to_extract` as the latter # references the former in `linkname`. tar.add("contents", recursive=False) tar.add("contents/elsewhere") tar.add("contents/to_extract", filter=ensure_link) os.chdir(old_dir) # Make sure our tarfile is actually created with the desired # attributes set with tarfile.open(src_tar, "r:gz") as tar: assert any(member.islnk() and member.path == "contents/to_extract/a" and member.linkname == "contents/elsewhere/a" for member in tar.getmembers()) # Assert that we will actually create a singular copy of the file result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target.bst", "--directory", checkoutdir ]) result.assert_success() original_dir = os.path.join(str(datafiles), "contents", "to_extract") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents
def test_missing_file(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None)
def test_fetch_bad_url(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) assert "FAILURE Try #" in result.stderr result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None)
def test_path_in_filename(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) # The bst file has a / in the filename param result.assert_main_error(ErrorDomain.SOURCE, "filename-contains-directory")
def test_track_warning(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Track it result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() assert "Potential man-in-the-middle attack!" in result.stderr
def test_fetch_bad_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None)
def test_homeless_environment(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Use a track, make sure the plugin tries to find a ~/.netrc result = cli.run(project=project, args=["source", "track", "target.bst"], env={"HOME": None}) result.assert_success()
def test_use_netrc(cli, datafiles, server_type, tmpdir): file_server_files = os.path.join(str(tmpdir), "file_server") fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"login testuser\n") f.write(b"password 12345\n") with create_file_server(server_type) as server: server.add_user("testuser", "12345", file_server_files) generate_project(project, config={"aliases": { "tmpdir": server.base_url() }}) src_tar = os.path.join(file_server_files, "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target.bst", "--directory", checkoutdir ]) result.assert_success() original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents
def test_stage_contains_links(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") # Create a hardlink, we wont trust git to store that info for us os.makedirs(os.path.join(str(datafiles), "content", "base-directory", "subdir2"), exist_ok=True) file1 = os.path.join(str(datafiles), "content", "base-directory", "subdir1", "file.txt") file2 = os.path.join(str(datafiles), "content", "base-directory", "subdir2", "file.txt") os.link(file1, file2) _assemble_tar(os.path.join(str(datafiles), "content"), "base-directory", src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target.bst", "--directory", checkoutdir ]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') original_dir = os.path.join(str(datafiles), "content", "base-directory") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents
def test_track_with_comments(cli, datafiles): project = str(datafiles) generate_project(project, {"aliases": { "project-root": "file:///" + project }}) target = "comments.bst" # Assert that it needs to be tracked assert cli.get_element_state(project, target) == "no reference" # Track and fetch the sources result = cli.run(project=project, args=["source", "track", target]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", target]) result.assert_success() # Assert that the sources are cached assert cli.get_element_state(project, target) == "buildable"
def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) # Create a maliciously-hardlinked tarball def ensure_link(member): # By default, python will simply duplicate files - we want # hardlinks! if member.path == "contents/elsewhere/malicious": member.type = tarfile.LNKTYPE # This should not be allowed member.linkname = "../../../malicious_target.bst" return member src_tar = os.path.join(str(tmpdir), "contents.tar.gz") old_dir = os.getcwd() os.chdir(str(tmpdir)) with tarfile.open(src_tar, "w:gz") as tar: tar.add("contents", filter=ensure_link) os.chdir(old_dir) # Make sure our tarfile is actually created with the desired # attributes set with tarfile.open(src_tar, "r:gz") as tar: assert any( member.islnk() and member.path == "contents/elsewhere/malicious" and member.linkname == "../../../malicious_target.bst" for member in tar.getmembers()) # Try to execute the exploit result = cli.run(project=project, args=["source", "track", "malicious_target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "malicious_target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None)
def test_executable(cli, tmpdir, datafiles): """This test confirms that the 'ecxecutable' parameter is honoured.""" project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) checkoutdir = os.path.join(str(tmpdir), "checkout") assert cli.get_element_state( project, "target-custom-executable.bst") == "fetch needed" # Try to fetch it cli.run(project=project, args=["build", "target-custom-executable.bst"]) cli.run(project=project, args=[ "artifact", "checkout", "target-custom-executable.bst", "--directory", checkoutdir ]) mode = os.stat(os.path.join(checkoutdir, "some-custom-file")).st_mode assert mode & stat.S_IEXEC # Assert executable by anyone assert mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
def test_track_deps(cli, datafiles, deps, expected_states): project = str(datafiles) generate_project(project, {"aliases": { "project-root": "file:///" + project }}) target = "bananas.bst" build_dep = "apples.bst" runtime_dep = "oranges.bst" # Assert that none of the sources have a reference states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert all(state == "no reference" for state in states.values()) # Now track the specified sources result = cli.run(project=project, args=["source", "track", "--deps", deps, target]) result.assert_success() # Finally assert that we have tracked _only_ the desired sources states = cli.get_element_states(project, [target, build_dep, runtime_dep]) states_flattened = (states[target], states[build_dep], states[runtime_dep]) assert states_flattened == expected_states
def test_simple_file_build(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target.bst", "--directory", checkoutdir ]) result.assert_success() # Note that the url of the file in target.bst is actually /dir/file # but this tests confirms we take the basename checkout_file = os.path.join(checkoutdir, "file") assert os.path.exists(checkout_file) mode = os.stat(checkout_file).st_mode # Assert not executable by anyone assert not mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) # Assert not writeable by anyone other than me (unless umask allows it) if utils.get_umask() & stat.S_IWGRP: assert not mode & stat.S_IWGRP if utils.get_umask() & stat.S_IWOTH: assert not mode & stat.S_IWOTH
def test_simple_file_custom_name_build(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=[ "artifact", "checkout", "target.bst", "--directory", checkoutdir ]) result.assert_success() assert not os.path.exists(os.path.join(checkoutdir, "file")) assert os.path.exists(os.path.join(checkoutdir, "custom-file"))