def test_filter_pass_integration(datafiles, cli):
project = str(datafiles)
# Explicitly not passing integration commands should be fine
result = cli.run(project=project, args=["build", "no-pass-integration.bst"])
result.assert_success()
# Passing integration commands should build nicely
result = cli.run(project=project, args=["build", "pass-integration.bst"])
result.assert_success()
# Checking out elements which don't pass integration commands should still work
checkout_dir = os.path.join(project, "no-pass")
result = cli.run(
project=project,
args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "no-pass-integration.bst"],
)
result.assert_success()
# Checking out the artifact should fail if we run integration commands, as
# the staged artifacts don't have a shell
checkout_dir = os.path.join(project, "pass")
result = cli.run(
project=project,
args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "pass-integration.bst"],
)
result.assert_main_error(ErrorDomain.STREAM, "missing-command")
def test_push_missing_source_after_build(cli, tmpdir, datafiles):
cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_name = "import-bin.bst"
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
# Delete source but keep artifact in cache
shutil.rmtree(os.path.join(cache_dir, "elementsources"))
shutil.rmtree(os.path.join(cache_dir, "source_protos"))
with create_artifact_share(os.path.join(str(tmpdir),
"sourceshare")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
res = cli.run(project=project_dir,
args=["source", "push", element_name])
res.assert_success()
assert "fetch:{}".format(element_name) in res.stderr
assert "Pushed source" in res.stderr
def test_custom_transform_source(cli, datafiles):
project = str(datafiles)
# Set the project_dir alias in project.conf to the path to the tested project
project_config_path = os.path.join(project, "project.conf")
project_config = load_yaml(project_config_path)
aliases = project_config.get_mapping("aliases")
aliases["project_dir"] = "file://{}".format(project)
generate_project(project, project_config)
# Ensure we can track
result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
# Ensure we can fetch
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
# Ensure we get correct output from foo_transform
cli.run(project=project, args=["build", "target.bst"])
destpath = os.path.join(cli.directory, "checkout")
result = cli.run(
project=project,
args=["artifact", "checkout", "target.bst", "--directory", destpath])
result.assert_success()
# Assert that files from both sources exist, and that they have
# the same content
assert os.path.exists(os.path.join(destpath, "file"))
assert os.path.exists(os.path.join(destpath, "filetransform"))
with open(os.path.join(destpath, "file"), encoding="utf-8") as file1:
with open(os.path.join(destpath, "filetransform"),
encoding="utf-8") as file2:
assert file1.read() == file2.read()
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage,
workspaced):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
element = {
"kind": "stack",
"depends": [{
"junction": "junction.bst",
"filename": "import-etc.bst"
}]
}
_yaml.roundtrip_dump(element, element_path)
# Open a workspace if we're testing workspaced behavior
if workspaced:
result = cli.run(
project=project,
silent=True,
args=[
"workspace", "open", "--no-checkout", "--directory",
subproject_path, "junction.bst"
],
)
result.assert_success()
# Assert the correct error when trying to show the pipeline
dep_result = cli.run(project=project,
silent=True,
args=["show", "junction-dep.bst"])
# Assert the correct error when trying to show the pipeline
etc_result = cli.run(project=project,
silent=True,
args=["show", "junction.bst:import-etc.bst"])
# If a workspace is open, no ref is needed
if workspaced:
dep_result.assert_success()
etc_result.assert_success()
else:
# Assert that we have the expected provenance encoded into the error
element_node = _yaml.load(element_path, shortname="junction-dep.bst")
ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in dep_result.stderr
dep_result.assert_main_error(ErrorDomain.LOAD,
LoadErrorReason.SUBPROJECT_INCONSISTENT)
etc_result.assert_main_error(ErrorDomain.LOAD,
LoadErrorReason.SUBPROJECT_INCONSISTENT)
def test_build_push_source_twice(cli, tmpdir, datafiles):
cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_name = "import-bin.bst"
with create_artifact_share(os.path.join(str(tmpdir),
"sourceshare")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
assert "fetch:{}".format(element_name) in res.stderr
assert "Pushed source" in res.stderr
# The second build pipeline is a no-op as everything is already cached.
# However, this verifies that the pipeline behaves as expected.
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
assert "fetch:{}".format(element_name) not in res.stderr
assert "Pushed source" not in res.stderr
def test_artifact_delete_artifact(cli, tmpdir, datafiles, with_project):
project = str(datafiles)
element = "target.bst"
# Configure a local cache
local_cache = os.path.join(str(tmpdir), "cache")
cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
result = cli.run(project=project, args=["build", element])
result.assert_success()
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the project.conf if we're going to try this without a project
if not with_project:
os.remove(os.path.join(project, "project.conf"))
# Delete the artifact
result = cli.run(project=project, args=["artifact", "delete", artifact])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
assert not os.path.exists(
os.path.join(local_cache, "cas", "refs", "heads", artifact))
def test_filter_pass_integration_uncached(datafiles, cli):
project = str(datafiles)
# Passing integration commands should build nicely
result = cli.run(project=project, args=["build", "filter/filter.bst"])
result.assert_success()
# Delete the build dependency of the filter element.
# The built filter element should be usable even if the build dependency
# is not available in the local cache.
result = cli.run(project=project,
args=["artifact", "delete", "filter/parent.bst"])
result.assert_success()
# Checking out the element should work
checkout_dir = os.path.join(project, "filter")
result = cli.run(
project=project,
args=[
"artifact", "checkout", "--integrate", "--directory", checkout_dir,
"filter/filter.bst"
],
)
result.assert_success()
# Check that the integration command was run
assert_contains(checkout_dir, ["/foo"])
shutil.rmtree(checkout_dir)
def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create a local tar
src_tar = os.path.join(str(tmpdir), "a.tar.lz")
_assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
# Track, fetch, build, checkout
result = cli.run(project=project,
args=["source", "track", "target-lz.bst"])
result.assert_success()
result = cli.run(project=project,
args=["source", "fetch", "target-lz.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target-lz.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target-lz.bst",
"--directory", checkoutdir
])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
original_dir = os.path.join(str(datafiles), "content", "a")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
def test_rebuild(datafiles, cli, strict):
project = str(datafiles)
# First build intermediate target.bst
result = cli.run(project=project,
args=strict_args(["build", "target.bst"], strict))
result.assert_success()
# Modify base import
with open(os.path.join(project, "files", "dev-files", "usr", "include",
"new.h"),
"w",
encoding="utf-8") as f:
f.write("#define NEW")
# Rebuild base import and build top-level rebuild-target.bst
# In non-strict mode, this does not rebuild intermediate target.bst,
# which means that a weakly cached target.bst will be staged as dependency.
result = cli.run(project=project,
args=strict_args(["build", "rebuild-target.bst"], strict))
result.assert_success()
built_elements = result.get_built_elements()
assert "rebuild-target.bst" in built_elements
if strict == "strict":
assert "target.bst" in built_elements
else:
assert "target.bst" not in built_elements
def test_stage_symlink(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Workaround datafiles bug:
#
# https://github.com/omarkohl/pytest-datafiles/issues/1
#
# Create the symlink by hand.
symlink = os.path.join(project, "files", "symlink-to-file.txt")
os.symlink("file.txt", symlink)
# Build, checkout
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst", "--directory",
checkoutdir
])
result.assert_success()
# Check that the checkout contains the expected file and directory and other file
assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
assert os.path.exists(os.path.join(checkoutdir, "symlink-to-file.txt"))
assert os.path.islink(os.path.join(checkoutdir, "symlink-to-file.txt"))
def test_build_checkout_nonempty(datafiles, cli, hardlinks):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
filename = os.path.join(checkout, "file.txt")
# First build it
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Assert that after a successful build, the builddir is empty
builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
# Create the checkout dir and add a file to it, should cause checkout to fail
os.makedirs(checkout, exist_ok=True)
with open(filename, "w", encoding="utf-8") as f:
f.write("Hello")
# Prepare checkout args
checkout_args = ["artifact", "checkout"]
if hardlinks == "hardlinks":
checkout_args += ["--hardlinks"]
checkout_args += ["target.bst", "--directory", checkout]
# Now check it out
result = cli.run(project=project, args=checkout_args)
result.assert_main_error(ErrorDomain.STREAM, None)
def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
# Set up remote and local shares
local_cache = os.path.join(str(tmpdir), "artifacts")
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote:
cli.configure({
"artifacts": {
"servers": [{
"url": remote.repo,
"push": True
}]
},
"cachedir": local_cache,
})
# Build the element
result = cli.run(project=project, args=["build", element])
result.assert_success()
# Make sure it's in the share
assert remote.get_artifact(
cli.get_artifact_name(project, "test", element))
# Delete the artifact from the local cache
result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
assert cli.get_element_state(project, element) != "cached"
result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
assert "available {}".format(element) in result.output
def test_build_checkout_tarball_is_deterministic(datafiles, cli):
project = str(datafiles)
tarball1 = os.path.join(cli.directory, "tarball1.tar")
tarball2 = os.path.join(cli.directory, "tarball2.tar")
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
checkout_args = ["artifact", "checkout", "--force", "target.bst"]
checkout_args1 = checkout_args + ["--tar", tarball1]
result = cli.run(project=project, args=checkout_args1)
result.assert_success()
checkout_args2 = checkout_args + ["--tar", tarball2]
result = cli.run(project=project, args=checkout_args2)
result.assert_success()
with open(tarball1, "rb") as f:
contents = f.read()
hash1 = hashlib.sha1(contents).hexdigest()
with open(tarball2, "rb") as f:
contents = f.read()
hash2 = hashlib.sha1(contents).hexdigest()
assert hash1 == hash2
def test_build_checkout_links(datafiles, cli):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
# Create the link before running the tests.
# This is needed for users working on Windows, git checks out symlinks as files which content is the name
# of the symlink and the test therefore doesn't have the correct content
os.symlink(
os.path.join("..", "basicfile"),
os.path.join(project, "files", "files-and-links", "basicfolder",
"basicsymlink"),
)
result = cli.run(project=project, args=["build", "import-links.bst"])
result.assert_success()
builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
checkout_args = [
"artifact", "checkout", "--directory", checkout, "import-links.bst"
]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
with open(os.path.join(checkout, "basicfolder", "basicsymlink"),
encoding="utf-8") as fp:
data = fp.read()
assert data == "file contents\n"
def test_build_checkout_tarball(datafiles, cli):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout.tar")
# Work-around datafiles not preserving mode
os.chmod(os.path.join(project, "files/bin-files/usr/bin/hello"), 0o0755)
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
with tarfile.TarFile(checkout) as tar:
tarinfo = tar.getmember(os.path.join(".", "usr", "bin", "hello"))
assert tarinfo.mode == 0o755
assert tarinfo.uid == 0 and tarinfo.gid == 0
assert tarinfo.uname == "" and tarinfo.gname == ""
tarinfo = tar.getmember(os.path.join(".", "usr", "include", "pony.h"))
assert tarinfo.mode == 0o644
assert tarinfo.uid == 0 and tarinfo.gid == 0
assert tarinfo.uname == "" and tarinfo.gname == ""
def test_build_checkout(datafiles, cli, strict, hardlinks):
if CASD_SEPARATE_USER and hardlinks == "hardlinks":
pytest.xfail(
"Cannot hardlink with buildbox-casd running as a separate user")
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
# First build it
result = cli.run(project=project,
args=strict_args(["build", "target.bst"], strict))
result.assert_success()
# Assert that after a successful build, the builddir is empty
builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
# Prepare checkout args
checkout_args = strict_args(["artifact", "checkout"], strict)
if hardlinks == "hardlinks":
checkout_args += ["--hardlinks"]
checkout_args += ["target.bst", "--directory", checkout]
# Now check it out
result = cli.run(project=project, args=checkout_args)
result.assert_success()
# Check that the executable hello file is found in the checkout
filename = os.path.join(checkout, "usr", "bin", "hello")
assert os.path.exists(filename)
filename = os.path.join(checkout, "usr", "include", "pony.h")
assert os.path.exists(filename)
def test_unique_key(cli, tmpdir, datafiles):
"""This test confirms that the 'filename' parameter is honoured when it comes
to generating a cache key for the source.
"""
project = str(datafiles)
generate_project(project,
{"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
states = cli.get_element_states(
project,
["target.bst", "target-custom.bst", "target-custom-executable.bst"])
assert states["target.bst"] == "fetch needed"
assert states["target-custom.bst"] == "fetch needed"
assert states["target-custom-executable.bst"] == "fetch needed"
# Try to fetch it
cli.run(project=project, args=["source", "fetch", "target.bst"])
# We should download_yaml the file only once
states = cli.get_element_states(
project,
["target.bst", "target-custom.bst", "target-custom-executable.bst"])
assert states["target.bst"] == "buildable"
assert states["target-custom.bst"] == "buildable"
assert states["target-custom-executable.bst"] == "buildable"
# But the cache key is different because the 'filename' is different.
assert (cli.get_element_key(project, "target.bst") != cli.get_element_key(
project, "target-custom.bst") != cli.get_element_key(
project, "target-custom-executable.bst"))
def test_use_netrc(cli, datafiles, server_type, tmpdir):
fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
os.environ["HOME"] = fake_home
with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
f.write(b"machine 127.0.0.1\n")
f.write(b"login testuser\n")
f.write(b"password 12345\n")
with create_file_server(server_type) as server:
server.add_user("testuser", "12345", project)
generate_project(project, {"aliases": {"tmpdir": server.base_url()}})
server.start()
result = cli.run(project=project,
args=["source", "fetch", "target.bst"])
result.assert_success()
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", "target.bst",
"--directory", checkoutdir
])
result.assert_success()
checkout_file = os.path.join(checkoutdir, "file")
assert os.path.exists(checkout_file)
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
repo = create_repo("tar", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "pull-missing-local-blob",
"min-version": "2.0",
"element-path": "elements",
}
project_file = os.path.join(str(tmpdir), "project.conf")
_yaml.roundtrip_dump(project_config, project_file)
input_config = {
"kind": "import",
"sources": [repo.source_config()],
}
input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
depends_name = "depends.bst"
depends_config = {"kind": "stack", "depends": [input_name]}
depends_file = os.path.join(element_dir, depends_name)
_yaml.roundtrip_dump(depends_config, depends_file)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the import-bin element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
result = cli.run(project=project, args=["build", input_name])
result.assert_success()
assert cli.get_element_state(project, input_name) == "cached"
# Delete a file blob from the local cache.
# This is a placeholder to test partial CAS handling until we support
# partial artifact pulling (or blob-based CAS expiry).
#
digest = utils.sha256sum(
os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
objpath = os.path.join(cli.directory, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# Now try bst build
result = cli.run(project=project, args=["build", depends_name])
result.assert_success()
# Assert that the import-bin artifact was pulled (completing the partial artifact)
assert result.get_pulled_elements() == [input_name]
def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace,
guess_element):
tmpdir = tmpdir_factory.mktemp(os.path.basename(__file__))
project = str(datafiles)
checkout = os.path.join(cli.directory, "source-checkout")
target = "checkout-deps.bst"
workspace = os.path.join(str(tmpdir), "workspace")
elm_cmd = [target] if not guess_element else []
if with_workspace:
ws_cmd = ["-C", workspace]
result = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, target])
result.assert_success()
else:
ws_cmd = []
args = ws_cmd + [
"source", "checkout", "--deps", "none", "--directory", checkout,
*elm_cmd
]
result = cli.run(project=project, args=args)
result.assert_success()
assert os.path.exists(
os.path.join(checkout, "checkout-deps", "etc", "buildstream",
"config"))
def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
dep = "compose-all.bst"
# Configure a local cache
local_cache = os.path.join(str(tmpdir), "cache")
cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
result = cli.run(project=project, args=["build", element])
result.assert_success()
assert cli.get_element_states(project, [element, dep], deps="none") == {
element: "cached",
dep: "cached",
}
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the artifact
result = cli.run(project=project,
args=["artifact", "delete", artifact, dep])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact))
# Check that the dependency ELEMENT is no longer cached
assert cli.get_element_state(project, dep) != "cached"
def test_nested_junction_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in
# junctions of junctions
project = str(datafiles)
sub1_path = os.path.join(project, "files", "sub-project")
sub2_path = os.path.join(project, "files", "sub2-project")
# A junction element which pulls sub1 into sub2
sub1_element = os.path.join(project, "files", "sub2-project", "elements", "sub-junction.bst")
# A junction element which pulls sub2 into the main project
sub2_element = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
generate_junction(tmpdir / "sub-project", sub1_path, sub1_element, store_ref=True)
generate_junction(tmpdir / "sub2-project", sub2_path, sub2_element, store_ref=True)
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-sub.bst"}]}
_yaml.roundtrip_dump(element, element_path)
result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"])
result.assert_success()
# Assert the correct progress tallies are in the logging
result = cli.run(project=project, args=["show", "junction-dep.bst"])
assert " 3 subtasks processed" in result.stderr
assert "3 of 3 subtasks processed" in result.stderr
def test_artifact_list_exact_contents_long(cli, datafiles, target):
project = str(datafiles)
# Ensure we have an artifact to read
result = cli.run(project=project, args=["build", "import-bin.bst"])
assert result.exit_code == 0
if target == "element-name":
arg = "import-bin.bst"
elif target == "artifact-name":
key = cli.get_element_key(project, "import-bin.bst")
arg = "test/import-bin/" + key
# List the contents via the element name
result = cli.run(project=project, args=["artifact", "list-contents", "--long", arg])
assert result.exit_code == 0
expected_output_template = (
"{target}:\n"
"\tdrwxr-xr-x dir 0 usr\n"
"\tdrwxr-xr-x dir 0 usr/bin\n"
"\t-rw-r--r-- reg 28 usr/bin/hello\n\n"
)
expected_output = expected_output_template.format(target=arg)
assert expected_output in result.output
def test_artifact_list_exact_contents(cli, datafiles, target, with_project):
project = str(datafiles)
# Get the cache key of our test element
key = cli.get_element_key(project, "import-bin.bst")
# Ensure we have an artifact to read
result = cli.run(project=project, args=["build", "import-bin.bst"])
result.assert_success()
if target == "element-name":
arg = "import-bin.bst"
elif target == "artifact-name":
key = cli.get_element_key(project, "import-bin.bst")
arg = "test/import-bin/" + key
# Delete the project.conf if we're going to try this without a project
if not with_project:
os.remove(os.path.join(project, "project.conf"))
# List the contents via the key
result = cli.run(project=project, args=["artifact", "list-contents", arg])
# Expect to fail if we try to list by element name and there is no project
if target == "element-name" and not with_project:
result.assert_main_error(ErrorDomain.STREAM, "project-not-loaded")
else:
result.assert_success()
expected_output_template = "{target}:\n\tusr\n\tusr/bin\n\tusr/bin/hello\n\n"
expected_output = expected_output_template.format(target=arg)
assert expected_output in result.output
def test_artifact_list_exact_contents_glob(cli, datafiles):
project = str(datafiles)
# Ensure we have an artifact to read
result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
# List the contents via glob
result = cli.run(project=project, args=["artifact", "list-contents", "test/**"])
assert result.exit_code == 0
# get the cahe keys for each element in the glob
import_bin_key = cli.get_element_key(project, "import-bin.bst")
import_dev_key = cli.get_element_key(project, "import-dev.bst")
compose_all_key = cli.get_element_key(project, "compose-all.bst")
target_key = cli.get_element_key(project, "target.bst")
expected_artifacts = [
"test/import-bin/" + import_bin_key,
"test/import-dev/" + import_dev_key,
"test/compose-all/" + compose_all_key,
"test/target/" + target_key,
]
for artifact in expected_artifacts:
assert artifact in result.output
def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
project_dir = str(datafiles)
element_name, repo, ref = create_test_element(tmpdir, project_dir)
cache_dir = os.path.join(str(tmpdir), "cache")
# use artifact cache for sources for now, they should work the same
with create_artifact_share(os.path.join(str(tmpdir),
"sourceshare")) as share:
with context_with_source_cache(cli, cache_dir, share,
tmpdir) as context:
project = Project(project_dir, context)
project.ensure_fully_loaded()
element = project.load_elements([element_name])[0]
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
assert not share.get_artifact_proto(source._get_source_name())
# Just check that we sensibly fetch and build the element
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
assert os.listdir(
os.path.join(str(tmpdir), "cache", "sources", "tar")) != []
# get root digest of source
sourcecache = context.sourcecache
digest = sourcecache.export(source)._get_digest()
# Push the source to the remote
res = cli.run(project=project_dir,
args=[
"source", "push", "--source-remote", share.repo,
element_name
])
res.assert_success()
# Remove the cas content, only keep the proto and such around
shutil.rmtree(
os.path.join(str(tmpdir), "sourceshare", "repo", "cas",
"objects"))
# check the share doesn't have the object
assert not share.has_object(digest)
# Delete the source locally
shutil.rmtree(os.path.join(str(cache_dir), "sources"))
shutil.rmtree(os.path.join(str(cache_dir), "cas"))
state = cli.get_element_state(project_dir, element_name)
assert state == "fetch needed"
# Now fetch the source and check
res = cli.run(project=project_dir,
args=["source", "fetch", element_name])
res.assert_success()
assert ("SUCCESS Fetching {}".format(
repo.source_config(ref=ref)["url"])) in res.stderr
def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
try:
project = str(datafiles)
generate_project(
project, config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
tar_file = "{}.tar.gz".format(tar_name)
generate_element(
project,
"target.bst",
{
"kind":
"import",
"sources": [{
"kind": "tar",
"url": "tmpdir:/{}".format(tar_file),
"ref": "foo",
"base-dir": base_dir
}],
},
)
# Get the tarball in tests/sources/tar/read-only/content
#
# NOTE that we need to do this because tarfile.open and tar.add()
# are packing the tar up with writeable files and dirs
tarball = os.path.join(str(datafiles), "content", tar_file)
if not os.path.exists(tarball):
raise FileNotFoundError("{} does not exist".format(tarball))
copyfile(tarball, os.path.join(str(tmpdir), tar_file))
# Because this test can potentially leave directories behind
# which are difficult to remove, ask buildstream to use
# our temp directory, so we can clean up.
tmpdir_str = str(tmpdir)
if not tmpdir_str.endswith(os.path.sep):
tmpdir_str += os.path.sep
env = {"TMP": tmpdir_str}
# Track, fetch, build, checkout
result = cli.run(project=project,
args=["source", "track", "target.bst"],
env=env)
result.assert_success()
result = cli.run(project=project,
args=["source", "fetch", "target.bst"],
env=env)
result.assert_success()
result = cli.run(project=project,
args=["build", "target.bst"],
env=env)
result.assert_success()
finally:
utils._force_rmtree(str(tmpdir))
def test_push_pull_specific_remote(cli, tmpdir, datafiles):
project = str(datafiles)
with create_artifact_share(os.path.join(
str(tmpdir),
"goodartifactshare")) as good_share, create_artifact_share(
os.path.join(str(tmpdir), "badartifactshare")) as bad_share:
# Build the target so we have it cached locally only.
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
state = cli.get_element_state(project, "target.bst")
assert state == "cached"
# Configure the default push location to be bad_share; we will assert that
# nothing actually gets pushed there.
cli.configure({
"artifacts": {
"servers": [
{
"url": bad_share.repo,
"push": True
},
]
}
})
# Now try `bst artifact push` to the good_share.
result = cli.run(project=project,
args=[
"artifact", "push", "target.bst",
"--artifact-remote", good_share.repo
])
result.assert_success()
# Assert that all the artifacts are in the share we pushed
# to, and not the other.
assert_shared(cli, good_share, project, "target.bst")
assert_not_shared(cli, bad_share, project, "target.bst")
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the good_share.
#
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
result = cli.run(project=project,
args=[
"artifact", "pull", "target.bst",
"--artifact-remote", good_share.repo
])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert cli.get_element_state(project, "target.bst") == "cached"
def test_filter_orphans(datafiles, cli, tmpdir):
project = str(datafiles)
result = cli.run(project=project, args=["build", "output-orphans.bst"])
result.assert_success()
checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout")
result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, "baz"))
def test_filter_deps_ok(datafiles, cli):
project = str(datafiles)
result = cli.run(project=project, args=["build", "deps-permitted.bst"])
result.assert_success()
result = cli.run(project=project, args=["show", "--deps=run", "--format='%{name}'", "deps-permitted.bst"])
result.assert_success()
assert "output-exclude.bst" in result.output
assert "output-orphans.bst" in result.output
