def test_build_depend_on_cached_fail(cli, datafiles):
project = str(datafiles)
dep_path = os.path.join(project, "elements", "dep.bst")
target_path = os.path.join(project, "elements", "target.bst")
dep = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",},],
"config": {"commands": ["touch %{install-root}/foo", "false",],},
}
_yaml.roundtrip_dump(dep, dep_path)
target = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",}, {"filename": "dep.bst", "type": "build",},],
"config": {"commands": ["test -e /foo",],},
}
_yaml.roundtrip_dump(target, target_path)
# Try to build it, this should result in caching a failure to build dep
result = cli.run(project=project, args=["build", "dep.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's cached in a failed artifact
assert cli.get_element_state(project, "dep.bst") == "failed"
# Now we should fail because we've a cached fail of dep
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's not yet built, since one of its dependencies isn't ready.
assert cli.get_element_state(project, "target.bst") == "waiting"
def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
buildbox_casd = tmp_path.joinpath("bin/buildbox-casd")
buildbox_casd.parent.mkdir()
os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd))
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",},],
"config": {"commands": ["true",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this will fail
result1 = cli.run(project=project, args=["build", "element.bst"], env={"PATH": str(tmp_path.joinpath("bin"))},)
result1.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox")
assert cli.get_element_state(project, "element.bst") == "buildable"
# When rebuilding, this should work
result2 = cli.run(project=project, args=["build", "element.bst"])
result2.assert_success()
assert cli.get_element_state(project, "element.bst") == "cached"
def test_build_shell_fetch(cli, datafiles):
project = str(datafiles)
element_name = "build-shell-fetch.bst"
# Create a file with unique contents such that it cannot be in the cache already
test_filepath = os.path.join(project, "files", "hello.txt")
test_message = "Hello World! {}".format(uuid.uuid4())
with open(test_filepath, "w") as f:
f.write(test_message)
checksum = utils.sha256sum(test_filepath)
# Create an element that has this unique file as a source
element = {
"kind": "manual",
"depends": ["base.bst"],
"sources": [{"kind": "remote", "url": "project_dir:/files/hello.txt", "ref": checksum}],
}
_yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name))
# Ensure our dependencies are cached
result = cli.run(project=project, args=["build", "base.bst"])
result.assert_success()
# Ensure our sources are not cached
assert cli.get_element_state(project, element_name) == "fetch needed"
# Launching a shell should fetch any uncached sources
result = cli.run(project=project, args=["shell", "--build", element_name, "cat", "hello.txt"])
result.assert_success()
assert result.output == test_message
def test_workspace_failed_logs(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "failing_amhello")
element_name = "autotools/amhello-failure.bst"
# Open workspace
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
res.assert_success()
# Try to build and ensure the build fails
res = cli.run(project=project, args=["build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
assert cli.get_element_state(project, element_name) == "failed"
res = cli.run(project=project, args=["artifact", "log", element_name])
res.assert_success()
log = res.output
# Assert that we can get the log
assert log != ""
fail_str = "FAILURE {}: Running build-commands".format(element_name)
batch_fail_str = "FAILURE {}: Running commands".format(element_name)
assert fail_str in log or batch_fail_str in log
def test_build_checkout_cached_fail(cli, datafiles):
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
checkout = os.path.join(cli.directory, "checkout")
# Write out our test target
element = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",},],
"config": {"commands": ["touch %{install-root}/foo", "false",],},
}
_yaml.roundtrip_dump(element, element_path)
# Try to build it, this should result in a failure that contains the content
result = cli.run(project=project, args=["build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's cached in a failed artifact
assert cli.get_element_state(project, "element.bst") == "failed"
# Now check it out
result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout])
result.assert_success()
# Check that the checkout contains the file created before failure
filename = os.path.join(checkout, "foo")
assert os.path.exists(filename)
def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
if on_error == "quit":
pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534")
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",},],
"config": {
"commands": [
"false",
# Ensure unique cache key for different test variants
'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")),
],
},
}
_yaml.roundtrip_dump(element, element_path)
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share:
cli.configure(
{"artifacts": {"url": share.repo, "push": True},}
)
# Build the element, continuing to finish active jobs on error.
result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# This element should have failed
assert cli.get_element_state(project, "element.bst") == "failed"
# This element should have been pushed to the remote
assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst"))
def test_manual_command_subdir(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_path = os.path.join(project, "elements")
element_name = "manual/command-subdir.bst"
sources = [{"kind": "local", "path": "files/manual-element/root"}]
create_manual_element(
element_name,
element_path,
{"install-commands": ["cp hello %{install-root}"]},
{},
{},
sources=sources,
)
# First, verify that element builds, and has the correct expected output.
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
result = cli.run(
project=project,
args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
with open(os.path.join(checkout, "hello")) as f:
assert f.read() == "hello from root\n"
# Now, change element configuration to have a different command-subdir.
# This should result in a different cache key.
create_manual_element(
element_name,
element_path,
{"install-commands": ["cp hello %{install-root}"]},
{"command-subdir": "subdir"},
{},
sources=sources,
)
# Verify that the element needs to be rebuilt.
assert cli.get_element_state(project, element_name) == "buildable"
# Finally, ensure that the variable actually takes effect.
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
shutil.rmtree(checkout)
result = cli.run(
project=project,
args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
with open(os.path.join(checkout, "hello")) as f:
assert f.read() == "hello from subdir\n"
def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
"""Test that we can upload a built artifact that didn't have a valid shell inside.
When we don't have a valid shell, the artifact will be empty, not even the root directory.
This ensures we handle the case of an entirely empty artifact correctly.
"""
if on_error == "quit":
pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534")
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
"kind": "script",
"config": {
"commands": [
"false",
# Ensure unique cache key for different test variants
'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")),
],
},
}
_yaml.roundtrip_dump(element, element_path)
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
# Build the element, continuing to finish active jobs on error.
result = cli.run(
project=project,
args=["--on-error={}".format(on_error), "build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# This element should have failed
assert cli.get_element_state(project, "element.bst") == "failed"
# This element should have been pushed to the remote
assert share.get_artifact(
cli.get_artifact_name(project, "test", "element.bst"))
def test_cache_buildtrees(cli, tmpdir, datafiles):
project = str(datafiles)
element_name = "autotools/amhello.bst"
cwd = str(tmpdir)
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(
str(tmpdir), "share1")) as share1, create_artifact_share(
os.path.join(str(tmpdir),
"share2")) as share2, create_artifact_share(
os.path.join(str(tmpdir),
"share3")) as share3:
cli.configure({
"artifacts": {
"url": share1.repo,
"push": True
},
"cachedir": str(tmpdir)
})
# Build autotools element with the default behavior of caching buildtrees
# only when necessary. The artifact should be successfully pushed to the share1 remote
# and cached locally with an 'empty' buildtree digest, as it's not a
# dangling ref
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == "cached"
assert share1.get_artifact(
cli.get_artifact_name(project, "test", element_name))
# The buildtree dir should not exist, as we set the config to not cache buildtrees.
artifact_name = cli.get_artifact_name(project, "test", element_name)
assert share1.get_artifact(artifact_name)
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
# Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
# that is was cached in share1 as expected without a buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
assert cli.get_element_state(project, element_name) != "cached"
result = cli.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Assert that the default behaviour of pull to not include buildtrees on the artifact
# in share1 which was purposely cached with an empty one behaves as expected. As such the
# pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
# leading to no buildtreedir being extracted
result = cli.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Repeat building the artifacts, this time with cache-buildtrees set to
# 'always' via the cli, as such the buildtree dir should not be empty
cli.configure({
"artifacts": {
"url": share2.repo,
"push": True
},
"cachedir": str(tmpdir)
})
result = cli.run(
project=project,
args=["--cache-buildtrees", "always", "build", element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == "cached"
assert share2.get_artifact(
cli.get_artifact_name(project, "test", element_name))
# Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
# Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
# that it was cached in share2 as expected with a populated buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
assert cli.get_element_state(project, element_name) != "cached"
result = cli.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Clarify that the user config option for cache-buildtrees works as the cli
# main option does. Point to share3 which does not have the artifacts cached to force
# a build
cli.configure({
"artifacts": {
"url": share3.repo,
"push": True
},
"cachedir": str(tmpdir),
"cache": {
"cache-buildtrees": "always"
},
})
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == "cached"
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
def test_pullbuildtrees(cli2, tmpdir, datafiles):
project = str(datafiles)
element_name = "autotools/amhello.bst"
cwd = str(tmpdir)
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(
str(tmpdir), "share1")) as share1, create_artifact_share(
os.path.join(str(tmpdir),
"share2")) as share2, create_artifact_share(
os.path.join(str(tmpdir),
"share3")) as share3:
cli2.configure({
"artifacts": {
"servers": [{
"url": share1.repo,
"push": True
}]
},
"cachedir": str(tmpdir),
"cache": {
"cache-buildtrees": "always"
},
})
# Build autotools element, checked pushed, delete local
result = cli2.run(project=project, args=["build", element_name])
assert result.exit_code == 0
assert cli2.get_element_state(project, element_name) == "cached"
assert share1.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, assert that pulling again
# doesn't create a pull job, then assert with buildtrees user
# config set creates a pull job.
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
cli2.configure({"cache": {"pull-buildtrees": True}})
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, then assert that pulling
# with buildtrees cli flag set creates a pull job.
# Also assert that the buildtree is added to the local CAS.
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
artifact_name = cli2.get_artifact_name(project, "test", element_name)
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
default_state(cli2, tmpdir, share1)
# Pull artifact with pullbuildtrees set in user config, then assert
# that pulling with the same user config doesn't creates a pull job,
# or when buildtrees cli flag is set.
cli2.configure({"cache": {"pull-buildtrees": True}})
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config and buildtrees cli flag set, then assert
# that pulling with pullbuildtrees set in user config doesn't create a pull
# job.
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure({"cache": {"pull-buildtrees": True}})
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Assert that a partial build element (not containing a populated buildtree dir)
# can't be pushed to an artifact share, then assert that a complete build element
# can be. This will attempt a partial pull from share1 and then a partial push
# to share2
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure(
{"artifacts": {
"servers": [{
"url": share2.repo,
"push": True
}]
}})
result = cli2.run(project=project,
args=["artifact", "push", element_name])
assert element_name not in result.get_pushed_elements()
assert not share2.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
# Assert that after pulling the missing buildtree the element artifact can be
# successfully pushed to the remote. This will attempt to pull the buildtree
# from share1 and then a 'complete' push to share2
cli2.configure(
{"artifacts": {
"servers": [{
"url": share1.repo,
"push": False
}]
}})
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure(
{"artifacts": {
"servers": [{
"url": share2.repo,
"push": True
}]
}})
result = cli2.run(project=project,
args=["artifact", "push", element_name])
assert element_name in result.get_pushed_elements()
assert share2.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
default_state(cli2, tmpdir, share1)
# Assert that bst artifact push will automatically attempt to pull a missing buildtree
# if pull-buildtrees is set, however as share3 is the only defined remote and is empty,
# assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the
# artifact cannot be pushed.
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure(
{"artifacts": {
"servers": [{
"url": share3.repo,
"push": True
}]
}})
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "push", element_name])
assert element_name not in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
assert element_name not in result.get_pushed_elements()
assert not share3.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
# Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will
# automatically attempt to pull it and will be successful, leading to the full artifact being pushed
# to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote,
# without exlipictly requiring a bst artifact pull.
cli2.configure({
"artifacts": {
"servers": [{
"url": share1.repo,
"push": False
}, {
"url": share3.repo,
"push": True
}]
}
})
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "push", element_name])
assert element_name in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert element_name in result.get_pushed_elements()
assert share3.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
