def test_integration_partial_artifact(cli, datafiles, tmpdir,
integration_cache):
project = str(datafiles)
element_name = "autotools/amhello.bst"
# push to an artifact server so we can pull from it later.
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
# If the build is cached then it might not push to the artifact cache
result = cli.run(project=project,
args=["artifact", "push", element_name])
result.assert_success()
result = cli.run(project=project, args=["shell", element_name])
result.assert_success()
# do a checkout and get the digest of the hello binary.
result = cli.run(
project=project,
args=[
"artifact",
"checkout",
"--deps",
"none",
"--directory",
os.path.join(str(tmpdir), "tmp"),
"autotools/amhello.bst",
],
)
result.assert_success()
digest = utils.sha256sum(
os.path.join(str(tmpdir), "tmp", "usr", "bin", "hello"))
# Remove the binary from the CAS
cachedir = cli.config["cachedir"]
objpath = os.path.join(cachedir, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# check shell doesn't work
result = cli.run(project=project,
args=["shell", element_name, "--", "hello"])
result.assert_main_error(ErrorDomain.APP, "shell-missing-deps")
# check the artifact gets completed with '--pull' specified
result = cli.run(project=project,
args=["shell", "--pull", element_name, "--", "hello"])
result.assert_success()
assert "autotools/amhello.bst" in result.get_pulled_elements()
def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
if on_error == "quit":
pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534")
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",},],
"config": {
"commands": [
"false",
# Ensure unique cache key for different test variants
'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")),
],
},
}
_yaml.roundtrip_dump(element, element_path)
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share:
cli.configure(
{"artifacts": {"url": share.repo, "push": True},}
)
# Build the element, continuing to finish active jobs on error.
result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# This element should have failed
assert cli.get_element_state(project, "element.bst") == "failed"
# This element should have been pushed to the remote
assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst"))
def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir,
datafiles):
# Test that if we stage a cached and empty buildtree, we warn the user.
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
# Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
# without caching a buildtree which is the default bst behaviour
cli_integration.configure({"cachedir": str(tmpdir)})
res = cli_integration.run(project=project, args=["build", element_name])
res.assert_success()
res = cli_integration.run(project=project,
args=[
"shell", "--build", "--use-buildtree",
"always", element_name, "--", "cat", "test"
])
res.assert_main_error(ErrorDomain.APP, None)
assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
# Now attempt the same with the try option, this should not attempt to find a buildtree
# and just launch the shell, however the cat should still fail.
res = cli_integration.run(project=project,
args=[
"shell", "--build", "--use-buildtree", "try",
element_name, "--", "cat", "test"
])
assert "Artifact created without buildtree, shell will be loaded without it" in res.stderr
assert "Hi" not in res.output
def test_buildtree_from_failure_option_always(cli_integration, tmpdir,
datafiles):
project = str(datafiles)
element_name = "build-shell/buildtree-fail.bst"
# build with --cache-buildtrees set to 'always', behaviour should match
# default behaviour (which is always) as the buildtree will explicitly have been
# cached with content.
cli_integration.configure({"cachedir": str(tmpdir)})
res = cli_integration.run(
project=project,
args=["--cache-buildtrees", "always", "build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
res = cli_integration.run(project=project,
args=[
"shell", "--build", element_name,
"--use-buildtree", "always", "--", "cat",
"test"
])
res.assert_success()
assert "WARNING: using a buildtree from a failed build" in res.stderr
assert "Hi" in res.output
def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
"""Test that we can upload a built artifact that didn't have a valid shell inside.
When we don't have a valid shell, the artifact will be empty, not even the root directory.
This ensures we handle the case of an entirely empty artifact correctly.
"""
if on_error == "quit":
pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534")
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
"kind": "script",
"config": {
"commands": [
"false",
# Ensure unique cache key for different test variants
'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")),
],
},
}
_yaml.roundtrip_dump(element, element_path)
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
# Build the element, continuing to finish active jobs on error.
result = cli.run(
project=project,
args=["--on-error={}".format(on_error), "build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# This element should have failed
assert cli.get_element_state(project, "element.bst") == "failed"
# This element should have been pushed to the remote
assert share.get_artifact(
cli.get_artifact_name(project, "test", "element.bst"))
def test_buildtree_from_failure_option_never(cli_integration, tmpdir,
datafiles):
project = str(datafiles)
element_name = "build-shell/buildtree-fail.bst"
# Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
# without caching a buildtree explicitly
cli_integration.configure({"cachedir": str(tmpdir)})
res = cli_integration.run(
project=project,
args=["--cache-buildtrees", "never", "build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
res = cli_integration.run(project=project,
args=[
"shell", "--build", element_name,
"--use-buildtree", "always", "--", "cat",
"test"
])
res.assert_main_error(ErrorDomain.APP, None)
assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
def test_cache_buildtrees(cli, tmpdir, datafiles):
project = str(datafiles)
element_name = "autotools/amhello.bst"
cwd = str(tmpdir)
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(
str(tmpdir), "share1")) as share1, create_artifact_share(
os.path.join(str(tmpdir),
"share2")) as share2, create_artifact_share(
os.path.join(str(tmpdir),
"share3")) as share3:
cli.configure({
"artifacts": {
"url": share1.repo,
"push": True
},
"cachedir": str(tmpdir)
})
# Build autotools element with the default behavior of caching buildtrees
# only when necessary. The artifact should be successfully pushed to the share1 remote
# and cached locally with an 'empty' buildtree digest, as it's not a
# dangling ref
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == "cached"
assert share1.get_artifact(
cli.get_artifact_name(project, "test", element_name))
# The buildtree dir should not exist, as we set the config to not cache buildtrees.
artifact_name = cli.get_artifact_name(project, "test", element_name)
assert share1.get_artifact(artifact_name)
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
# Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
# that is was cached in share1 as expected without a buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
assert cli.get_element_state(project, element_name) != "cached"
result = cli.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Assert that the default behaviour of pull to not include buildtrees on the artifact
# in share1 which was purposely cached with an empty one behaves as expected. As such the
# pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
# leading to no buildtreedir being extracted
result = cli.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Repeat building the artifacts, this time with cache-buildtrees set to
# 'always' via the cli, as such the buildtree dir should not be empty
cli.configure({
"artifacts": {
"url": share2.repo,
"push": True
},
"cachedir": str(tmpdir)
})
result = cli.run(
project=project,
args=["--cache-buildtrees", "always", "build", element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == "cached"
assert share2.get_artifact(
cli.get_artifact_name(project, "test", element_name))
# Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
# Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
# that it was cached in share2 as expected with a populated buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
assert cli.get_element_state(project, element_name) != "cached"
result = cli.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Clarify that the user config option for cache-buildtrees works as the cli
# main option does. Point to share3 which does not have the artifacts cached to force
# a build
cli.configure({
"artifacts": {
"url": share3.repo,
"push": True
},
"cachedir": str(tmpdir),
"cache": {
"cache-buildtrees": "always"
},
})
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == "cached"
with cli.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
def test_pullbuildtrees(cli2, tmpdir, datafiles):
project = str(datafiles)
element_name = "autotools/amhello.bst"
cwd = str(tmpdir)
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(
str(tmpdir), "share1")) as share1, create_artifact_share(
os.path.join(str(tmpdir),
"share2")) as share2, create_artifact_share(
os.path.join(str(tmpdir),
"share3")) as share3:
cli2.configure({
"artifacts": {
"servers": [{
"url": share1.repo,
"push": True
}]
},
"cachedir": str(tmpdir),
"cache": {
"cache-buildtrees": "always"
},
})
# Build autotools element, checked pushed, delete local
result = cli2.run(project=project, args=["build", element_name])
assert result.exit_code == 0
assert cli2.get_element_state(project, element_name) == "cached"
assert share1.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, assert that pulling again
# doesn't create a pull job, then assert with buildtrees user
# config set creates a pull job.
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
cli2.configure({"cache": {"pull-buildtrees": True}})
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, then assert that pulling
# with buildtrees cli flag set creates a pull job.
# Also assert that the buildtree is added to the local CAS.
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
artifact_name = cli2.get_artifact_name(project, "test", element_name)
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
default_state(cli2, tmpdir, share1)
# Pull artifact with pullbuildtrees set in user config, then assert
# that pulling with the same user config doesn't creates a pull job,
# or when buildtrees cli flag is set.
cli2.configure({"cache": {"pull-buildtrees": True}})
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config and buildtrees cli flag set, then assert
# that pulling with pullbuildtrees set in user config doesn't create a pull
# job.
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure({"cache": {"pull-buildtrees": True}})
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Assert that a partial build element (not containing a populated buildtree dir)
# can't be pushed to an artifact share, then assert that a complete build element
# can be. This will attempt a partial pull from share1 and then a partial push
# to share2
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure(
{"artifacts": {
"servers": [{
"url": share2.repo,
"push": True
}]
}})
result = cli2.run(project=project,
args=["artifact", "push", element_name])
assert element_name not in result.get_pushed_elements()
assert not share2.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
# Assert that after pulling the missing buildtree the element artifact can be
# successfully pushed to the remote. This will attempt to pull the buildtree
# from share1 and then a 'complete' push to share2
cli2.configure(
{"artifacts": {
"servers": [{
"url": share1.repo,
"push": False
}]
}})
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure(
{"artifacts": {
"servers": [{
"url": share2.repo,
"push": True
}]
}})
result = cli2.run(project=project,
args=["artifact", "push", element_name])
assert element_name in result.get_pushed_elements()
assert share2.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
default_state(cli2, tmpdir, share1)
# Assert that bst artifact push will automatically attempt to pull a missing buildtree
# if pull-buildtrees is set, however as share3 is the only defined remote and is empty,
# assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the
# artifact cannot be pushed.
result = cli2.run(project=project,
args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
cli2.configure(
{"artifacts": {
"servers": [{
"url": share3.repo,
"push": True
}]
}})
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "push", element_name])
assert element_name not in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert not buildtreedir
assert element_name not in result.get_pushed_elements()
assert not share3.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
# Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will
# automatically attempt to pull it and will be successful, leading to the full artifact being pushed
# to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote,
# without exlipictly requiring a bst artifact pull.
cli2.configure({
"artifacts": {
"servers": [{
"url": share1.repo,
"push": False
}, {
"url": share3.repo,
"push": True
}]
}
})
result = cli2.run(
project=project,
args=["--pull-buildtrees", "artifact", "push", element_name])
assert element_name in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd,
artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert element_name in result.get_pushed_elements()
assert share3.get_artifact(
cli2.get_artifact_name(project, "test", element_name))
