def test_push_missing_source_after_build(cli, tmpdir, datafiles):
cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_name = "import-bin.bst"
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
# Delete source but keep artifact in cache
shutil.rmtree(os.path.join(cache_dir, "elementsources"))
shutil.rmtree(os.path.join(cache_dir, "source_protos"))
with create_artifact_share(os.path.join(str(tmpdir),
"sourceshare")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
res = cli.run(project=project_dir,
args=["source", "push", element_name])
res.assert_success()
assert "fetch:{}".format(element_name) in res.stderr
assert "Pushed source" in res.stderr
def test_artifact_delete_artifact(cli, tmpdir, datafiles, with_project):
project = str(datafiles)
element = "target.bst"
# Configure a local cache
local_cache = os.path.join(str(tmpdir), "cache")
cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
result = cli.run(project=project, args=["build", element])
result.assert_success()
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the project.conf if we're going to try this without a project
if not with_project:
os.remove(os.path.join(project, "project.conf"))
# Delete the artifact
result = cli.run(project=project, args=["artifact", "delete", artifact])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
assert not os.path.exists(
os.path.join(local_cache, "cas", "refs", "heads", artifact))
def test_strict_dependencies(cli, datafiles, target, expected_state):
project = str(datafiles)
# Configure non strict mode, this will have
# an effect on the build and the `bst show`
# commands run via cli.get_element_states()
cli.configure({"projects": {"test": {"strict": False}}})
result = cli.run(project=project, silent=True, args=["build", target])
result.assert_success()
states = cli.get_element_states(project, ["base.bst", target])
assert states["base.bst"] == "cached"
assert states[target] == "cached"
# Now modify the file, effectively causing the common base.bst
# dependency to change it's cache key
hello_path = os.path.join(project, "files", "hello.txt")
with open(hello_path, "w", encoding="utf-8") as f:
f.write("Goodbye")
# Now assert that we have the states we expect as a result
states = cli.get_element_states(project, ["base.bst", target])
assert states["base.bst"] == "buildable"
assert states[target] == expected_state
def test_shell_pull_cached_buildtree(share_with_buildtrees, datafiles, cli,
pull_deps, pull_buildtree):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
cli.configure(
{"artifacts": {
"servers": [{
"url": share_with_buildtrees.repo
}]
}})
# Optionally pull the buildtree along with `bst artifact pull`
maybe_pull_deps(cli, project, element_name, pull_deps, pull_buildtree)
# Run the shell and request that required artifacts and buildtrees should be pulled
result = cli.run(
project=project,
args=[
"--pull-buildtrees",
"shell",
"--build",
element_name,
"--use-buildtree",
"--",
"cat",
"test",
],
)
# In this case, we should succeed every time, regardless of what was
# originally available in the local cache.
#
result.assert_success()
assert "Hi" in result.output
def test_unfetched_junction(cli, tmpdir, datafiles, strict, ref_storage):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
cli.configure({"projects": {"test": {"strict": strict}}})
# Create a repo to hold the subproject and generate a junction element for it
ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
if ref_storage == "project.refs":
project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
_yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Now try to fetch it, this should automatically result in fetching
# the junction itself.
result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"])
result.assert_success()
def test_build_push_source_twice(cli, tmpdir, datafiles):
cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_name = "import-bin.bst"
with create_artifact_share(os.path.join(str(tmpdir),
"sourceshare")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
assert "fetch:{}".format(element_name) in res.stderr
assert "Pushed source" in res.stderr
# The second build pipeline is a no-op as everything is already cached.
# However, this verifies that the pipeline behaves as expected.
res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
assert "fetch:{}".format(element_name) not in res.stderr
assert "Pushed source" not in res.stderr
def test_max_jobs(cli, datafiles, cli_value, config_value):
project = str(datafiles)
target = "target.bst"
# Specify `--max-jobs` if this test sets it
args = []
if cli_value is not None:
args += ["--max-jobs", cli_value]
args += ["show", "--deps", "none", "--format", "%{vars}", target]
# Specify `max-jobs` in user configuration if this test sets it
if config_value is not None:
cli.configure({"build": {"max-jobs": config_value}})
result = cli.run(project=project, silent=True, args=args)
result.assert_success()
loaded = _yaml.load_data(result.output)
loaded_value = loaded.get_int("max-jobs")
# We expect the value provided on the command line to take
# precedence over the configuration file value, if specified.
#
# If neither are specified then we expect the default
expected_value = cli_value or config_value or "0"
if expected_value == "0":
# If we are expecting the automatic behavior of using the maximum
# number of cores available, just check that it is a value > 0
assert loaded_value > 0, "Automatic setting of max-jobs didnt work"
else:
# Check that we got the explicitly set value
assert loaded_value == int(expected_value)
def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
# Set up remote and local shares
local_cache = os.path.join(str(tmpdir), "artifacts")
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote:
cli.configure({
"artifacts": {
"servers": [{
"url": remote.repo,
"push": True
}]
},
"cachedir": local_cache,
})
# Build the element
result = cli.run(project=project, args=["build", element])
result.assert_success()
# Make sure it's in the share
assert remote.get_artifact(
cli.get_artifact_name(project, "test", element))
# Delete the artifact from the local cache
result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
assert cli.get_element_state(project, element) != "cached"
result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
assert "available {}".format(element) in result.output
def test_shell_use_cached_buildtree(share_with_buildtrees, datafiles, cli,
pull_deps, pull_buildtree, expect_error):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
cli.configure(
{"artifacts": {
"servers": [{
"url": share_with_buildtrees.repo
}]
}})
# Optionally pull the buildtree along with `bst artifact pull`
maybe_pull_deps(cli, project, element_name, pull_deps, pull_buildtree)
# Disable access to the artifact server after pulling, so that `bst shell` cannot automatically
# pull the missing bits, this should be equivalent to the missing bits being missing in a
# remote server
cli.configure({"artifacts": {}})
# Run the shell without asking it to pull any buildtree, just asking to use a buildtree
result = cli.run(project=project,
args=[
"shell", "--build", element_name, "--use-buildtree",
"--", "cat", "test"
])
if expect_error:
result.assert_main_error(ErrorDomain.APP, expect_error)
else:
result.assert_success()
assert "Hi" in result.output
def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
dep = "compose-all.bst"
# Configure a local cache
local_cache = os.path.join(str(tmpdir), "cache")
cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
result = cli.run(project=project, args=["build", element])
result.assert_success()
assert cli.get_element_states(project, [element, dep], deps="none") == {
element: "cached",
dep: "cached",
}
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the artifact
result = cli.run(project=project,
args=["artifact", "delete", artifact, dep])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact))
# Check that the dependency ELEMENT is no longer cached
assert cli.get_element_state(project, dep) != "cached"
def test_custom_logging(cli, tmpdir, datafiles):
project = str(datafiles)
bin_files_path = os.path.join(project, "files", "bin-files")
element_path = os.path.join(project, "elements")
element_name = "fetch-test-git.bst"
custom_log_format = "%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us},%{key},%{element},%{action},%{message}"
user_config = {"logging": {"message-format": custom_log_format}}
cli.configure(user_config)
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
repo = create_repo("tar", str(tmpdir))
ref = repo.create(bin_files_path)
# Write out our test target
element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Now try to fetch it
result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
m = re.search(
r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*" r",SUCCESS,Query cache",
result.stderr,
)
assert m is not None
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
repo = create_repo("tar", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "pull-missing-local-blob",
"min-version": "2.0",
"element-path": "elements",
}
project_file = os.path.join(str(tmpdir), "project.conf")
_yaml.roundtrip_dump(project_config, project_file)
input_config = {
"kind": "import",
"sources": [repo.source_config()],
}
input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
depends_name = "depends.bst"
depends_config = {"kind": "stack", "depends": [input_name]}
depends_file = os.path.join(element_dir, depends_name)
_yaml.roundtrip_dump(depends_config, depends_file)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the import-bin element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
result = cli.run(project=project, args=["build", input_name])
result.assert_success()
assert cli.get_element_state(project, input_name) == "cached"
# Delete a file blob from the local cache.
# This is a placeholder to test partial CAS handling until we support
# partial artifact pulling (or blob-based CAS expiry).
#
digest = utils.sha256sum(
os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
objpath = os.path.join(cli.directory, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# Now try bst build
result = cli.run(project=project, args=["build", depends_name])
result.assert_success()
# Assert that the import-bin artifact was pulled (completing the partial artifact)
assert result.get_pulled_elements() == [input_name]
def test_shell_use_uncached_buildtree(share_without_buildtrees, datafiles,
cli):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
cli.configure(
{"artifacts": {
"servers": [{
"url": share_without_buildtrees.repo
}]
}})
# Pull everything we would need
maybe_pull_deps(cli, project, element_name, "all", True)
# Run the shell without asking it to pull any buildtree, just asking to use a buildtree
result = cli.run(project=project,
args=[
"shell", "--build", element_name, "--use-buildtree",
"--", "cat", "test"
])
# Sorry, a buildtree was never cached for this element
result.assert_main_error(
ErrorDomain.APP,
"missing-buildtree-artifact-created-without-buildtree")
def test_pull_missing_blob_split_share(cli, tmpdir, datafiles):
project = str(datafiles)
indexshare = os.path.join(str(tmpdir), "indexshare")
storageshare = os.path.join(str(tmpdir), "storageshare")
with create_split_share(indexshare, storageshare) as (index, storage):
cli.configure({
"artifacts": {
"servers": [
{
"url": index.repo,
"push": True,
"type": "index"
},
{
"url": storage.repo,
"push": True,
"type": "storage"
},
]
}
})
_test_pull_missing_blob(cli, project, index, storage)
def test_partial_checkout_fail(tmpdir, datafiles, cli):
project = str(datafiles)
build_elt = "import-bin.bst"
checkout_dir = os.path.join(str(tmpdir), "checkout")
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
res = cli.run(project=project,
args=[
"artifact", "checkout", build_elt, "--directory",
checkout_dir
])
res.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
assert re.findall(
r"Remote \((\S+)\) does not have artifact (\S+) cached",
res.stderr)
def test_push_pull_specific_remote(cli, tmpdir, datafiles):
project = str(datafiles)
with create_artifact_share(os.path.join(
str(tmpdir),
"goodartifactshare")) as good_share, create_artifact_share(
os.path.join(str(tmpdir), "badartifactshare")) as bad_share:
# Build the target so we have it cached locally only.
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
state = cli.get_element_state(project, "target.bst")
assert state == "cached"
# Configure the default push location to be bad_share; we will assert that
# nothing actually gets pushed there.
cli.configure({
"artifacts": {
"servers": [
{
"url": bad_share.repo,
"push": True
},
]
}
})
# Now try `bst artifact push` to the good_share.
result = cli.run(project=project,
args=[
"artifact", "push", "target.bst",
"--artifact-remote", good_share.repo
])
result.assert_success()
# Assert that all the artifacts are in the share we pushed
# to, and not the other.
assert_shared(cli, good_share, project, "target.bst")
assert_not_shared(cli, bad_share, project, "target.bst")
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the good_share.
#
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
result = cli.run(project=project,
args=[
"artifact", "pull", "target.bst",
"--artifact-remote", good_share.repo
])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert cli.get_element_state(project, "target.bst") == "cached"
def test_source_push(cli, tmpdir, datafiles):
cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
with create_artifact_share(os.path.join(str(tmpdir),
"sourceshare")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
repo = create_repo("tar", str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
element_name = "push.bst"
element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# get the source object
with dummy_context(config=user_config_file) as context:
project = Project(project_dir, context)
project.ensure_fully_loaded()
element = project.load_elements(["push.bst"])[0]
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
# check we don't have it in the current cache
assert not share.get_source_proto(source._get_source_name())
# build the element, this should fetch and then push the source to the
# remote
res = cli.run(project=project_dir, args=["build", "push.bst"])
res.assert_success()
assert "Pushed source" in res.stderr
# check that we've got the remote locally now
sourcecache = context.sourcecache
assert sourcecache.contains(source)
# check that the remote CAS now has it
digest = sourcecache.export(source)._get_digest()
assert share.has_object(digest)
def test_expiry_order(cli, datafiles):
project = str(datafiles)
element_path = "elements"
checkout = os.path.join(project, "workspace")
cli.configure({"cache": {"quota": 9000000}})
# Create an artifact
create_element_size("dep.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "dep.bst"])
res.assert_success()
# Create another artifact
create_element_size("unrelated.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# And build something else
create_element_size("target.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
create_element_size("target2.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
wait_for_cache_granularity()
# Now extract dep.bst
res = cli.run(
project=project,
args=["artifact", "checkout", "dep.bst", "--directory", checkout])
res.assert_success()
# Finally, build something that will cause the cache to overflow
create_element_size("expire.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "expire.bst"])
res.assert_success()
# While dep.bst was the first element to be created, it should not
# have been removed.
# Note that buildstream will reduce the cache to 50% of the
# original size - we therefore remove multiple elements.
check_elements = [
"unrelated.bst", "target.bst", "target2.bst", "dep.bst", "expire.bst"
]
states = cli.get_element_states(project, check_elements)
assert tuple(states[element] for element in check_elements) == (
"buildable",
"buildable",
"buildable",
"cached",
"cached",
)
def test_build_remote_option(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
caplog.set_level(1)
with create_artifact_share(os.path.join(
str(tmpdir),
"artifactshare1")) as shareuser, create_artifact_share(
os.path.join(str(tmpdir), "artifactshare2")) as sharecli:
# Configure shareuser remote in user conf
cli.configure({
"artifacts": {
"servers": [{
"url": shareuser.repo,
"push": True
}]
}
})
# Push the artifacts to the shareuser remote.
# Assert that shareuser has the artfifacts cached, but sharecli doesn't,
# then delete locally cached elements
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"]
for element_name in all_elements:
assert element_name in result.get_pushed_elements()
assert_not_shared(cli, sharecli, project, element_name)
assert_shared(cli, shareuser, project, element_name)
cli.remove_artifact_from_cache(project, element_name)
# Now check that a build with cli set as sharecli results in nothing being pulled,
# as it doesn't have them cached and shareuser should be ignored. This
# will however result in the artifacts being built and pushed to it
result = cli.run(
project=project,
args=["build", "--artifact-remote", sharecli.repo, "target.bst"])
result.assert_success()
for element_name in all_elements:
assert element_name not in result.get_pulled_elements()
assert_shared(cli, sharecli, project, element_name)
cli.remove_artifact_from_cache(project, element_name)
# Now check that a clean build with cli set as sharecli should result in artifacts only
# being pulled from it, as that was provided via the cli and is populated
result = cli.run(
project=project,
args=["build", "--artifact-remote", sharecli.repo, "target.bst"])
result.assert_success()
for element_name in all_elements:
assert cli.get_element_state(project, element_name) == "cached"
assert element_name in result.get_pulled_elements()
assert shareuser.repo not in result.stderr
assert sharecli.repo in result.stderr
def test_invalid_cache_pullbuildtrees(cli, datafiles, value, success):
project = str(datafiles)
cli.configure({"cache": {
"pull-buildtrees": value,
}})
res = cli.run(project=project, args=["workspace", "list"])
if success:
res.assert_success()
else:
res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
def test_keep_dependencies(cli, datafiles):
project = str(datafiles)
element_path = "elements"
# Skip this test if we do not have support for subsecond precision mtimes
#
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
cli.configure({"cache": {"quota": 10000000}})
# Create a pretty big dependency
create_element_size("dependency.bst", project, element_path, [], 5000000)
res = cli.run(project=project, args=["build", "dependency.bst"])
res.assert_success()
# Now create some other unrelated artifact
create_element_size("unrelated.bst", project, element_path, [], 4000000)
res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project,
["dependency.bst", "unrelated.bst"])
assert states["dependency.bst"] == "cached"
assert states["unrelated.bst"] == "cached"
# We try to build an element which depends on the LRU artifact,
# and could therefore fail if we didn't make sure dependencies
# aren't removed.
#
# Since some artifact caches may implement weak cache keys by
# duplicating artifacts (bad!) we need to make this equal in size
# or smaller than half the size of its dependencies.
#
create_element_size("target.bst", project, element_path,
["dependency.bst"], 2000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
states = cli.get_element_states(project, ["target.bst", "unrelated.bst"])
assert states["target.bst"] == "cached"
assert states["dependency.bst"] == "cached"
assert states["unrelated.bst"] != "cached"
def test_never_delete_required(cli, datafiles):
project = str(datafiles)
element_path = "elements"
# Skip this test if we do not have support for subsecond precision mtimes
#
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
cli.configure({
"cache": {
"quota": 10000000
},
"scheduler": {
"fetchers": 1,
"builders": 1
}
})
# Create a linear build tree
create_element_size("dep1.bst", project, element_path, [], 8000000)
create_element_size("dep2.bst", project, element_path, ["dep1.bst"],
8000000)
create_element_size("dep3.bst", project, element_path, ["dep2.bst"],
8000000)
create_element_size("target.bst", project, element_path, ["dep3.bst"],
8000000)
# Build dep1.bst, which should fit into the cache.
res = cli.run(project=project, args=["build", "dep1.bst"])
res.assert_success()
# We try to build this pipeline, but it's too big for the
# cache. Since all elements are required, the build should fail.
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
res.assert_task_error(ErrorDomain.CAS, "cache-too-full")
states = cli.get_element_states(project, ["target.bst"])
assert states["dep1.bst"] == "cached"
assert states["dep2.bst"] != "cached"
assert states["dep3.bst"] != "cached"
assert states["target.bst"] != "cached"
def test_dynamic_build_plan(cli, tmpdir, datafiles):
project = str(datafiles)
target = "checkout-deps.bst"
build_dep = "import-dev.bst"
runtime_dep = "import-bin.bst"
all_elements = [target, build_dep, runtime_dep]
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the target element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["build", target])
result.assert_success()
# Assert that everything is now cached in the remote.
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache directory
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
states = cli.get_element_states(project, all_elements)
assert not any(states[e] == "cached" for e in all_elements)
# Now try to rebuild target
result = cli.run(project=project, args=["build", target])
result.assert_success()
# Assert that target and runtime dependency were pulled
# but build dependency was not pulled as it wasn't needed
# (dynamic build plan).
assert target in result.get_pulled_elements()
assert runtime_dep in result.get_pulled_elements()
assert build_dep not in result.get_pulled_elements()
# And assert that the pulled elements are again in the local cache
states = cli.get_element_states(project, all_elements)
assert states[target] == "cached"
assert states[runtime_dep] == "cached"
assert states[build_dep] != "cached"
def default_state(cli, tmpdir, share):
shutil.rmtree(os.path.join(str(tmpdir), "cas"))
cli.configure({
"artifacts": {
"servers": [{
"url": share.repo,
"push": False
}]
},
"cachedir": str(tmpdir),
"cache": {
"pull-buildtrees": False
},
})
def test_pull_missing_blob(cli, tmpdir, datafiles):
project = str(datafiles)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
_test_pull_missing_blob(cli, project, share, share)
def test_pull_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
# Configure a local cache
local_cache = os.path.join(str(tmpdir), "cache")
cli.configure({"cachedir": local_cache})
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the target element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["build", element])
result.assert_success()
# Assert that the *artifact* is cached locally
cache_key = cli.get_element_key(project, element)
artifact_ref = os.path.join("test",
os.path.splitext(element)[0], cache_key)
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact_ref))
# Assert that the target is shared (note that assert shared will use the artifact name)
assert_shared(cli, share, project, element)
# Now we've pushed, remove the local cache
shutil.rmtree(os.path.join(local_cache, "artifacts"))
# Assert that nothing is cached locally anymore
assert not os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact_ref))
# Now try bst artifact pull
result = cli.run(project=project,
args=["artifact", "pull", artifact_ref])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact_ref))
def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason):
project = str(datafiles)
os.makedirs(os.path.join(project, "elements"))
cli.configure({
"cache": {
"quota": quota,
},
})
res = cli.run(project=project, args=["workspace", "list"])
if err_domain == "success":
res.assert_success()
else:
res.assert_main_error(err_domain, err_reason)
def test_push_pull_deps(cli, tmpdir, datafiles, deps, expected_states):
project = str(datafiles)
target = "checkout-deps.bst"
build_dep = "import-dev.bst"
runtime_dep = "import-bin.bst"
all_elements = [target, build_dep, runtime_dep]
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the target element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["build", target])
result.assert_success()
# Assert that everything is now cached in the remote.
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
states = cli.get_element_states(project, all_elements)
assert not any(states[e] == "cached" for e in all_elements)
# Now try bst artifact pull
result = cli.run(project=project,
args=["artifact", "pull", "--deps", deps, target])
result.assert_success()
# And assert that the pulled elements are again in the local cache
states = cli.get_element_states(project, all_elements)
states_flattended = (states[target], states[build_dep],
states[runtime_dep])
assert states_flattended == expected_states
def test_source_cache_empty_artifact_cache(cli, tmpdir, datafiles):
cachedir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_path = os.path.join(project_dir, "elements")
with create_artifact_share(os.path.join(str(tmpdir), "share")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"artifacts": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cachedir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
create_element_size("repo.bst", project_dir, element_path, [], 10000)
res = cli.run(project=project_dir, args=["source", "push", "repo.bst"])
res.assert_success()
assert "Pushed source " in res.stderr
# delete local sources and check it pulls sources, builds
# and then pushes the artifacts
shutil.rmtree(os.path.join(cachedir, "cas"))
shutil.rmtree(os.path.join(cachedir, "sources"))
res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Remote ({}) does not have artifact ".format(
share.repo) in res.stderr
assert "Pulled source" in res.stderr
assert "Caching artifact" in res.stderr
assert "Pushed artifact" in res.stderr
def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
caplog.set_level(1)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure({"artifacts": {"servers": [{"url": share.repo}]}})
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
assert not result.get_pulled_elements(
), "No elements should have been pulled since the cache was empty"
assert "INFO Remote ({}) does not have".format(
share.repo) in result.stderr
assert "SKIPPED Pull" in result.stderr
