def test_artifact_too_large(cli, datafiles, tmpdir):
project = str(datafiles)
element_path = "elements"
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Mock a file system with 5 MB total space
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"),
quota=int(5e6)) as share:
# Configure bst to push to the remote cache
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}],
}})
# Create and push a 3MB element
create_element_size("small_element.bst", project, element_path, [],
int(3e6))
result = cli.run(project=project, args=["build", "small_element.bst"])
result.assert_success()
# Create and try to push a 6MB element.
create_element_size("large_element.bst", project, element_path, [],
int(6e6))
result = cli.run(project=project, args=["build", "large_element.bst"])
# This should fail; the server will refuse to store the CAS
# blobs for the artifact, and then fail to find the files for
# the uploaded artifact proto.
#
# FIXME: This should be extremely uncommon in practice, since
# the artifact needs to be at least half the cache size for
# this to happen. Nonetheless, a nicer error message would be
# nice (perhaps we should just disallow uploading artifacts
# that large).
result.assert_main_error(ErrorDomain.STREAM, None)
# Ensure that the small artifact is still in the share
states = cli.get_element_states(
project, ["small_element.bst", "large_element.bst"])
assert states["small_element.bst"] == "cached"
assert_shared(cli, share, project, "small_element.bst")
# Ensure that the artifact is cached locally but NOT remotely
assert states["large_element.bst"] == "cached"
assert_not_shared(cli, share, project, "large_element.bst")
def test_source_cache_empty_artifact_cache(cli, tmpdir, datafiles):
cachedir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_path = os.path.join(project_dir, "elements")
with create_artifact_share(os.path.join(str(tmpdir), "share")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"artifacts": {
"servers": [{
"url": share.repo,
"push": True,
}]
},
"cachedir": cachedir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
create_element_size("repo.bst", project_dir, element_path, [], 10000)
res = cli.run(project=project_dir, args=["source", "push", "repo.bst"])
res.assert_success()
assert "Pushed source " in res.stderr
# delete local sources and check it pulls sources, builds
# and then pushes the artifacts
shutil.rmtree(os.path.join(cachedir, "cas"))
shutil.rmtree(os.path.join(cachedir, "sources"))
res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Remote ({}) does not have artifact ".format(
share.repo) in res.stderr
assert "Pulled source" in res.stderr
assert "Caching artifact" in res.stderr
assert "Pushed artifact" in res.stderr
def test_cleanup_first(cli, datafiles):
project = str(datafiles)
element_path = "elements"
cli.configure({"cache": {
"quota": 10000000,
}})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
create_element_size("target.bst", project, element_path, [], 8000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
assert cli.get_element_state(project, "target.bst") == "cached"
# Now configure with a smaller quota, create a situation
# where the cache must be cleaned up before building anything else.
#
# Fix the fetchers and builders just to ensure a predictable
# sequence of events (although it does not effect this test)
cli.configure({
"cache": {
"quota": 5000000,
},
"scheduler": {
"fetchers": 1,
"builders": 1
},
})
# Our cache is now more than full, BuildStream
create_element_size("target2.bst", project, element_path, [], 4000000)
res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project, ["target.bst", "target2.bst"])
assert states["target.bst"] != "cached"
assert states["target2.bst"] == "cached"
def test_artifact_expires(cli, datafiles):
project = str(datafiles)
element_path = "elements"
# Skip this test if we do not have support for subsecond precision mtimes
#
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
cli.configure({"cache": {
"quota": 10000000,
}})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
create_element_size("target.bst", project, element_path, [], 6000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
assert cli.get_element_state(project, "target.bst") == "cached"
# Our cache should now be almost full. Let's create another
# artifact and see if we can cause buildstream to delete the old
# one.
create_element_size("target2.bst", project, element_path, [], 6000000)
res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project, ["target.bst", "target2.bst"])
assert states["target.bst"] != "cached"
assert states["target2.bst"] == "cached"
def test_never_delete_required(cli, datafiles):
project = str(datafiles)
element_path = "elements"
# Skip this test if we do not have support for subsecond precision mtimes
#
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
cli.configure({
"cache": {
"quota": 10000000
},
"scheduler": {
"fetchers": 1,
"builders": 1
}
})
# Create a linear build tree
create_element_size("dep1.bst", project, element_path, [], 8000000)
create_element_size("dep2.bst", project, element_path, ["dep1.bst"],
8000000)
create_element_size("dep3.bst", project, element_path, ["dep2.bst"],
8000000)
create_element_size("target.bst", project, element_path, ["dep3.bst"],
8000000)
# Build dep1.bst, which should fit into the cache.
res = cli.run(project=project, args=["build", "dep1.bst"])
res.assert_success()
# We try to build this pipeline, but it's too big for the
# cache. Since all elements are required, the build should fail.
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
res.assert_task_error(ErrorDomain.CAS, "cache-too-full")
states = cli.get_element_states(project, ["target.bst"])
assert states["dep1.bst"] == "cached"
assert states["dep2.bst"] != "cached"
assert states["dep3.bst"] != "cached"
assert states["target.bst"] != "cached"
def test_source_artifact_caches(cli, tmpdir, datafiles):
cachedir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
element_path = os.path.join(project_dir, "elements")
with create_artifact_share(os.path.join(str(tmpdir), "share")) as share:
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {
"pushers": 1
},
"source-caches": {
"url": share.repo,
"push": True,
},
"artifacts": {
"url": share.repo,
"push": True,
},
"cachedir": cachedir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
create_element_size("repo.bst", project_dir, element_path, [], 10000)
res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Pushed source " in res.stderr
assert "Pushed artifact " in res.stderr
# delete local sources and artifacts and check it pulls them
shutil.rmtree(os.path.join(cachedir, "cas"))
shutil.rmtree(os.path.join(cachedir, "sources"))
# this should just fetch the artifacts
res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Pulled artifact " in res.stderr
assert "Pulled source " not in res.stderr
def test_artifact_too_large(cli, datafiles, size):
project = str(datafiles)
element_path = "elements"
# Skip this test if we do not have support for subsecond precision mtimes
#
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project))
cli.configure({"cache": {"quota": 400000}})
# Create an element whose artifact is too large
create_element_size("target.bst", project, element_path, [], size)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
res.assert_task_error(ErrorDomain.CAS, "cache-too-full")
def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
project = str(datafiles)
element_path = "elements"
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Set a 22 MB quota
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share:
# Configure bst to push to the cache
cli.configure(
{"artifacts": {"url": share.repo, "push": True},}
)
# Create and build 2 elements, one 5 MB and one 15 MB.
create_element_size("element1.bst", project, element_path, [], int(5e6))
result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
create_element_size("element2.bst", project, element_path, [], int(15e6))
result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# Ensure they are cached locally
states = cli.get_element_states(project, ["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
# Ensure that they have been pushed to the cache
assert_shared(cli, share, project, "element1.bst")
assert_shared(cli, share, project, "element2.bst")
# Pull the element1 from the remote cache (this should update its mtime).
# Use a separate local cache for this to ensure the complete element is pulled.
cli2_path = os.path.join(str(tmpdir), "cli2")
os.mkdir(cli2_path)
cli2 = Cli(cli2_path)
result = cli2.run(project=project, args=["artifact", "pull", "element1.bst", "--remote", share.repo])
result.assert_success()
# Ensure element1 is cached locally
assert cli2.get_element_state(project, "element1.bst") == "cached"
wait_for_cache_granularity()
# Create and build the element3 (of 5 MB)
create_element_size("element3.bst", project, element_path, [], int(5e6))
result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Make sure it's cached locally and remotely
assert cli.get_element_state(project, "element3.bst") == "cached"
assert_shared(cli, share, project, "element3.bst")
# Ensure that element2 was deleted from the share and element1 remains
assert_not_shared(cli, share, project, "element2.bst")
assert_shared(cli, share, project, "element1.bst")
def test_artifact_expires(cli, datafiles, tmpdir):
project = str(datafiles)
element_path = "elements"
# Create an artifact share (remote artifact cache) in the tmpdir/artifactshare
# Set a 22 MB quota
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"),
quota=int(22e6)) as share:
# Configure bst to push to the cache
cli.configure(
{"artifacts": {
"servers": [
{
"url": share.repo,
"push": True
},
]
}})
# Create and build an element of 15 MB
create_element_size("element1.bst", project, element_path, [],
int(15e6))
result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
# Create and build an element of 5 MB
create_element_size("element2.bst", project, element_path, [],
int(5e6))
result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# check that element's 1 and 2 are cached both locally and remotely
states = cli.get_element_states(project,
["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
assert_shared(cli, share, project, "element1.bst")
assert_shared(cli, share, project, "element2.bst")
# Create and build another element of 5 MB (This will exceed the free disk space available)
create_element_size("element3.bst", project, element_path, [],
int(5e6))
result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Ensure it is cached both locally and remotely
assert cli.get_element_state(project, "element3.bst") == "cached"
assert_shared(cli, share, project, "element3.bst")
# Ensure element1 has been removed from the share
assert_not_shared(cli, share, project, "element1.bst")
# Ensure that elemen2 remains
assert_shared(cli, share, project, "element2.bst")
def test_keep_dependencies(cli, datafiles):
project = str(datafiles)
element_path = "elements"
# Skip this test if we do not have support for subsecond precision mtimes
#
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
cli.configure({"cache": {"quota": 10000000}})
# Create a pretty big dependency
create_element_size("dependency.bst", project, element_path, [], 5000000)
res = cli.run(project=project, args=["build", "dependency.bst"])
res.assert_success()
# Now create some other unrelated artifact
create_element_size("unrelated.bst", project, element_path, [], 4000000)
res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project,
["dependency.bst", "unrelated.bst"])
assert states["dependency.bst"] == "cached"
assert states["unrelated.bst"] == "cached"
# We try to build an element which depends on the LRU artifact,
# and could therefore fail if we didn't make sure dependencies
# aren't removed.
#
# Since some artifact caches may implement weak cache keys by
# duplicating artifacts (bad!) we need to make this equal in size
# or smaller than half the size of its dependencies.
#
create_element_size("target.bst", project, element_path,
["dependency.bst"], 2000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
states = cli.get_element_states(project, ["target.bst", "unrelated.bst"])
assert states["target.bst"] == "cached"
assert states["dependency.bst"] == "cached"
assert states["unrelated.bst"] != "cached"
def test_never_delete_required(cli, datafiles):
project = str(datafiles)
element_path = "elements"
cli.configure({
"cache": {
"quota": 10000000
},
"scheduler": {
"fetchers": 1,
"builders": 1
}
})
# Create a linear build tree
create_element_size("dep1.bst", project, element_path, [], 8000000)
create_element_size("dep2.bst", project, element_path, ["dep1.bst"],
8000000)
create_element_size("dep3.bst", project, element_path, ["dep2.bst"],
8000000)
create_element_size("target.bst", project, element_path, ["dep3.bst"],
8000000)
# Build dep1.bst, which should fit into the cache.
res = cli.run(project=project, args=["build", "dep1.bst"])
res.assert_success()
# We try to build this pipeline, but it's too big for the
# cache. Since all elements are required, the build should fail.
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
res.assert_task_error(ErrorDomain.CAS, "cache-too-full")
states = cli.get_element_states(project, ["target.bst"])
assert states["dep1.bst"] == "cached"
assert states["dep2.bst"] != "cached"
assert states["dep3.bst"] != "cached"
assert states["target.bst"] != "cached"
def test_keep_dependencies(cli, datafiles):
project = str(datafiles)
element_path = "elements"
cli.configure({"cache": {"quota": 10000000}})
# Create a pretty big dependency
create_element_size("dependency.bst", project, element_path, [], 5000000)
res = cli.run(project=project, args=["build", "dependency.bst"])
res.assert_success()
# Now create some other unrelated artifact
create_element_size("unrelated.bst", project, element_path, [], 4000000)
res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project,
["dependency.bst", "unrelated.bst"])
assert states["dependency.bst"] == "cached"
assert states["unrelated.bst"] == "cached"
# We try to build an element which depends on the LRU artifact,
# and could therefore fail if we didn't make sure dependencies
# aren't removed.
#
# Since some artifact caches may implement weak cache keys by
# duplicating artifacts (bad!) we need to make this equal in size
# or smaller than half the size of its dependencies.
#
create_element_size("target.bst", project, element_path,
["dependency.bst"], 2000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
states = cli.get_element_states(project, ["target.bst", "unrelated.bst"])
assert states["target.bst"] == "cached"
assert states["dependency.bst"] == "cached"
assert states["unrelated.bst"] != "cached"
def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
project = str(datafiles)
element_path = "elements"
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Set a 22 MB quota
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"),
quota=int(22e6)) as share:
# Configure bst to push to the cache
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}],
}})
# Create and build 2 elements, one 5 MB and one 15 MB.
create_element_size("element1.bst", project, element_path, [],
int(5e6))
result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
create_element_size("element2.bst", project, element_path, [],
int(15e6))
result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# Ensure they are cached locally
states = cli.get_element_states(project,
["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
# Ensure that they have been pushed to the cache
assert_shared(cli, share, project, "element1.bst")
assert_shared(cli, share, project, "element2.bst")
# Pull the element1 from the remote cache (this should update its mtime).
# Use a separate local cache for this to ensure the complete element is pulled.
cli2_path = os.path.join(str(tmpdir), "cli2")
cli2 = Cli(cli2_path)
result = cli2.run(project=project,
args=[
"artifact", "pull", "element1.bst",
"--artifact-remote", share.repo
])
result.assert_success()
# Ensure element1 is cached locally
assert cli2.get_element_state(project, "element1.bst") == "cached"
wait_for_cache_granularity()
# Create and build the element3 (of 5 MB)
create_element_size("element3.bst", project, element_path, [],
int(5e6))
result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Make sure it's cached locally and remotely
assert cli.get_element_state(project, "element3.bst") == "cached"
assert_shared(cli, share, project, "element3.bst")
# Ensure that element2 was deleted from the share and element1 remains
assert_not_shared(cli, share, project, "element2.bst")
assert_shared(cli, share, project, "element1.bst")
