def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
project = str(datafiles)
element_path = "elements"
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Set a 22 MB quota
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share:
# Configure bst to push to the cache
cli.configure(
{"artifacts": {"url": share.repo, "push": True},}
)
# Create and build 2 elements, one 5 MB and one 15 MB.
create_element_size("element1.bst", project, element_path, [], int(5e6))
result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
create_element_size("element2.bst", project, element_path, [], int(15e6))
result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# Ensure they are cached locally
states = cli.get_element_states(project, ["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
# Ensure that they have been pushed to the cache
assert_shared(cli, share, project, "element1.bst")
assert_shared(cli, share, project, "element2.bst")
# Pull the element1 from the remote cache (this should update its mtime).
# Use a separate local cache for this to ensure the complete element is pulled.
cli2_path = os.path.join(str(tmpdir), "cli2")
os.mkdir(cli2_path)
cli2 = Cli(cli2_path)
result = cli2.run(project=project, args=["artifact", "pull", "element1.bst", "--remote", share.repo])
result.assert_success()
# Ensure element1 is cached locally
assert cli2.get_element_state(project, "element1.bst") == "cached"
wait_for_cache_granularity()
# Create and build the element3 (of 5 MB)
create_element_size("element3.bst", project, element_path, [], int(5e6))
result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Make sure it's cached locally and remotely
assert cli.get_element_state(project, "element3.bst") == "cached"
assert_shared(cli, share, project, "element3.bst")
# Ensure that element2 was deleted from the share and element1 remains
assert_not_shared(cli, share, project, "element2.bst")
assert_shared(cli, share, project, "element1.bst")
def test_expiry_order(cli, datafiles):
project = str(datafiles)
element_path = "elements"
checkout = os.path.join(project, "workspace")
cli.configure({"cache": {"quota": 9000000}})
# Create an artifact
create_element_size("dep.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "dep.bst"])
res.assert_success()
# Create another artifact
create_element_size("unrelated.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# And build something else
create_element_size("target.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
create_element_size("target2.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
wait_for_cache_granularity()
# Now extract dep.bst
res = cli.run(
project=project,
args=["artifact", "checkout", "dep.bst", "--directory", checkout])
res.assert_success()
# Finally, build something that will cause the cache to overflow
create_element_size("expire.bst", project, element_path, [], 2000000)
res = cli.run(project=project, args=["build", "expire.bst"])
res.assert_success()
# While dep.bst was the first element to be created, it should not
# have been removed.
# Note that buildstream will reduce the cache to 50% of the
# original size - we therefore remove multiple elements.
check_elements = [
"unrelated.bst", "target.bst", "target2.bst", "dep.bst", "expire.bst"
]
states = cli.get_element_states(project, check_elements)
assert tuple(states[element] for element in check_elements) == (
"buildable",
"buildable",
"buildable",
"cached",
"cached",
)
def test_incremental_partial(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_path = os.path.join(project, "elements")
element_name = "workspace/incremental.bst"
element = {
"kind": "manual",
"depends": [{
"filename": "base.bst",
"type": "build"
}],
"sources": [{
"kind": "local",
"path": "files/workspace-partial"
}],
"config": {
"build-commands": ["make random", "make copy1", "make copy2"]
},
}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# We open a workspace on the above element
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
res.assert_success()
# Initial (non-incremental) build of the workspace
res = cli.run(project=project, args=["build", element_name])
res.assert_success()
# Save the random hash
random_hash = get_buildtree_file_contents(cli, project, element_name,
"random")
# Verify the expected output files of the initial build
assert get_buildtree_file_contents(cli, project, element_name,
"copy1") == "1"
assert get_buildtree_file_contents(cli, project, element_name,
"copy2") == "1"
wait_for_cache_granularity()
# Delete source1 and replace source2 file contents with '2'
os.unlink(os.path.join(workspace, "source1"))
with open(os.path.join(workspace, "source2"), "w", encoding="utf-8") as f:
f.write("2")
# Perform incremental build of the workspace
# This should fail because of the missing source1 file.
res = cli.run(project=project, args=["build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
wait_for_cache_granularity()
# Recreate source1 file
with open(os.path.join(workspace, "source1"), "w", encoding="utf-8") as f:
f.write("2")
# Perform incremental build of the workspace
res = cli.run(project=project, args=["build", element_name])
res.assert_success()
# Verify that this was an incremental build by comparing the random hash
assert get_buildtree_file_contents(cli, project, element_name,
"random") == random_hash
# Verify that both files got rebuilt
assert get_buildtree_file_contents(cli, project, element_name,
"copy1") == "2"
assert get_buildtree_file_contents(cli, project, element_name,
"copy2") == "2"
def test_incremental(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_path = os.path.join(project, "elements")
element_name = "workspace/incremental.bst"
element = {
"kind": "manual",
"depends": [{
"filename": "base.bst",
"type": "build"
}],
"sources": [{
"kind": "local",
"path": "files/workspace-incremental"
}],
"config": {
"build-commands": ["make"]
},
}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# We open a workspace on the above element
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
res.assert_success()
# Initial (non-incremental) build of the workspace
res = cli.run(project=project, args=["build", element_name])
res.assert_success()
# Save the random hash
random_hash = get_buildtree_file_contents(cli, project, element_name,
"random")
# Verify the expected output file of the initial build
assert get_buildtree_file_contents(cli, project, element_name,
"copy") == "1"
wait_for_cache_granularity()
# Replace source file contents with '2'
with open(os.path.join(workspace, "source"), "w", encoding="utf-8") as f:
f.write("2")
# Perform incremental build of the workspace
res = cli.run(project=project, args=["build", element_name])
res.assert_success()
# Verify that this was an incremental build by comparing the random hash
assert get_buildtree_file_contents(cli, project, element_name,
"random") == random_hash
# Verify that the output file matches the new source file
assert get_buildtree_file_contents(cli, project, element_name,
"copy") == "2"
wait_for_cache_granularity()
# Replace source file contents with '3', however, set an old mtime such
# that `make` will not pick up the change
with open(os.path.join(workspace, "source"), "w", encoding="utf-8") as f:
f.write("3")
os.utime(os.path.join(workspace, "source"),
(BST_ARBITRARY_TIMESTAMP, BST_ARBITRARY_TIMESTAMP))
# Perform incremental build of the workspace
res = cli.run(project=project, args=["build", element_name])
res.assert_success()
# Verify that this was an incremental build by comparing the random hash
assert get_buildtree_file_contents(cli, project, element_name,
"random") == random_hash
# Verify that the output file still matches the previous content '2'
assert get_buildtree_file_contents(cli, project, element_name,
"copy") == "2"
def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
project = str(datafiles)
element_path = "elements"
# The artifact expiry logic relies on mtime changes, in real life second precision
# should be enough for this to work almost all the time, but test cases happen very
# quickly, resulting in all artifacts having the same mtime.
#
# This test requires subsecond mtime to be reliable.
#
if not have_subsecond_mtime(project):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
project))
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Set a 22 MB quota
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"),
quota=int(22e6)) as share:
# Configure bst to push to the cache
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}],
}})
# Create and build 2 elements, one 5 MB and one 15 MB.
create_element_size("element1.bst", project, element_path, [],
int(5e6))
result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
create_element_size("element2.bst", project, element_path, [],
int(15e6))
result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# Ensure they are cached locally
states = cli.get_element_states(project,
["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
# Ensure that they have been pushed to the cache
assert_shared(cli, share, project, "element1.bst")
assert_shared(cli, share, project, "element2.bst")
# Pull the element1 from the remote cache (this should update its mtime).
# Use a separate local cache for this to ensure the complete element is pulled.
cli2_path = os.path.join(str(tmpdir), "cli2")
cli2 = Cli(cli2_path)
result = cli2.run(project=project,
args=[
"artifact", "pull", "element1.bst",
"--artifact-remote", share.repo
])
result.assert_success()
# Ensure element1 is cached locally
assert cli2.get_element_state(project, "element1.bst") == "cached"
wait_for_cache_granularity()
# Create and build the element3 (of 5 MB)
create_element_size("element3.bst", project, element_path, [],
int(5e6))
result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Make sure it's cached locally and remotely
assert cli.get_element_state(project, "element3.bst") == "cached"
assert_shared(cli, share, project, "element3.bst")
# Ensure that element2 was deleted from the share and element1 remains
assert_not_shared(cli, share, project, "element2.bst")
assert_shared(cli, share, project, "element1.bst")
