def test_push_fails(cli, tmpdir, datafiles):
project = str(datafiles)
# Set up the share
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# Configure bst to be able to push to the share
cli.configure(
{"artifacts": {
"servers": [
{
"url": share.repo,
"push": True
},
]
}})
# First ensure that the target is *NOT* cache
assert cli.get_element_state(project, "target.bst") != "cached"
# Now try and push the target
result = cli.run(project=project,
args=["artifact", "push", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "Push failed: target.bst is not cached" in result.stderr
# Now ensure that deps are also not cached
assert cli.get_element_state(project, "import-bin.bst") != "cached"
assert cli.get_element_state(project, "import-dev.bst") != "cached"
assert cli.get_element_state(project, "compose-all.bst") != "cached"
def test_buildtree_pulled(cli, tmpdir, datafiles):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# Build the element to push it to cache
cli.configure({"artifacts": {"url": share.repo, "push": True}})
result = cli.run(
project=project,
args=["--cache-buildtrees", "always", "build", element_name])
result.assert_success()
assert cli.get_element_state(project, element_name) == "cached"
# Discard the cache
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
assert cli.get_element_state(project, element_name) != "cached"
# Pull from cache, ensuring cli options is set to pull the buildtree
result = cli.run(project=project,
args=[
"--pull-buildtrees", "artifact", "pull", "--deps",
"all", element_name
])
result.assert_success()
# Check it's using the cached build tree
res = cli.run(project=project,
args=[
"shell", "--build", element_name, "--use-buildtree",
"always", "--", "cat", "test"
])
res.assert_success()
def test_default_target_track(cli, tmpdir, datafiles):
project = str(datafiles)
project_path = os.path.join(project, "project.conf")
target = "track-fetch-test.bst"
# First, create an element with trackable sources
repo = create_repo("git", str(tmpdir))
repo.create(project)
element_conf = {"kind": "import", "sources": [repo.source_config()]}
_yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target))
# Then, make it the default target
project_conf = {
"name": "test-default-target",
"min-version": "2.0",
"element-path": "elements",
"defaults": {"targets": [target]},
}
_yaml.roundtrip_dump(project_conf, project_path)
# Setup finished. Track it now
assert cli.get_element_state(project, target) == "no reference"
result = cli.run(project=project, args=["source", "track"])
result.assert_success()
# Tracking will result in fetching it automatically, so we expect the state
# to be buildable.
assert cli.get_element_state(project, target) == "buildable"
def test_no_default_with_junction(cli, datafiles):
project = str(datafiles)
junction_path = os.path.join(project, "elements", "junction.bst")
target_path = os.path.join(project, "elements", "junction-target.bst")
# First, create a junction element to refer to the subproject
junction_config = {
"kind": "junction",
"sources": [{
"kind": "local",
"path": "files/sub-project",
}]
}
_yaml.roundtrip_dump(junction_config, junction_path)
# Then, create a stack element with dependency on cross junction element
target_config = {
"kind": "stack",
"depends": ["junction.bst:dummy_subproject.bst"]
}
_yaml.roundtrip_dump(target_config, target_path)
# Now try to perform a build
# This should automatically fetch the junction at load time.
result = cli.run(project=project, args=["build"])
result.assert_success()
assert cli.get_element_state(
project, "junction.bst:dummy_subproject.bst") == "cached"
assert cli.get_element_state(project, "junction-target.bst") == "cached"
def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind):
checkout = os.path.join(cli.directory, "checkout")
project = str(datafiles)
dev_files_path = os.path.join(project, "files", "dev-files")
element_path = os.path.join(project, "elements")
element_name = "build-test-{}.bst".format(kind)
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
repo = create_repo(kind, str(tmpdir))
ref = repo.create(dev_files_path)
# Write out our test target
element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
assert cli.get_element_state(project, element_name) == "fetch needed"
result = cli.run(project=project, args=strict_args(["build", element_name], strict))
result.assert_success()
assert cli.get_element_state(project, element_name) == "cached"
# Now check it out
result = cli.run(
project=project, args=strict_args(["artifact", "checkout", element_name, "--directory", checkout], strict)
)
result.assert_success()
# Check that the pony.h include from files/dev-files exists
filename = os.path.join(checkout, "usr", "include", "pony.h")
assert os.path.exists(filename)
def test_push_fails_with_on_error_continue(cli, tmpdir, datafiles):
project = str(datafiles)
# Set up the share
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target (and its deps)
result = cli.run(project=project, args=["build", "target.bst"])
assert cli.get_element_state(project, "target.bst") == "cached"
assert cli.get_element_state(project, "import-dev.bst") == "cached"
# Now delete the artifact of a dependency and ensure it is not in the cache
result = cli.run(project=project, args=["artifact", "delete", "import-dev.bst"])
assert cli.get_element_state(project, "import-dev.bst") != "cached"
# Configure bst to be able to push to the share
cli.configure({"artifacts": [{"url": share.repo, "push": True},]})
# Now try and push the target with its deps using --on-error continue
# and assert that push failed, but what could be pushed was pushed
result = cli.run(
project=project, args=["--on-error=continue", "artifact", "push", "--deps", "all", "target.bst"]
)
# The overall process should return as failed
result.assert_main_error(ErrorDomain.STREAM, None)
# We should still have pushed what we could
assert_shared(cli, share, project, "import-bin.bst")
assert_shared(cli, share, project, "compose-all.bst")
assert_shared(cli, share, project, "target.bst")
assert_not_shared(cli, share, project, "import-dev.bst")
assert "Push failed: import-dev.bst is not cached" in result.stderr
def test_fetch_gpg_verify(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
gpg_homedir = os.path.join(DATA_DIR, "gpghome")
# Create the repo from 'repofiles' subdir
repo = create_repo("ostree", str(tmpdir))
ref = repo.create(
os.path.join(project, "repofiles"),
gpg_sign="FFFF54C070353B52D046DEB087FA0F41A6EFD9E9",
gpg_homedir=gpg_homedir,
)
# Write out our test target
ostreesource = repo.source_config(ref=ref, gpg_key="test.gpg")
element = {"kind": "import", "sources": [ostreesource]}
_yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Assert that a fetch is needed
assert cli.get_element_state(project, "target.bst") == "fetch needed"
# Now try to fetch it
result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
# Assert that we are now buildable because the source is
# now cached.
assert cli.get_element_state(project, "target.bst") == "buildable"
def test_push_after_rebuild(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
generate_project(
project,
config={
"element-path": "elements",
"min-version": "2.0",
"plugins": [{"origin": "local", "path": "plugins", "elements": ["randomelement"]}],
},
)
# First build the element
result = cli.run(project=project, args=["build", "random.bst"])
result.assert_success()
assert cli.get_element_state(project, "random.bst") == "cached"
# Delete the artifact blobs but keep the artifact proto,
# i.e., now we have an incomplete artifact
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
assert cli.get_element_state(project, "random.bst") != "cached"
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
cli.configure({"artifacts": {"url": share.repo, "push": True}})
# Now rebuild the element and push it
result = cli.run(project=project, args=["build", "random.bst"])
result.assert_success()
assert result.get_pushed_elements() == ["random.bst"]
assert cli.get_element_state(project, "random.bst") == "cached"
def test_push_update_after_rebuild(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
generate_project(
project,
config={
"element-path": "elements",
"min-version": "2.0",
"plugins": [{"origin": "local", "path": "plugins", "elements": ["randomelement"]}],
},
)
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
cli.configure({"artifacts": {"url": share.repo, "push": True}})
# Build the element and push the artifact
result = cli.run(project=project, args=["build", "random.bst"])
result.assert_success()
assert result.get_pushed_elements() == ["random.bst"]
assert cli.get_element_state(project, "random.bst") == "cached"
# Now delete the artifact and ensure it is not in the cache
result = cli.run(project=project, args=["artifact", "delete", "random.bst"])
assert cli.get_element_state(project, "random.bst") != "cached"
# Now rebuild the element. Reset config to disable pulling.
cli.config = None
result = cli.run(project=project, args=["build", "random.bst"])
result.assert_success()
assert cli.get_element_state(project, "random.bst") == "cached"
# Push the new build
cli.configure({"artifacts": {"url": share.repo, "push": True}})
result = cli.run(project=project, args=["artifact", "push", "random.bst"])
assert result.get_pushed_elements() == ["random.bst"]
def test_default_target_push_pull(cli, tmpdir, datafiles):
project = str(datafiles)
project_path = os.path.join(project, "project.conf")
target = "dummy_1.bst"
# Set a default target
project_conf = {
"name": "test-default-target",
"min-version": "2.0",
"element-path": "elements",
"defaults": {"targets": [target]},
}
_yaml.roundtrip_dump(project_conf, project_path)
# Build the target
result = cli.run(project=project, args=["build"])
result.assert_success()
assert cli.get_element_state(project, target) == "cached"
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Push the artifacts
cli.configure({"artifacts": {"url": share.repo, "push": True}})
result = cli.run(project=project, args=["artifact", "push"])
result.assert_success()
# Delete local artifacts
# Note that `artifact delete` does not support default targets
result = cli.run(project=project, args=["artifact", "delete", target])
result.assert_success()
# Target should be buildable now, and we should be able to pull it
assert cli.get_element_state(project, target) == "buildable"
result = cli.run(project=project, args=["artifact", "pull"])
assert cli.get_element_state(project, target) == "cached"
def test_push_pull_non_strict(cli, tmpdir, datafiles):
project = str(datafiles)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the target element and push to the remote.
cli.configure({
"artifacts": {
"url": share.repo,
"push": True
},
"projects": {
"test": {
"strict": False
}
}
})
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
assert cli.get_element_state(project, "target.bst") == "cached"
# Assert that everything is now cached in the remote.
all_elements = [
"target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"
]
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
assert cli.get_element_state(project, element_name) != "cached"
# Add a file to force change in strict cache key of import-bin.bst
with open(
os.path.join(str(project), "files", "bin-files", "usr", "bin",
"world"), "w") as f:
f.write("world")
# Assert that the workspaced element requires a rebuild
assert cli.get_element_state(project, "import-bin.bst") == "buildable"
# Assert that the target is still waiting due to --no-strict
assert cli.get_element_state(project, "target.bst") == "waiting"
# Now try bst artifact pull
result = cli.run(
project=project,
args=["artifact", "pull", "--deps", "all", "target.bst"])
result.assert_success()
# And assert that the target is again in the local cache, without having built
assert cli.get_element_state(project, "target.bst") == "cached"
def test_push_pull_specific_remote(cli, tmpdir, datafiles):
project = str(datafiles)
with create_artifact_share(os.path.join(
str(tmpdir),
"goodartifactshare")) as good_share, create_artifact_share(
os.path.join(str(tmpdir), "badartifactshare")) as bad_share:
# Build the target so we have it cached locally only.
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
state = cli.get_element_state(project, "target.bst")
assert state == "cached"
# Configure the default push location to be bad_share; we will assert that
# nothing actually gets pushed there.
cli.configure({
"artifacts": {
"servers": [
{
"url": bad_share.repo,
"push": True
},
]
}
})
# Now try `bst artifact push` to the good_share.
result = cli.run(project=project,
args=[
"artifact", "push", "target.bst",
"--artifact-remote", good_share.repo
])
result.assert_success()
# Assert that all the artifacts are in the share we pushed
# to, and not the other.
assert_shared(cli, good_share, project, "target.bst")
assert_not_shared(cli, bad_share, project, "target.bst")
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the good_share.
#
casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
result = cli.run(project=project,
args=[
"artifact", "pull", "target.bst",
"--artifact-remote", good_share.repo
])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert cli.get_element_state(project, "target.bst") == "cached"
def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# Build the element to push it to cache
cli.configure({"artifacts": {"url": share.repo, "push": True}})
result = cli.run(
project=project,
args=["--cache-buildtrees", "always", "build", element_name])
result.assert_success()
assert cli.get_element_state(project, element_name) == "cached"
# Discard the cache
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
assert cli.get_element_state(project, element_name) != "cached"
# Check it's not using the cached build tree, because --pull
# and pull-buildtrees were not both set
res = cli.run(
project=project,
args=[
"shell",
"--build",
element_name,
"--pull",
"--use-buildtree",
"always",
"--",
"cat",
"test",
],
)
res.assert_main_error(ErrorDomain.APP, None)
assert "Artifact not cached locally. Can be retried with --pull and pull-buildtrees configured" in res.stderr
# Check it's using the cached build tree, because --pull
# and pull-buildtrees were both set
res = cli.run(
project=project,
args=[
"--pull-buildtrees",
"shell",
"--build",
element_name,
"--pull",
"--use-buildtree",
"always",
"--",
"cat",
"test",
],
)
result.assert_success()
assert "Hi" in res.output
def test_artifact_delete_element(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
# Build the element and ensure it's cached
result = cli.run(project=project, args=["build", element])
result.assert_success()
assert cli.get_element_state(project, element) == "cached"
result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
assert cli.get_element_state(project, element) != "cached"
def test_generate_key(cli, datafiles):
project_dir = str(datafiles)
# check that we don't fail if not tracking due to get_unique_key
res = cli.run(project=project_dir, args=["build", "key-test.bst"])
res.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline")
assert cli.get_element_state(project_dir, "key-test.bst") == "no reference"
res = cli.run(project=project_dir, args=["source", "track", "key-test.bst"])
res.assert_success()
assert cli.get_element_state(project_dir, "key-test.bst") == "fetch needed"
res = cli.run(project=project_dir, args=["build", "key-test.bst"])
res.assert_success()
assert cli.get_element_state(project_dir, "key-test.bst") == "cached"
def test_build_junction_short_notation_filename(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the content of /etc/animal.conf
filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
with open(filename, "r", encoding="utf-8") as f:
contents = f.read()
assert contents == "animal=Pony\n"
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
if ref_storage == "project.refs":
project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
_yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
assert cli.get_element_state(project, "junction-dep.bst") == "cached"
def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
dep = "compose-all.bst"
# Configure a local cache
local_cache = os.path.join(str(tmpdir), "cache")
cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
result = cli.run(project=project, args=["build", element])
result.assert_success()
assert cli.get_element_states(project, [element, dep], deps="none") == {
element: "cached",
dep: "cached",
}
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
assert os.path.exists(
os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the artifact
result = cli.run(project=project,
args=["artifact", "delete", artifact, dep])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact))
# Check that the dependency ELEMENT is no longer cached
assert cli.get_element_state(project, dep) != "cached"
def test_artifact_expires(cli, datafiles):
project = str(datafiles)
element_path = "elements"
cli.configure({"cache": {
"quota": 10000000,
}})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
create_element_size("target.bst", project, element_path, [], 6000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
assert cli.get_element_state(project, "target.bst") == "cached"
# Our cache should now be almost full. Let's create another
# artifact and see if we can cause buildstream to delete the old
# one.
create_element_size("target2.bst", project, element_path, [], 6000000)
res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project, ["target.bst", "target2.bst"])
assert states["target.bst"] != "cached"
assert states["target2.bst"] == "cached"
def test_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project,
config={"aliases": {
"tmpdir": "file:///" + str(tmpdir)
}})
assert cli.get_element_state(project, "target.bst") == "no reference"
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
repo = create_repo("git", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "pull-missing-local-blob",
"min-version": "2.0",
"element-path": "elements",
}
project_file = os.path.join(str(tmpdir), "project.conf")
_yaml.roundtrip_dump(project_config, project_file)
input_config = {
"kind": "import",
"sources": [repo.source_config()],
}
input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
depends_name = "depends.bst"
depends_config = {"kind": "stack", "depends": [input_name]}
depends_file = os.path.join(element_dir, depends_name)
_yaml.roundtrip_dump(depends_config, depends_file)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the import-bin element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
result = cli.run(project=project, args=["build", input_name])
result.assert_success()
assert cli.get_element_state(project, input_name) == "cached"
# Delete a file blob from the local cache.
# This is a placeholder to test partial CAS handling until we support
# partial artifact pulling (or blob-based CAS expiry).
#
digest = utils.sha256sum(
os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
objpath = os.path.join(cli.directory, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# Now try bst build
result = cli.run(project=project, args=["build", depends_name])
result.assert_success()
# Assert that the import-bin artifact was pulled (completing the partial artifact)
assert result.get_pulled_elements() == [input_name]
def test_push(cli, tmpdir, datafiles):
project_dir = str(datafiles)
# First build the project without the artifact cache configured
result = cli.run(project=project_dir, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
assert cli.get_element_state(project_dir, "target.bst") == "cached"
# Set up an artifact cache.
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure artifact share
rootcache_dir = os.path.join(str(tmpdir), "cache")
user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
"scheduler": {"pushers": 1},
"artifacts": {"servers": [{"url": share.repo, "push": True,}]},
"cachedir": rootcache_dir,
}
# Write down the user configuration file
_yaml.roundtrip_dump(user_config, file=user_config_file)
element_key = _push(cli, rootcache_dir, project_dir, user_config_file, "target.bst")
assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key))
def test_large_directory(cli, tmpdir, datafiles):
project = str(datafiles)
# Number of files chosen to ensure the complete list of digests exceeds
# our 1 MB gRPC message limit. I.e., test message splitting.
MAX_MESSAGE_LENGTH = 1024 * 1024
NUM_FILES = MAX_MESSAGE_LENGTH // 64 + 1
large_directory_dir = os.path.join(project, "files", "large-directory")
os.mkdir(large_directory_dir)
for i in range(NUM_FILES):
with open(os.path.join(large_directory_dir, str(i)), "w") as f:
# The files need to have different content as we want different digests.
f.write(str(i))
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure bst to push to the artifact share
cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True},]}})
# Enforce 1 MB gRPC message limit
with limit_grpc_message_length(MAX_MESSAGE_LENGTH):
# Build and push
result = cli.run(project=project, args=["build", "import-large-directory.bst"])
result.assert_success()
# Assert that we are now cached locally
assert cli.get_element_state(project, "import-large-directory.bst") == "cached"
# Assert that the push was successful
assert_shared(cli, share, project, "import-large-directory.bst")
def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles):
project = str(datafiles)
element = "target.bst"
# Set up remote and local shares
local_cache = os.path.join(str(tmpdir), "artifacts")
with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote:
cli.configure({
"artifacts": {
"servers": [{
"url": remote.repo,
"push": True
}]
},
"cachedir": local_cache,
})
# Build the element
result = cli.run(project=project, args=["build", element])
result.assert_success()
# Make sure it's in the share
assert remote.get_artifact(
cli.get_artifact_name(project, "test", element))
# Delete the artifact from the local cache
result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
assert cli.get_element_state(project, element) != "cached"
result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
assert "available {}".format(element) in result.output
def test_push_cross_junction(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
result = cli.run(project=project,
args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
assert cli.get_element_state(project,
"junction.bst:import-etc.bst") == "cached"
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}],
}})
cli.run(project=project,
args=["artifact", "push", "junction.bst:import-etc.bst"])
cache_key = cli.get_element_key(project, "junction.bst:import-etc.bst")
assert share.get_artifact(
cli.get_artifact_name(project,
"subtest",
"import-etc.bst",
cache_key=cache_key))
def test_cleanup_first(cli, datafiles):
project = str(datafiles)
element_path = "elements"
cli.configure({"cache": {"quota": 10000000,}})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
create_element_size("target.bst", project, element_path, [], 8000000)
res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
assert cli.get_element_state(project, "target.bst") == "cached"
# Now configure with a smaller quota, create a situation
# where the cache must be cleaned up before building anything else.
#
# Fix the fetchers and builders just to ensure a predictable
# sequence of events (although it does not effect this test)
cli.configure({"cache": {"quota": 5000000,}, "scheduler": {"fetchers": 1, "builders": 1}})
# Our cache is now more than full, BuildStream
create_element_size("target2.bst", project, element_path, [], 4000000)
res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
# Check that the correct element remains in the cache
states = cli.get_element_states(project, ["target.bst", "target2.bst"])
assert states["target.bst"] != "cached"
assert states["target2.bst"] == "cached"
def test_source_checkout_fetch(datafiles, cli):
project = str(datafiles)
checkout = os.path.join(cli.directory, "source-checkout")
target = "remote-import-dev.bst"
target_path = os.path.join(project, "elements", target)
# Create an element with remote source
element = generate_remote_import_element(
os.path.join(project, "files", "dev-files", "usr", "include",
"pony.h"), "pony.h")
_yaml.roundtrip_dump(element, target_path)
# Testing implicit fetching requires that we do not have the sources
# cached already
assert cli.get_element_state(project, target) == "fetch needed"
args = ["source", "checkout"]
args += [target, checkout]
result = cli.run(
project=project,
args=["source", "checkout", "--directory", checkout, target])
result.assert_success()
assert os.path.exists(os.path.join(checkout, "remote-import-dev",
"pony.h"))
def test_push_split(cli, tmpdir, datafiles):
project_dir = str(datafiles)
# First build the project without the artifact cache configured
result = cli.run(project=project_dir, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
assert cli.get_element_state(project_dir, "target.bst") == "cached"
indexshare = os.path.join(str(tmpdir), "indexshare")
storageshare = os.path.join(str(tmpdir), "storageshare")
# Set up an artifact cache.
with create_split_share(indexshare, storageshare) as (index, storage):
rootcache_dir = os.path.join(str(tmpdir), "cache")
user_config = {
"scheduler": {"pushers": 1},
"artifacts": {
"servers": [
{"url": index.repo, "push": True, "type": "index"},
{"url": storage.repo, "push": True, "type": "storage"},
],
},
"cachedir": rootcache_dir,
}
config_path = str(tmpdir.join("buildstream.conf"))
_yaml.roundtrip_dump(user_config, file=config_path)
element_key = _push(cli, rootcache_dir, project_dir, config_path, "target.bst")
proto = index.get_artifact_proto(
cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key)
)
assert storage.get_cas_files(proto) is not None
def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
buildbox_casd = tmp_path.joinpath("bin/buildbox-casd")
buildbox_casd.parent.mkdir()
os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd))
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
"kind": "script",
"depends": [{"filename": "base.bst", "type": "build",},],
"config": {"commands": ["true",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this will fail
result = cli.run(project=project, args=["build", "element.bst"], env={"PATH": str(tmp_path.joinpath("bin"))},)
# But if we dont spesify a sandbox then we fall back to dummy, we still
# fail early but only once we know we need a facny sandbox and that
# dumy is not enough, there for element gets fetched and so is buildable
result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox")
assert cli.get_element_state(project, "element.bst") == "buildable"
def test_push_after_pull(cli, tmpdir, datafiles):
project = str(datafiles)
# Set up two artifact shares.
with create_artifact_share(os.path.join(
str(tmpdir), "artifactshare1")) as share1, create_artifact_share(
os.path.join(str(tmpdir), "artifactshare2")) as share2:
# Set the scene: share1 has the artifact, share2 does not.
#
cli.configure(
{"artifacts": {
"servers": [
{
"url": share1.repo,
"push": True
},
]
}})
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
cli.remove_artifact_from_cache(project, "target.bst")
assert_shared(cli, share1, project, "target.bst")
assert_not_shared(cli, share2, project, "target.bst")
assert cli.get_element_state(project, "target.bst") != "cached"
# Now run the build again. Correct `bst build` behaviour is to download the
# artifact from share1 but not push it back again.
#
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
assert "target.bst" in result.get_pulled_elements()
assert "target.bst" not in result.get_pushed_elements()
# Delete the artifact locally again.
cli.remove_artifact_from_cache(project, "target.bst")
# Now we add share2 into the mix as a second push remote. This time,
# `bst build` should push to share2 after pulling from share1.
cli.configure({
"artifacts": {
"servers": [
{
"url": share1.repo,
"push": True
},
{
"url": share2.repo,
"push": True
},
]
}
})
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
assert "target.bst" in result.get_pulled_elements()
assert "target.bst" in result.get_pushed_elements()
