def test_compose_include(cli, datafiles, include_domains, exclude_domains,
expected):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_path = os.path.join(project, "elements")
element_name = "compose/compose-amhello.bst"
# Create a yaml configuration from the specified include and
# exclude domains
config = {"include": include_domains, "exclude": exclude_domains}
create_compose_element(element_name, element_path, config=config)
result = cli.run(project=project,
args=["source", "track", "compose/amhello.bst"])
assert result.exit_code == 0
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
result = cli.run(
project=project,
args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
assert set(walk_dir(checkout)) == set(expected)
def test_disable_message_lines(cli, datafiles):
project = str(datafiles)
element_path = os.path.join(project, "elements")
element_name = "message.bst"
element = {
"kind": "manual",
"depends": [{
"filename": "base.bst"
}],
"config": {
"build-commands": ['echo "Silly message"'],
"strip-commands": []
},
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)),
exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# First we check that we get the "Silly message"
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
assert 'echo "Silly message"' in result.stderr
# Let's now build it again, but with --message-lines 0
cli.remove_artifact_from_cache(project, element_name)
result = cli.run(project=project,
args=["--message-lines", "0", "build", element_name])
result.assert_success()
assert "Message contains " not in result.stderr
def test_autotools_build(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
# Check that the project can be built correctly.
result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
result = cli.run(
project=project,
args=["artifact", "checkout", "hello.bst", "--directory", checkout])
result.assert_success()
assert_contains(
checkout,
[
"/usr",
"/usr/lib",
"/usr/bin",
"/usr/share",
"/usr/bin/hello",
"/usr/share/doc",
"/usr/share/doc/amhello",
"/usr/share/doc/amhello/README",
],
)
def test_disable_error_lines(cli, datafiles):
project = str(datafiles)
element_path = os.path.join(project, "elements")
element_name = "message.bst"
element = {
"kind": "manual",
"depends": [{
"filename": "base.bst"
}],
"config": {
"build-commands": ["This is a syntax error > >"],
"strip-commands": []
},
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)),
exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# First we check that we get the syntax error
result = cli.run(project=project,
args=["--error-lines", "0", "build", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "This is a syntax error" in result.stderr
# Let's now build it again, but with --error-lines 0
cli.remove_artifact_from_cache(project, element_name)
result = cli.run(project=project,
args=["--error-lines", "0", "build", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "Printing the last" not in result.stderr
def test_workspace_commanddir(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_name = "workspace/workspace-commanddir.bst"
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Check that the object file was created in the command-subdir `build`
# using the cached buildtree.
res = cli.run(
project=project,
args=[
"shell",
"--build",
element_name,
"--use-buildtree",
"--",
"find",
"..",
"-mindepth",
"1",
],
)
res.assert_success()
files = res.output.splitlines()
assert "../build/hello.o" in files
def get_value_for_mask(mask):
checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(mask))
create_test_file(sourcedir, "a.txt", mode=0o644 & mask)
create_test_file(sourcedir, "b.txt", mode=0o755 & mask)
create_test_file(sourcedir, "c.txt", mode=0o4755 & mask)
create_test_file(sourcedir, "d.txt", mode=0o2755 & mask)
create_test_file(sourcedir, "e.txt", mode=0o1755 & mask)
create_test_directory(sourcedir, "dir-a", mode=0o0755 & mask)
create_test_directory(sourcedir, "dir-b", mode=0o4755 & mask)
create_test_directory(sourcedir, "dir-c", mode=0o2755 & mask)
create_test_directory(sourcedir, "dir-d", mode=0o1755 & mask)
try:
test_values = []
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
result = cli.run(project=project,
args=[
"artifact", "checkout", element_name,
"--directory", checkoutdir
])
result.assert_success()
with open(os.path.join(checkoutdir, "ls-l"), "r",
encoding="utf-8") as f:
for line in f.readlines():
test_values.append(line.split()[0] + " " +
line.split()[-1])
return test_values
finally:
cli.remove_artifact_from_cache(project, element_name)
def test_script_cwd(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_path = os.path.join(project, "elements")
element_name = "script/script-layout.bst"
create_script_element(
element_name,
element_path,
config={
"commands": ["echo 'test' > test", "cp /buildstream/test %{install-root}"],
},
variables={"cwd": "/buildstream"},
)
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
with open(os.path.join(checkout, "test"), encoding="utf-8") as f:
text = f.read()
assert text == "test\n"
def test_script_root(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_path = os.path.join(project, "elements")
element_name = "script/script-layout.bst"
create_script_element(
element_name,
element_path,
config={
# Root-read only is False by default, we
# want to check the default here
# 'root-read-only': False,
"commands": ["mkdir -p %{install-root}", "echo 'I can write to root' > /test", "cp /test %{install-root}"],
},
)
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
with open(os.path.join(checkout, "test"), encoding="utf-8") as f:
text = f.read()
assert text == "I can write to root\n"
def test_regression_cache_corruption_2(cli, datafiles):
project = str(datafiles)
checkout_original = os.path.join(cli.directory, "checkout-original")
checkout_after = os.path.join(cli.directory, "checkout-after")
element_name = "script/corruption-2.bst"
canary_element_name = "script/corruption-image.bst"
res = cli.run(project=project, args=["build", canary_element_name])
assert res.exit_code == 0
res = cli.run(
project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original]
)
assert res.exit_code == 0
with open(os.path.join(checkout_original, "canary"), encoding="utf-8") as f:
assert f.read() == "alive\n"
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after])
assert res.exit_code == 0
with open(os.path.join(checkout_after, "canary"), encoding="utf-8") as f:
assert f.read() == "alive\n"
def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
tmpdir = tmpdir_factory.mktemp(os.path.basename(__file__))
project = str(datafiles)
element_name = "autotools/amhello.bst"
workspace_dir = os.path.join(str(tmpdir), "workspace")
if guess_element:
# Mutate the project.conf to use a default shell command
project_file = os.path.join(project, "project.conf")
config_text = "shell:\n command: ['true']\n"
with open(project_file, "a", encoding="utf-8") as f:
f.write(config_text)
result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, element_name])
result.assert_success()
result = cli.run(project=project, args=["-C", workspace_dir, "build", element_name])
result.assert_success()
command = ["-C", workspace_dir, "shell"]
if build_shell == "build":
command.append("--build")
if not guess_element:
command.extend([element_name, "--", "true"])
result = cli.run(project=project, cwd=workspace_dir, args=command)
result.assert_success()
def test_build_shell_fetch(cli, datafiles):
project = str(datafiles)
element_name = "build-shell-fetch.bst"
# Create a file with unique contents such that it cannot be in the cache already
test_filepath = os.path.join(project, "files", "hello.txt")
test_message = "Hello World! {}".format(uuid.uuid4())
with open(test_filepath, "w", encoding="utf-8") as f:
f.write(test_message)
checksum = utils.sha256sum(test_filepath)
# Create an element that has this unique file as a source
element = {
"kind": "manual",
"depends": ["base.bst"],
"sources": [{"kind": "remote", "url": "project_dir:/files/hello.txt", "ref": checksum}],
}
_yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name))
# Ensure our dependencies are cached
result = cli.run(project=project, args=["build", "base.bst"])
result.assert_success()
# Ensure our sources are not cached
assert cli.get_element_state(project, element_name) == "fetch needed"
# Launching a shell should fetch any uncached sources
result = cli.run(project=project, args=["shell", "--build", element_name, "cat", "hello.txt"])
result.assert_success()
assert result.output == test_message
def test_workspace_visible(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_name = "workspace/workspace-mount-fail.bst"
# Open a workspace on our build failing element
#
res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# Ensure the dependencies of our build failing element are built
result = cli.run(project=project, args=["build", "base.bst"])
assert result.exit_code == 0
# Obtain a copy of the hello.c content from the workspace
#
workspace_hello_path = os.path.join(cli.directory, "workspace", "hello.c")
assert os.path.exists(workspace_hello_path)
with open(workspace_hello_path, "r", encoding="utf-8") as f:
workspace_hello = f.read()
# Cat the hello.c file from a bst shell command, and assert
# that we got the same content here
#
result = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "hello.c"])
assert result.exit_code == 0
assert result.output == workspace_hello
def test_workspace_failed_logs(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "failing_amhello")
element_name = "autotools/amhello-failure.bst"
# Open workspace
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
res.assert_success()
# Try to build and ensure the build fails
res = cli.run(project=project, args=["build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
assert cli.get_element_state(project, element_name) == "failed"
res = cli.run(project=project, args=["artifact", "log", element_name])
res.assert_success()
log = res.output
# Assert that we can get the log
assert log != ""
fail_str = "FAILURE {}: Running build-commands".format(element_name)
batch_fail_str = "FAILURE {}: Running commands".format(element_name)
assert fail_str in log or batch_fail_str in log
def test_autotools_build(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_name = "autotools/amhello.bst"
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
assert_contains(
checkout,
[
"/usr",
"/usr/lib",
"/usr/bin",
"/usr/share",
"/usr/bin/hello",
"/usr/share/doc",
"/usr/share/doc/amhello",
"/usr/share/doc/amhello/README",
],
)
# Check the log
result = cli.run(project=project, args=["artifact", "log", element_name])
assert result.exit_code == 0
log = result.output
# Verify we get expected output exactly once
assert log.count("Making all in src") == 1
def test_run_unmodified_hello(cli, datafiles):
project = str(datafiles)
result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
result = cli.run(project=project, args=["shell", "hello.bst", "hello"])
result.assert_success()
assert result.output == "Hello World\n"
def test_running_commands_run(cli, datafiles):
project = str(datafiles)
result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"])
assert result.exit_code == 0
assert result.output == "Hello World\n"
def test_autotools_run(cli, datafiles):
project = str(datafiles)
result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
result = cli.run(project=project, args=["shell", "hello.bst", "hello"])
result.assert_success()
assert result.output == "Hello World!\nThis is amhello 1.0.\n"
def test_integration_commands_run(cli, datafiles):
project = str(datafiles)
result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
result = cli.run(project=project,
args=["shell", "hello.bst", "--", "hello", "pony"])
assert result.exit_code == 0
assert result.output == "Hello pony\n"
def test_autotools_run(cli, datafiles):
project = str(datafiles)
element_name = "autotools/amhello.bst"
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
assert result.exit_code == 0
assert result.output == "Hello World!\nThis is amhello 1.0.\n"
def test_build_depend_on_cached_fail(cli, datafiles):
project = str(datafiles)
dep_path = os.path.join(project, "elements", "dep.bst")
target_path = os.path.join(project, "elements", "target.bst")
dep = {
"kind": "script",
"depends": [
{
"filename": "base.bst",
"type": "build",
},
],
"config": {
"commands": [
"touch %{install-root}/foo",
"false",
],
},
}
_yaml.roundtrip_dump(dep, dep_path)
target = {
"kind":
"script",
"depends": [
{
"filename": "base.bst",
"type": "build",
},
{
"filename": "dep.bst",
"type": "build",
},
],
"config": {
"commands": [
"test -e /foo",
],
},
}
_yaml.roundtrip_dump(target, target_path)
# Try to build it, this should result in caching a failure to build dep
result = cli.run(project=project, args=["build", "dep.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's cached in a failed artifact
assert cli.get_element_state(project, "dep.bst") == "failed"
# Now we should fail because we've a cached fail of dep
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's not yet built, since one of its dependencies isn't ready.
assert cli.get_element_state(project, "target.bst") == "waiting"
def test_workspace_updated_dependency(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_path = os.path.join(project, "elements")
element_name = "workspace/workspace-updated-dependency.bst"
dep_name = "workspace/dependency.bst"
dependency = {
"kind": "manual",
"depends": [{
"filename": "base.bst",
"type": "build"
}],
"config": {
"build-commands": [
"mkdir -p %{install-root}/etc/test/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
]
},
}
os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)),
exist_ok=True)
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# First open the workspace
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# We build the workspaced element, so that we have an artifact
# with specific built dependencies
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Now we update a dependency of our element.
dependency["config"]["build-commands"] = [
"mkdir -p %{install-root}/etc/test/",
'echo "Hello china!" > %{install-root}/etc/test/hello.txt',
]
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# `Make` would look at timestamps and normally not realize that
# our dependency's header files changed. BuildStream must
# therefore ensure that we change the mtimes of any files touched
# since the last successful build of this element, otherwise this
# build will fail.
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
res = cli.run(project=project,
args=["shell", element_name, "/usr/bin/test.sh"])
assert res.exit_code == 0
assert res.output == "Hello china!\n\n"
def test_shell_call_hello(cli, datafiles):
project = str(datafiles)
result = cli.run(project=project, args=["build", "callHello.bst"])
result.assert_success()
result = cli.run(
project=project,
args=["shell", "callHello.bst", "--", "/bin/sh", "callHello.sh"])
result.assert_success()
assert result.output == "Calling hello:\nHello World!\nThis is amhello 1.0.\n"
def test_symlink_in_sandbox_path(cli, datafiles):
project = str(datafiles)
element_name = "symlinks/link-on-path-use.bst"
base_element_name = "symlinks/link-on-path.bst"
# This test is inspired by how freedesktop-SDK has /bin -> /usr/bin
# Create a element that has sh in altbin and a link from bin to altbin
result1 = cli.run(project=project, args=["build", base_element_name])
result1.assert_success()
# Build a element that uses the element that has sh in altbin.
result2 = cli.run(project=project, args=["build", element_name])
result2.assert_success()
def test_workspace_stages_once(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_name = "workspace/workspace-mount.bst"
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
assert cli.get_element_key(project, element_name) != "{:?<64}".format("")
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
def test_updated_dependency_nested(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_path = os.path.join(project, "elements")
element_name = "workspace/workspace-updated-dependency-nested.bst"
dep_name = "workspace/dependency.bst"
dependency = {
"kind": "manual",
"depends": [{
"filename": "base.bst",
"type": "build"
}],
"config": {
"build-commands": [
"mkdir -p %{install-root}/etc/test/tests/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
'echo "Hello brazil!" > %{install-root}/etc/test/tests/brazil.txt',
]
},
}
os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)),
exist_ok=True)
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# First open the workspace
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# We build the workspaced element, so that we have an artifact
# with specific built dependencies
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Now we update a dependency of our element.
dependency["config"]["build-commands"] = [
"mkdir -p %{install-root}/etc/test/tests/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt',
]
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Buildstream should pick up the newly added element, and pick up
# the lack of the newly removed element
res = cli.run(project=project,
args=["shell", element_name, "/usr/bin/test.sh"])
assert res.exit_code == 0
assert res.output == "Hello world!\nHello test!\n\n"
def test_first_project_build_checkout(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
result = cli.run(
project=project,
args=["artifact", "checkout", "hello.bst", "--directory", checkout])
assert result.exit_code == 0
assert_contains(checkout, ["/hello.world"])
def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_name = "symlinks/dangling-symlink-overlap.bst"
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
result = cli.run(
project=project,
args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == -1
assert "Destination is a symlink, not a directory: /opt/orgname" in result.stderr
def test_manual_command_subdir(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_path = os.path.join(project, "elements")
element_name = "manual/command-subdir.bst"
sources = [{"kind": "local", "path": "files/manual-element/root"}]
create_manual_element(
element_name,
element_path,
{"install-commands": ["cp hello %{install-root}"]},
{},
{},
sources=sources,
)
# First, verify that element builds, and has the correct expected output.
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
result = cli.run(
project=project,
args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
with open(os.path.join(checkout, "hello"), encoding="utf-8") as f:
assert f.read() == "hello from root\n"
# Now, change element configuration to have a different command-subdir.
# This should result in a different cache key.
create_manual_element(
element_name,
element_path,
{"install-commands": ["cp hello %{install-root}"]},
{"command-subdir": "subdir"},
{},
sources=sources,
)
# Verify that the element needs to be rebuilt.
assert cli.get_element_state(project, element_name) == "buildable"
# Finally, ensure that the variable actually takes effect.
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
shutil.rmtree(checkout)
result = cli.run(
project=project,
args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
with open(os.path.join(checkout, "hello"), encoding="utf-8") as f:
assert f.read() == "hello from subdir\n"
def test_workspace_mount(cli, datafiles):
project = str(datafiles)
workspace = os.path.join(cli.directory, "workspace")
element_name = "workspace/workspace-mount.bst"
res = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
assert os.path.exists(os.path.join(cli.directory, "workspace"))
def test_script_layout(cli, datafiles):
project = str(datafiles)
checkout = os.path.join(cli.directory, "checkout")
element_name = "script/script-layout.bst"
res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
with open(os.path.join(checkout, "test"), encoding="utf-8") as f:
text = f.read()
assert text == "Hi\n"