def test_build_depend_on_cached_fail(cli, datafiles): project = str(datafiles) dep_path = os.path.join(project, "elements", "dep.bst") target_path = os.path.join(project, "elements", "target.bst") dep = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build",},], "config": {"commands": ["touch %{install-root}/foo", "false",],}, } _yaml.roundtrip_dump(dep, dep_path) target = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build",}, {"filename": "dep.bst", "type": "build",},], "config": {"commands": ["test -e /foo",],}, } _yaml.roundtrip_dump(target, target_path) # Try to build it, this should result in caching a failure to build dep result = cli.run(project=project, args=["build", "dep.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "dep.bst") == "failed" # Now we should fail because we've a cached fail of dep result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's not yet built, since one of its dependencies isn't ready. assert cli.get_element_state(project, "target.bst") == "waiting"
def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path): # Create symlink to buildbox-casd to work with custom PATH buildbox_casd = tmp_path.joinpath("bin/buildbox-casd") buildbox_casd.parent.mkdir() os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd)) project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build",},], "config": {"commands": ["true",],}, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this will fail result1 = cli.run(project=project, args=["build", "element.bst"], env={"PATH": str(tmp_path.joinpath("bin"))},) result1.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") assert cli.get_element_state(project, "element.bst") == "buildable" # When rebuilding, this should work result2 = cli.run(project=project, args=["build", "element.bst"]) result2.assert_success() assert cli.get_element_state(project, "element.bst") == "cached"
def test_build_shell_fetch(cli, datafiles): project = str(datafiles) element_name = "build-shell-fetch.bst" # Create a file with unique contents such that it cannot be in the cache already test_filepath = os.path.join(project, "files", "hello.txt") test_message = "Hello World! {}".format(uuid.uuid4()) with open(test_filepath, "w") as f: f.write(test_message) checksum = utils.sha256sum(test_filepath) # Create an element that has this unique file as a source element = { "kind": "manual", "depends": ["base.bst"], "sources": [{"kind": "remote", "url": "project_dir:/files/hello.txt", "ref": checksum}], } _yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name)) # Ensure our dependencies are cached result = cli.run(project=project, args=["build", "base.bst"]) result.assert_success() # Ensure our sources are not cached assert cli.get_element_state(project, element_name) == "fetch needed" # Launching a shell should fetch any uncached sources result = cli.run(project=project, args=["shell", "--build", element_name, "cat", "hello.txt"]) result.assert_success() assert result.output == test_message
def test_workspace_failed_logs(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "failing_amhello") element_name = "autotools/amhello-failure.bst" # Open workspace res = cli.run( project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Try to build and ensure the build fails res = cli.run(project=project, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) assert cli.get_element_state(project, element_name) == "failed" res = cli.run(project=project, args=["artifact", "log", element_name]) res.assert_success() log = res.output # Assert that we can get the log assert log != "" fail_str = "FAILURE {}: Running build-commands".format(element_name) batch_fail_str = "FAILURE {}: Running commands".format(element_name) assert fail_str in log or batch_fail_str in log
def test_build_checkout_cached_fail(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") checkout = os.path.join(cli.directory, "checkout") # Write out our test target element = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build",},], "config": {"commands": ["touch %{install-root}/foo", "false",],}, } _yaml.roundtrip_dump(element, element_path) # Try to build it, this should result in a failure that contains the content result = cli.run(project=project, args=["build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "element.bst") == "failed" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout]) result.assert_success() # Check that the checkout contains the file created before failure filename = os.path.join(checkout, "foo") assert os.path.exists(filename)
def test_push_cached_fail(cli, tmpdir, datafiles, on_error): if on_error == "quit": pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534") project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build",},], "config": { "commands": [ "false", # Ensure unique cache key for different test variants 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( {"artifacts": {"url": share.repo, "push": True},} ) # Build the element, continuing to finish active jobs on error. result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst"))
def test_manual_command_subdir(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "manual/command-subdir.bst" sources = [{"kind": "local", "path": "files/manual-element/root"}] create_manual_element( element_name, element_path, {"install-commands": ["cp hello %{install-root}"]}, {}, {}, sources=sources, ) # First, verify that element builds, and has the correct expected output. result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run( project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() with open(os.path.join(checkout, "hello")) as f: assert f.read() == "hello from root\n" # Now, change element configuration to have a different command-subdir. # This should result in a different cache key. create_manual_element( element_name, element_path, {"install-commands": ["cp hello %{install-root}"]}, {"command-subdir": "subdir"}, {}, sources=sources, ) # Verify that the element needs to be rebuilt. assert cli.get_element_state(project, element_name) == "buildable" # Finally, ensure that the variable actually takes effect. result = cli.run(project=project, args=["build", element_name]) result.assert_success() shutil.rmtree(checkout) result = cli.run( project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() with open(os.path.join(checkout, "hello")) as f: assert f.read() == "hello from subdir\n"
def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error): """Test that we can upload a built artifact that didn't have a valid shell inside. When we don't have a valid shell, the artifact will be empty, not even the root directory. This ensures we handle the case of an entirely empty artifact correctly. """ if on_error == "quit": pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534") project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "config": { "commands": [ "false", # Ensure unique cache key for different test variants 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }] }}) # Build the element, continuing to finish active jobs on error. result = cli.run( project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote assert share.get_artifact( cli.get_artifact_name(project, "test", "element.bst"))
def test_cache_buildtrees(cli, tmpdir, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing with create_artifact_share(os.path.join( str(tmpdir), "share1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "share2")) as share2, create_artifact_share( os.path.join(str(tmpdir), "share3")) as share3: cli.configure({ "artifacts": { "url": share1.repo, "push": True }, "cachedir": str(tmpdir) }) # Build autotools element with the default behavior of caching buildtrees # only when necessary. The artifact should be successfully pushed to the share1 remote # and cached locally with an 'empty' buildtree digest, as it's not a # dangling ref result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share1.get_artifact( cli.get_artifact_name(project, "test", element_name)) # The buildtree dir should not exist, as we set the config to not cache buildtrees. artifact_name = cli.get_artifact_name(project, "test", element_name) assert share1.get_artifact(artifact_name) with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees # that is was cached in share1 as expected without a buildtree dir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) assert cli.get_element_state(project, element_name) != "cached" result = cli.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Assert that the default behaviour of pull to not include buildtrees on the artifact # in share1 which was purposely cached with an empty one behaves as expected. As such the # pulled artifact will have a dangling ref for the buildtree dir, regardless of content, # leading to no buildtreedir being extracted result = cli.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Repeat building the artifacts, this time with cache-buildtrees set to # 'always' via the cli, as such the buildtree dir should not be empty cli.configure({ "artifacts": { "url": share2.repo, "push": True }, "cachedir": str(tmpdir) }) result = cli.run( project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share2.get_artifact( cli.get_artifact_name(project, "test", element_name)) # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees # that it was cached in share2 as expected with a populated buildtree dir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) assert cli.get_element_state(project, element_name) != "cached" result = cli.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Clarify that the user config option for cache-buildtrees works as the cli # main option does. Point to share3 which does not have the artifacts cached to force # a build cli.configure({ "artifacts": { "url": share3.repo, "push": True }, "cachedir": str(tmpdir), "cache": { "cache-buildtrees": "always" }, }) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir)
def test_pullbuildtrees(cli2, tmpdir, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing with create_artifact_share(os.path.join( str(tmpdir), "share1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "share2")) as share2, create_artifact_share( os.path.join(str(tmpdir), "share3")) as share3: cli2.configure({ "artifacts": { "servers": [{ "url": share1.repo, "push": True }] }, "cachedir": str(tmpdir), "cache": { "cache-buildtrees": "always" }, }) # Build autotools element, checked pushed, delete local result = cli2.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli2.get_element_state(project, element_name) == "cached" assert share1.get_artifact( cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Pull artifact with default config, assert that pulling again # doesn't create a pull job, then assert with buildtrees user # config set creates a pull job. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config, then assert that pulling # with buildtrees cli flag set creates a pull job. # Also assert that the buildtree is added to the local CAS. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() artifact_name = cli2.get_artifact_name(project, "test", element_name) with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) default_state(cli2, tmpdir, share1) # Pull artifact with pullbuildtrees set in user config, then assert # that pulling with the same user config doesn't creates a pull job, # or when buildtrees cli flag is set. cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config and buildtrees cli flag set, then assert # that pulling with pullbuildtrees set in user config doesn't create a pull # job. result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Assert that a partial build element (not containing a populated buildtree dir) # can't be pushed to an artifact share, then assert that a complete build element # can be. This will attempt a partial pull from share1 and then a partial push # to share2 result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure( {"artifacts": { "servers": [{ "url": share2.repo, "push": True }] }}) result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name not in result.get_pushed_elements() assert not share2.get_artifact( cli2.get_artifact_name(project, "test", element_name)) # Assert that after pulling the missing buildtree the element artifact can be # successfully pushed to the remote. This will attempt to pull the buildtree # from share1 and then a 'complete' push to share2 cli2.configure( {"artifacts": { "servers": [{ "url": share1.repo, "push": False }] }}) result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure( {"artifacts": { "servers": [{ "url": share2.repo, "push": True }] }}) result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name in result.get_pushed_elements() assert share2.get_artifact( cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Assert that bst artifact push will automatically attempt to pull a missing buildtree # if pull-buildtrees is set, however as share3 is the only defined remote and is empty, # assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the # artifact cannot be pushed. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure( {"artifacts": { "servers": [{ "url": share3.repo, "push": True }] }}) result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert element_name not in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir assert element_name not in result.get_pushed_elements() assert not share3.get_artifact( cli2.get_artifact_name(project, "test", element_name)) # Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will # automatically attempt to pull it and will be successful, leading to the full artifact being pushed # to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote, # without exlipictly requiring a bst artifact pull. cli2.configure({ "artifacts": { "servers": [{ "url": share1.repo, "push": False }, { "url": share3.repo, "push": True }] } }) result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert element_name in result.get_pushed_elements() assert share3.get_artifact( cli2.get_artifact_name(project, "test", element_name))