def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache): project = str(datafiles) element_name = "autotools/amhello.bst" # push to an artifact server so we can pull from it later. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }] }}) result = cli.run(project=project, args=["build", element_name]) result.assert_success() # If the build is cached then it might not push to the artifact cache result = cli.run(project=project, args=["artifact", "push", element_name]) result.assert_success() result = cli.run(project=project, args=["shell", element_name]) result.assert_success() # do a checkout and get the digest of the hello binary. result = cli.run( project=project, args=[ "artifact", "checkout", "--deps", "none", "--directory", os.path.join(str(tmpdir), "tmp"), "autotools/amhello.bst", ], ) result.assert_success() digest = utils.sha256sum( os.path.join(str(tmpdir), "tmp", "usr", "bin", "hello")) # Remove the binary from the CAS cachedir = cli.config["cachedir"] objpath = os.path.join(cachedir, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # check shell doesn't work result = cli.run(project=project, args=["shell", element_name, "--", "hello"]) result.assert_main_error(ErrorDomain.APP, "shell-missing-deps") # check the artifact gets completed with '--pull' specified result = cli.run(project=project, args=["shell", "--pull", element_name, "--", "hello"]) result.assert_success() assert "autotools/amhello.bst" in result.get_pulled_elements()
def test_push_cached_fail(cli, tmpdir, datafiles, on_error): if on_error == "quit": pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534") project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build",},], "config": { "commands": [ "false", # Ensure unique cache key for different test variants 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( {"artifacts": {"url": share.repo, "push": True},} ) # Build the element, continuing to finish active jobs on error. result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst"))
def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles): # Test that if we stage a cached and empty buildtree, we warn the user. project = str(datafiles) element_name = "build-shell/buildtree.bst" # Switch to a temp artifact cache dir to ensure the artifact is rebuilt, # without caching a buildtree which is the default bst behaviour cli_integration.configure({"cachedir": str(tmpdir)}) res = cli_integration.run(project=project, args=["build", element_name]) res.assert_success() res = cli_integration.run(project=project, args=[ "shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test" ]) res.assert_main_error(ErrorDomain.APP, None) assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr # Now attempt the same with the try option, this should not attempt to find a buildtree # and just launch the shell, however the cat should still fail. res = cli_integration.run(project=project, args=[ "shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test" ]) assert "Artifact created without buildtree, shell will be loaded without it" in res.stderr assert "Hi" not in res.output
def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles): project = str(datafiles) element_name = "build-shell/buildtree-fail.bst" # build with --cache-buildtrees set to 'always', behaviour should match # default behaviour (which is always) as the buildtree will explicitly have been # cached with content. cli_integration.configure({"cachedir": str(tmpdir)}) res = cli_integration.run( project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) res = cli_integration.run(project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test" ]) res.assert_success() assert "WARNING: using a buildtree from a failed build" in res.stderr assert "Hi" in res.output
def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error): """Test that we can upload a built artifact that didn't have a valid shell inside. When we don't have a valid shell, the artifact will be empty, not even the root directory. This ensures we handle the case of an entirely empty artifact correctly. """ if on_error == "quit": pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534") project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "config": { "commands": [ "false", # Ensure unique cache key for different test variants 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }] }}) # Build the element, continuing to finish active jobs on error. result = cli.run( project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote assert share.get_artifact( cli.get_artifact_name(project, "test", "element.bst"))
def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles): project = str(datafiles) element_name = "build-shell/buildtree-fail.bst" # Switch to a temp artifact cache dir to ensure the artifact is rebuilt, # without caching a buildtree explicitly cli_integration.configure({"cachedir": str(tmpdir)}) res = cli_integration.run( project=project, args=["--cache-buildtrees", "never", "build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) res = cli_integration.run(project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test" ]) res.assert_main_error(ErrorDomain.APP, None) assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
def test_cache_buildtrees(cli, tmpdir, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing with create_artifact_share(os.path.join( str(tmpdir), "share1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "share2")) as share2, create_artifact_share( os.path.join(str(tmpdir), "share3")) as share3: cli.configure({ "artifacts": { "url": share1.repo, "push": True }, "cachedir": str(tmpdir) }) # Build autotools element with the default behavior of caching buildtrees # only when necessary. The artifact should be successfully pushed to the share1 remote # and cached locally with an 'empty' buildtree digest, as it's not a # dangling ref result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share1.get_artifact( cli.get_artifact_name(project, "test", element_name)) # The buildtree dir should not exist, as we set the config to not cache buildtrees. artifact_name = cli.get_artifact_name(project, "test", element_name) assert share1.get_artifact(artifact_name) with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees # that is was cached in share1 as expected without a buildtree dir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) assert cli.get_element_state(project, element_name) != "cached" result = cli.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Assert that the default behaviour of pull to not include buildtrees on the artifact # in share1 which was purposely cached with an empty one behaves as expected. As such the # pulled artifact will have a dangling ref for the buildtree dir, regardless of content, # leading to no buildtreedir being extracted result = cli.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Repeat building the artifacts, this time with cache-buildtrees set to # 'always' via the cli, as such the buildtree dir should not be empty cli.configure({ "artifacts": { "url": share2.repo, "push": True }, "cachedir": str(tmpdir) }) result = cli.run( project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share2.get_artifact( cli.get_artifact_name(project, "test", element_name)) # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees # that it was cached in share2 as expected with a populated buildtree dir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) assert cli.get_element_state(project, element_name) != "cached" result = cli.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Clarify that the user config option for cache-buildtrees works as the cli # main option does. Point to share3 which does not have the artifacts cached to force # a build cli.configure({ "artifacts": { "url": share3.repo, "push": True }, "cachedir": str(tmpdir), "cache": { "cache-buildtrees": "always" }, }) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir)
def test_pullbuildtrees(cli2, tmpdir, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing with create_artifact_share(os.path.join( str(tmpdir), "share1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "share2")) as share2, create_artifact_share( os.path.join(str(tmpdir), "share3")) as share3: cli2.configure({ "artifacts": { "servers": [{ "url": share1.repo, "push": True }] }, "cachedir": str(tmpdir), "cache": { "cache-buildtrees": "always" }, }) # Build autotools element, checked pushed, delete local result = cli2.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli2.get_element_state(project, element_name) == "cached" assert share1.get_artifact( cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Pull artifact with default config, assert that pulling again # doesn't create a pull job, then assert with buildtrees user # config set creates a pull job. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config, then assert that pulling # with buildtrees cli flag set creates a pull job. # Also assert that the buildtree is added to the local CAS. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() artifact_name = cli2.get_artifact_name(project, "test", element_name) with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) default_state(cli2, tmpdir, share1) # Pull artifact with pullbuildtrees set in user config, then assert # that pulling with the same user config doesn't creates a pull job, # or when buildtrees cli flag is set. cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config and buildtrees cli flag set, then assert # that pulling with pullbuildtrees set in user config doesn't create a pull # job. result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Assert that a partial build element (not containing a populated buildtree dir) # can't be pushed to an artifact share, then assert that a complete build element # can be. This will attempt a partial pull from share1 and then a partial push # to share2 result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure( {"artifacts": { "servers": [{ "url": share2.repo, "push": True }] }}) result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name not in result.get_pushed_elements() assert not share2.get_artifact( cli2.get_artifact_name(project, "test", element_name)) # Assert that after pulling the missing buildtree the element artifact can be # successfully pushed to the remote. This will attempt to pull the buildtree # from share1 and then a 'complete' push to share2 cli2.configure( {"artifacts": { "servers": [{ "url": share1.repo, "push": False }] }}) result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure( {"artifacts": { "servers": [{ "url": share2.repo, "push": True }] }}) result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name in result.get_pushed_elements() assert share2.get_artifact( cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Assert that bst artifact push will automatically attempt to pull a missing buildtree # if pull-buildtrees is set, however as share3 is the only defined remote and is empty, # assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the # artifact cannot be pushed. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure( {"artifacts": { "servers": [{ "url": share3.repo, "push": True }] }}) result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert element_name not in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir assert element_name not in result.get_pushed_elements() assert not share3.get_artifact( cli2.get_artifact_name(project, "test", element_name)) # Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will # automatically attempt to pull it and will be successful, leading to the full artifact being pushed # to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote, # without exlipictly requiring a bst artifact pull. cli2.configure({ "artifacts": { "servers": [{ "url": share1.repo, "push": False }, { "url": share3.repo, "push": True }] } }) result = cli2.run( project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert element_name in result.get_pushed_elements() assert share3.get_artifact( cli2.get_artifact_name(project, "test", element_name))