def test_unique_key(cli, tmpdir, datafiles): """This test confirms that the 'filename' parameter is honoured when it comes to generating a cache key for the source. """ project = str(datafiles) generate_project(project, {"aliases": { "tmpdir": "file:///" + str(tmpdir) }}) states = cli.get_element_states( project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) assert states["target.bst"] == "fetch needed" assert states["target-custom.bst"] == "fetch needed" assert states["target-custom-executable.bst"] == "fetch needed" # Try to fetch it cli.run(project=project, args=["source", "fetch", "target.bst"]) # We should download_yaml the file only once states = cli.get_element_states( project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) assert states["target.bst"] == "buildable" assert states["target-custom.bst"] == "buildable" assert states["target-custom-executable.bst"] == "buildable" # But the cache key is different because the 'filename' is different. assert (cli.get_element_key(project, "target.bst") != cli.get_element_key( project, "target-custom.bst") != cli.get_element_key( project, "target-custom-executable.bst"))
def test_strict_dependencies(cli, datafiles, target, expected_state): project = str(datafiles) # Configure non strict mode, this will have # an effect on the build and the `bst show` # commands run via cli.get_element_states() cli.configure({"projects": {"test": {"strict": False}}}) result = cli.run(project=project, silent=True, args=["build", target]) result.assert_success() states = cli.get_element_states(project, ["base.bst", target]) assert states["base.bst"] == "cached" assert states[target] == "cached" # Now modify the file, effectively causing the common base.bst # dependency to change it's cache key hello_path = os.path.join(project, "files", "hello.txt") with open(hello_path, "w", encoding="utf-8") as f: f.write("Goodbye") # Now assert that we have the states we expect as a result states = cli.get_element_states(project, ["base.bst", target]) assert states["base.bst"] == "buildable" assert states[target] == expected_state
def test_keep_dependencies(cli, datafiles): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip( "Filesystem does not support subsecond mtime precision: {}".format( project)) cli.configure({"cache": {"quota": 10000000}}) # Create a pretty big dependency create_element_size("dependency.bst", project, element_path, [], 5000000) res = cli.run(project=project, args=["build", "dependency.bst"]) res.assert_success() # Now create some other unrelated artifact create_element_size("unrelated.bst", project, element_path, [], 4000000) res = cli.run(project=project, args=["build", "unrelated.bst"]) res.assert_success() # Check that the correct element remains in the cache states = cli.get_element_states(project, ["dependency.bst", "unrelated.bst"]) assert states["dependency.bst"] == "cached" assert states["unrelated.bst"] == "cached" # We try to build an element which depends on the LRU artifact, # and could therefore fail if we didn't make sure dependencies # aren't removed. # # Since some artifact caches may implement weak cache keys by # duplicating artifacts (bad!) we need to make this equal in size # or smaller than half the size of its dependencies. # create_element_size("target.bst", project, element_path, ["dependency.bst"], 2000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() states = cli.get_element_states(project, ["target.bst", "unrelated.bst"]) assert states["target.bst"] == "cached" assert states["dependency.bst"] == "cached" assert states["unrelated.bst"] != "cached"
def test_dynamic_build_plan(cli, tmpdir, datafiles): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" all_elements = [target, build_dep, runtime_dep] with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }] }}) result = cli.run(project=project, args=["build", target]) result.assert_success() # Assert that everything is now cached in the remote. for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache directory casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore states = cli.get_element_states(project, all_elements) assert not any(states[e] == "cached" for e in all_elements) # Now try to rebuild target result = cli.run(project=project, args=["build", target]) result.assert_success() # Assert that target and runtime dependency were pulled # but build dependency was not pulled as it wasn't needed # (dynamic build plan). assert target in result.get_pulled_elements() assert runtime_dep in result.get_pulled_elements() assert build_dep not in result.get_pulled_elements() # And assert that the pulled elements are again in the local cache states = cli.get_element_states(project, all_elements) assert states[target] == "cached" assert states[runtime_dep] == "cached" assert states[build_dep] != "cached"
def test_default_target(cli, datafiles): project = str(datafiles) project_path = os.path.join(project, "project.conf") # First, modify project configuration to set a default target project_conf = { "name": "test-default-target", "min-version": "2.0", "element-path": "elements", "defaults": { "targets": ["dummy_stack.bst"] }, } _yaml.roundtrip_dump(project_conf, project_path) # dummy_stack only depends on dummy_1 and dummy_2, but not dummy_3 all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_stack.bst"] result = cli.run(project=project, args=["build"]) result.assert_success() states = cli.get_element_states(project, all_targets) assert all(states[e] == "cached" for e in all_targets) # assert that dummy_3 isn't included in the output assert "dummy_3.bst" not in states
def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" dep = "compose-all.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact result = cli.run(project=project, args=["build", element]) result.assert_success() assert cli.get_element_states(project, [element, dep], deps="none") == { element: "cached", dep: "cached", } # Obtain the artifact ref cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache assert os.path.exists( os.path.join(local_cache, "artifacts", "refs", artifact)) # Delete the artifact result = cli.run(project=project, args=["artifact", "delete", artifact, dep]) result.assert_success() # Check that the ARTIFACT is no longer in the cache assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact)) # Check that the dependency ELEMENT is no longer cached assert cli.get_element_state(project, dep) != "cached"
def test_push_pull_deps(cli, tmpdir, datafiles, deps, expected_states): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" all_elements = [target, build_dep, runtime_dep] with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }] }}) result = cli.run(project=project, args=["build", target]) result.assert_success() # Assert that everything is now cached in the remote. for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore states = cli.get_element_states(project, all_elements) assert not any(states[e] == "cached" for e in all_elements) # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", "--deps", deps, target]) result.assert_success() # And assert that the pulled elements are again in the local cache states = cli.get_element_states(project, all_elements) states_flattended = (states[target], states[build_dep], states[runtime_dep]) assert states_flattended == expected_states
def test_filter_track_multi(datafiles, cli, tmpdir): repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) input_name = "input.bst" input2_name = "input2.bst" project_config = { "name": "filter-track-test", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) input2_config = dict(input_config) input2_file = os.path.join(elements_dir, input2_name) _yaml.roundtrip_dump(input2_config, input2_file) filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = {"kind": "filter", "depends": [{"filename": input2_name, "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed states = cli.get_element_states(project, [input_name, input2_name]) assert states == { input_name: "no reference", input2_name: "no reference", } # Now try to track it result = cli.run(project=project, args=["source", "track", "filter1.bst", "filter2.bst"]) result.assert_success() # Now check that a ref field exists new_input = _yaml.load(input_file, shortname=None) source_node = new_input.get_sequence("sources").mapping_at(0) new_ref = source_node.get_str("ref") assert new_ref == ref new_input2 = _yaml.load(input2_file, shortname=None) source_node2 = new_input2.get_sequence("sources").mapping_at(0) new_ref2 = source_node2.get_str("ref") assert new_ref2 == ref
def test_fetch_deps(cli, datafiles, deps, expected_states): project = str(datafiles) target = "bananas.bst" build_dep = "apples.bst" runtime_dep = "oranges.bst" # Assert that none of the sources are cached states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert all(state == "fetch needed" for state in states.values()) # Now fetch the specified sources result = cli.run(project=project, args=["source", "fetch", "--deps", deps, target]) result.assert_success() # Finally assert that we have fetched _only_ the desired sources states = cli.get_element_states(project, [target, build_dep, runtime_dep]) states_flattened = (states[target], states[build_dep], states[runtime_dep]) assert states_flattened == expected_states
def test_track_single(cli, tmpdir, datafiles): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_dep_name = "track-test-dep.bst" element_target_name = "track-test-target.bst" # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = Git(str(tmpdir)) repo.create(dev_files_path) # Write out our test targets generate_element(repo, os.path.join(element_path, element_dep_name)) generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name) # Assert that tracking is needed for both elements states = cli.get_element_states(project, [element_target_name]) assert states == { element_dep_name: "no reference", element_target_name: "no reference", } # Now first try to track only one element result = cli.run( project=project, args=["source", "track", "--deps", "none", element_target_name]) result.assert_success() # And now fetch it result = cli.run( project=project, args=["source", "fetch", "--deps", "none", element_target_name]) result.assert_success() # Assert that the dependency is waiting and the target has still never been tracked states = cli.get_element_states(project, [element_target_name]) assert states == { element_dep_name: "no reference", element_target_name: "waiting", }
def test_expiry_order(cli, datafiles): project = str(datafiles) element_path = "elements" checkout = os.path.join(project, "workspace") cli.configure({"cache": {"quota": 9000000}}) # Create an artifact create_element_size("dep.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "dep.bst"]) res.assert_success() # Create another artifact create_element_size("unrelated.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "unrelated.bst"]) res.assert_success() # And build something else create_element_size("target.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() create_element_size("target2.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() wait_for_cache_granularity() # Now extract dep.bst res = cli.run( project=project, args=["artifact", "checkout", "dep.bst", "--directory", checkout]) res.assert_success() # Finally, build something that will cause the cache to overflow create_element_size("expire.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "expire.bst"]) res.assert_success() # While dep.bst was the first element to be created, it should not # have been removed. # Note that buildstream will reduce the cache to 50% of the # original size - we therefore remove multiple elements. check_elements = [ "unrelated.bst", "target.bst", "target2.bst", "dep.bst", "expire.bst" ] states = cli.get_element_states(project, check_elements) assert tuple(states[element] for element in check_elements) == ( "buildable", "buildable", "buildable", "cached", "cached", )
def test_no_default(cli, datafiles): project = str(datafiles) all_targets = [ "dummy_1.bst", "dummy_2.bst", "dummy_3.bst", "dummy_stack.bst" ] result = cli.run(project=project, args=["build"]) result.assert_success() states = cli.get_element_states(project, all_targets) assert all(states[e] == "cached" for e in all_targets)
def test_never_delete_required(cli, datafiles): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip( "Filesystem does not support subsecond mtime precision: {}".format( project)) cli.configure({ "cache": { "quota": 10000000 }, "scheduler": { "fetchers": 1, "builders": 1 } }) # Create a linear build tree create_element_size("dep1.bst", project, element_path, [], 8000000) create_element_size("dep2.bst", project, element_path, ["dep1.bst"], 8000000) create_element_size("dep3.bst", project, element_path, ["dep2.bst"], 8000000) create_element_size("target.bst", project, element_path, ["dep3.bst"], 8000000) # Build dep1.bst, which should fit into the cache. res = cli.run(project=project, args=["build", "dep1.bst"]) res.assert_success() # We try to build this pipeline, but it's too big for the # cache. Since all elements are required, the build should fail. res = cli.run(project=project, args=["build", "target.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.CAS, "cache-too-full") states = cli.get_element_states(project, ["target.bst"]) assert states["dep1.bst"] == "cached" assert states["dep2.bst"] != "cached" assert states["dep3.bst"] != "cached" assert states["target.bst"] != "cached"
def test_build_deps_build(datafiles, cli): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" result = cli.run(project=project, args=["build", "--deps", "build", target]) result.assert_success() states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert states[build_dep] == "cached" assert states[target] == "buildable" assert states[runtime_dep] == "buildable"
def test_artifact_delete_elements_build_deps(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Build the element and ensure it's cached result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert element and build deps are cached assert cli.get_element_state(project, element) == "cached" bdep_states = cli.get_element_states(project, [element], deps="build") for state in bdep_states.values(): assert state == "cached" result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", element]) result.assert_success() # Assert that the build deps have been deleted and that the artifact remains cached assert cli.get_element_state(project, element) == "cached" bdep_states = cli.get_element_states(project, [element], deps="build") for state in bdep_states.values(): assert state != "cached"
def test_track_deps(cli, datafiles, deps, expected_states): project = str(datafiles) generate_project(project, {"aliases": { "project-root": "file:///" + project }}) target = "bananas.bst" build_dep = "apples.bst" runtime_dep = "oranges.bst" # Assert that none of the sources have a reference states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert all(state == "no reference" for state in states.values()) # Now track the specified sources result = cli.run(project=project, args=["source", "track", "--deps", deps, target]) result.assert_success() # Finally assert that we have tracked _only_ the desired sources states = cli.get_element_states(project, [target, build_dep, runtime_dep]) states_flattened = (states[target], states[build_dep], states[runtime_dep]) assert states_flattened == expected_states
def test_artifact_delete_artifacts_build_deps(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache assert os.path.exists( os.path.join(local_cache, "artifacts", "refs", artifact)) # get the artifact refs of the build dependencies bdep_refs = [] bdep_states = cli.get_element_states(project, [element], deps="build") for bdep in bdep_states.keys(): bdep_refs.append( os.path.join("test", _get_normal_name(bdep), cli.get_element_key(project, bdep))) # Assert build dependencies are cached for ref in bdep_refs: assert os.path.exists( os.path.join(local_cache, "artifacts", "refs", ref)) # Delete the artifact result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", artifact]) result.assert_success() # Check that the artifact's build deps are no longer in the cache # Assert build dependencies have been deleted and that the artifact remains for ref in bdep_refs: assert not os.path.exists( os.path.join(local_cache, "artifacts", "refs", ref)) assert os.path.exists( os.path.join(local_cache, "artifacts", "refs", artifact))
def test_no_needless_overwrite(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") target = "track-test-target.bst" # Skip this test if we do not have support for subsecond precision mtimes # if not have_subsecond_mtime(project): pytest.skip( "Filesystem does not support subsecond mtime precision: {}".format( project)) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = Git(str(tmpdir)) repo.create(dev_files_path) # Write out our test target and assert it exists generate_element(repo, os.path.join(element_path, target)) path_to_target = os.path.join(element_path, target) assert os.path.exists(path_to_target) creation_mtime = os.path.getmtime(path_to_target) # Assert tracking is needed states = cli.get_element_states(project, [target]) assert states[target] == "no reference" # Perform the track result = cli.run(project=project, args=["source", "track", target]) result.assert_success() track1_mtime = os.path.getmtime(path_to_target) assert creation_mtime != track1_mtime # Now (needlessly) track again result = cli.run(project=project, args=["source", "track", target]) result.assert_success() track2_mtime = os.path.getmtime(path_to_target) assert track1_mtime == track2_mtime
def test_artifact_expires(cli, datafiles): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip( "Filesystem does not support subsecond mtime precision: {}".format( project)) cli.configure({"cache": { "quota": 10000000, }}) # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) create_element_size("target.bst", project, element_path, [], 6000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() assert cli.get_element_state(project, "target.bst") == "cached" # Our cache should now be almost full. Let's create another # artifact and see if we can cause buildstream to delete the old # one. create_element_size("target2.bst", project, element_path, [], 6000000) res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() # Check that the correct element remains in the cache states = cli.get_element_states(project, ["target.bst", "target2.bst"]) assert states["target.bst"] != "cached" assert states["target2.bst"] == "cached"
def test_cleanup_first(cli, datafiles): project = str(datafiles) element_path = "elements" cli.configure({"cache": { "quota": 10000000, }}) # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) create_element_size("target.bst", project, element_path, [], 8000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() assert cli.get_element_state(project, "target.bst") == "cached" # Now configure with a smaller quota, create a situation # where the cache must be cleaned up before building anything else. # # Fix the fetchers and builders just to ensure a predictable # sequence of events (although it does not effect this test) cli.configure({ "cache": { "quota": 5000000, }, "scheduler": { "fetchers": 1, "builders": 1 }, }) # Our cache is now more than full, BuildStream create_element_size("target2.bst", project, element_path, [], 4000000) res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() # Check that the correct element remains in the cache states = cli.get_element_states(project, ["target.bst", "target2.bst"]) assert states["target.bst"] != "cached" assert states["target2.bst"] == "cached"