def test_junction_dep_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in junctions project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Add dependencies to the junction (not allowed, but let's do it # anyway) with open(junction_path, "a", encoding="utf-8") as f: deps = {"depends": ["manual.bst"]} _yaml.roundtrip_dump(deps, f) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction-dep.bst"]) # Since we aren't allowed to specify any dependencies on a # junction, we should fail result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # We don't get a final tally in this case assert "subtasks processed" not in result.stderr
def test_nested_junction_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in # junctions of junctions project = str(datafiles) sub1_path = os.path.join(project, "files", "sub-project") sub2_path = os.path.join(project, "files", "sub2-project") # A junction element which pulls sub1 into sub2 sub1_element = os.path.join(project, "files", "sub2-project", "elements", "sub-junction.bst") # A junction element which pulls sub2 into the main project sub2_element = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") generate_junction(tmpdir / "sub-project", sub1_path, sub1_element, store_ref=True) generate_junction(tmpdir / "sub2-project", sub2_path, sub2_element, store_ref=True) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-sub.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Assert the correct progress tallies are in the logging result = cli.run(project=project, args=["show", "junction-dep.bst"]) assert " 3 subtasks processed" in result.stderr assert "3 of 3 subtasks processed" in result.stderr
def test_build_junction_short_notation_filename(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: contents = f.read() assert contents == "animal=Pony\n"
def test_junction_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in junctions project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Create a stack element to depend on a cross junction element # element = { "kind": "stack", "depends": [{ "junction": "junction.bst", "filename": "import-etc.bst" }] } _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Assert the correct progress tallies are in the logging result = cli.run(project=project, args=["show", "junction-dep.bst"]) assert " 2 subtasks processed" in result.stderr assert "2 of 2 subtasks processed" in result.stderr
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr
def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced): project = str(datafiles) project = os.path.join(datafiles.dirname, datafiles.basename) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Open a workspace if we're testing workspaced behavior if workspaced: result = cli.run( project=project, silent=True, args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], ) result.assert_success() # Assert the correct error when trying to show the pipeline result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}-%{state}", element_name]) results = result.output.strip().splitlines() assert "junction.bst:import-etc.bst-buildable" in results
def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_policy): project_dir = str(datafiles) subproject_dir = os.path.join(project_dir, "sub-project") junction_path = os.path.join(project_dir, "sub-project.bst") gen_project(project_dir, bool(project_policy == "fail"), project_name="test") gen_project(subproject_dir, bool(subproject_policy == "fail"), project_name="subtest") generate_junction(tmpdir, subproject_dir, junction_path) # Here we have a dependency chain where the project element # always overlaps with the subproject element. # # Test that overlap error vs warning policy for this overlap # is always controlled by the project and not the subproject. # result = cli.run(project=project_dir, silent=True, args=["build", "sub-collect.bst"]) if project_policy == "fail": result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() assert "WARNING [overlaps]" in result.stderr
def test_option_from_deep_junction(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "junction_options_deep") junction_repo_a = os.path.join(tmpdir, "a") junction_repo_b = os.path.join(tmpdir, "b") generate_junction( junction_repo_a, os.path.join(project, "subproject-2"), os.path.join(project, "subproject-1", "junction-2.bst"), store_ref=True, options={"local_option": "set"}, ) generate_junction( junction_repo_b, os.path.join(project, "subproject-1"), os.path.join(project, "junction-1.bst"), store_ref=True, ) result = cli.run( project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert not loaded.get_bool("is-default")
def test_push_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }], }}) cli.run(project=project, args=["artifact", "push", "junction.bst:import-etc.bst"]) cache_key = cli.get_element_key(project, "junction.bst:import-etc.bst") assert share.get_artifact( cli.get_artifact_name(project, "subtest", "import-etc.bst", cache_key=cache_key))
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = { "kind": "stack", "depends": [{ "junction": "junction.bst", "filename": "import-etc.bst" }] } _yaml.roundtrip_dump(element, element_path) # Open a workspace if we're testing workspaced behavior if workspaced: result = cli.run( project=project, silent=True, args=[ "workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst" ], ) result.assert_success() # Assert the correct error when trying to show the pipeline dep_result = cli.run(project=project, silent=True, args=["show", "junction-dep.bst"]) # Assert the correct error when trying to show the pipeline etc_result = cli.run(project=project, silent=True, args=["show", "junction.bst:import-etc.bst"]) # If a workspace is open, no ref is needed if workspaced: dep_result.assert_success() etc_result.assert_success() else: # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in dep_result.stderr dep_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) etc_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = { "kind": "stack", "depends": [{ "filename": "junction.bst:import-etc.bst", "junction": "junction.bst", }] } _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as filenames should not contain # `:` when junction is explicity specified result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
def test_junction_build_remote(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") subproject_element_path = os.path.join(subproject_path, "elements") amhello_files_path = os.path.join(subproject_path, "files") element_path = os.path.join(project, "elements") junction_path = os.path.join(element_path, "junction.bst") # We need a repo for real trackable elements repo = create_repo("git", str(tmpdir)) ref = repo.create(amhello_files_path) # ensure that the correct project directory is also listed in the junction subproject_conf = os.path.join(subproject_path, "project.conf") with open(subproject_conf, encoding="utf-8") as f: config = f.read() config = config.format(project_dir=subproject_path) with open(subproject_conf, "w", encoding="utf-8") as f: f.write(config) # Create a trackable element to depend on the cross junction element, # this one has it's ref resolved already create_element(repo, "sub-target.bst", subproject_element_path, ["autotools/amhello.bst"], ref=ref) # Create a trackable element to depend on the cross junction element create_element(repo, "target.bst", element_path, [{"junction": "junction.bst", "filename": "sub-target.bst"}]) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Now create a compose element at the top level element = {"kind": "compose", "depends": [{"filename": "target.bst", "type": "build"}]} _yaml.roundtrip_dump(element, os.path.join(element_path, "composed.bst")) # We're doing remote execution so ensure services are available services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # track the junction first to ensure we have refs result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # track target to ensure we have refs result = cli.run(project=project, args=["source", "track", "--deps", "all", "composed.bst"]) result.assert_success() # build result = cli.run(project=project, silent=True, args=["build", "composed.bst"]) result.assert_success() # Assert that the main target is cached as a result assert cli.get_element_state(project, "composed.bst") == "cached"
def test_include_junction_file(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "junction") generate_junction(tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True) result = cli.run( project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_bool("included")
def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "overrides-junction") generate_junction(tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True) result = cli.run( project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("main_override", default=None) is not None assert loaded.get_str("included_override", default=None) is None
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, workspaced): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # if ref_storage == "project.refs": project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Open a workspace if we're testing workspaced behavior if workspaced: result = cli.run( project=project, silent=True, args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], ) result.assert_success() # Assert successful bst show (requires implicit subproject fetching) result = cli.run(project=project, silent=True, args=["show", element_name]) result.assert_success()
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # if ref_storage == "project.refs": project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached"
def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst:foo.bst"]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as recursive lookups for # cross-junction elements is not allowed. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
def test_build_checkout_cross_junction(datafiles, cli, tmpdir): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") checkout = os.path.join(cli.directory, "checkout") generate_junction(tmpdir, subproject_path, junction_path) result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() result = cli.run( project=project, args=["artifact", "checkout", "junction.bst:import-etc.bst", "--directory", checkout] ) result.assert_success() filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename)
def test_push_pull_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # First build the target element and push to the remote. cli.configure( {"artifacts": { "servers": [{ "url": share.repo, "push": True }] }}) result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" cache_dir = os.path.join(project, "cache", "cas") shutil.rmtree(cache_dir) artifact_dir = os.path.join(project, "cache", "artifacts") shutil.rmtree(artifact_dir) assert cli.get_element_state( project, "junction.bst:import-etc.bst") == "buildable" # Now try bst artifact pull result = cli.run( project=project, args=["artifact", "pull", "junction.bst:import-etc.bst"]) result.assert_success() # And assert that it's again in the local cache, without having built assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached"
def test_junction_element(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = { "kind": "stack", "depends": [{ "junction": "junction.bst", "filename": "import-etc.bst" }] } _yaml.roundtrip_dump(element, element_path) # First demonstrate that showing the pipeline yields an error result = cli.run(project=project, args=["show", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr # Now track the junction itself result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # Now assert element state (via bst show under the hood) of the dep again assert cli.get_element_state(project, "junction-dep.bst") == "waiting"
def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") workspace = os.path.join(cli.directory, "workspace") checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element # element = { "kind": "stack", "depends": [{ "junction": "junction.bst", "filename": "import-etc.bst" }] } _yaml.roundtrip_dump(element, element_path) # Now open a workspace on the junction # result = cli.run( project=project, args=["workspace", "open", "--directory", workspace, "junction.bst"]) result.assert_success() filename = os.path.join(workspace, "files", "etc-files", "etc", "animal.conf") # Assert the content of /etc/animal.conf in the workspace assert os.path.exists(filename) with open(filename, "r") as f: contents = f.read() assert contents == "animal=Pony\n" # Modify the content of the animal.conf in the workspace with open(filename, "w") as f: f.write("animal=Horsy\n") # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out result = cli.run(project=project, args=[ "artifact", "checkout", "junction-dep.bst", "--directory", checkout ]) result.assert_success() # Assert the workspace modified content of /etc/animal.conf filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) with open(filename, "r") as f: contents = f.read() assert contents == "animal=Horsy\n"