def test_junction_dep_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in junctions
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# Add dependencies to the junction (not allowed, but let's do it
# anyway)
with open(junction_path, "a", encoding="utf-8") as f:
deps = {"depends": ["manual.bst"]}
_yaml.roundtrip_dump(deps, f)
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
result = cli.run(project=project, silent=True, args=["source", "fetch", "junction-dep.bst"])
# Since we aren't allowed to specify any dependencies on a
# junction, we should fail
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
# We don't get a final tally in this case
assert "subtasks processed" not in result.stderr
def test_nested_junction_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in
# junctions of junctions
project = str(datafiles)
sub1_path = os.path.join(project, "files", "sub-project")
sub2_path = os.path.join(project, "files", "sub2-project")
# A junction element which pulls sub1 into sub2
sub1_element = os.path.join(project, "files", "sub2-project", "elements", "sub-junction.bst")
# A junction element which pulls sub2 into the main project
sub2_element = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
generate_junction(tmpdir / "sub-project", sub1_path, sub1_element, store_ref=True)
generate_junction(tmpdir / "sub2-project", sub2_path, sub2_element, store_ref=True)
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-sub.bst"}]}
_yaml.roundtrip_dump(element, element_path)
result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"])
result.assert_success()
# Assert the correct progress tallies are in the logging
result = cli.run(project=project, args=["show", "junction-dep.bst"])
assert " 3 subtasks processed" in result.stderr
assert "3 of 3 subtasks processed" in result.stderr
def test_build_junction_short_notation_filename(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the content of /etc/animal.conf
filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
with open(filename, "r", encoding="utf-8") as f:
contents = f.read()
assert contents == "animal=Pony\n"
def test_junction_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in junctions
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# Create a stack element to depend on a cross junction element
#
element = {
"kind": "stack",
"depends": [{
"junction": "junction.bst",
"filename": "import-etc.bst"
}]
}
_yaml.roundtrip_dump(element, element_path)
result = cli.run(project=project,
silent=True,
args=["source", "fetch", "junction.bst"])
result.assert_success()
# Assert the correct progress tallies are in the logging
result = cli.run(project=project, args=["show", "junction-dep.bst"])
assert " 2 subtasks processed" in result.stderr
assert "2 of 2 subtasks processed" in result.stderr
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to track it, this will bail with the appropriate error
# informing the user to track the junction first
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
element_node = _yaml.load(element_path, shortname="junction-dep.bst")
ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in result.stderr
def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced):
project = str(datafiles)
project = os.path.join(datafiles.dirname, datafiles.basename)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"])
result.assert_success()
# Open a workspace if we're testing workspaced behavior
if workspaced:
result = cli.run(
project=project,
silent=True,
args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"],
)
result.assert_success()
# Assert the correct error when trying to show the pipeline
result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}-%{state}", element_name])
results = result.output.strip().splitlines()
assert "junction.bst:import-etc.bst-buildable" in results
def test_overlap_subproject(cli, tmpdir, datafiles, project_policy,
subproject_policy):
project_dir = str(datafiles)
subproject_dir = os.path.join(project_dir, "sub-project")
junction_path = os.path.join(project_dir, "sub-project.bst")
gen_project(project_dir,
bool(project_policy == "fail"),
project_name="test")
gen_project(subproject_dir,
bool(subproject_policy == "fail"),
project_name="subtest")
generate_junction(tmpdir, subproject_dir, junction_path)
# Here we have a dependency chain where the project element
# always overlaps with the subproject element.
#
# Test that overlap error vs warning policy for this overlap
# is always controlled by the project and not the subproject.
#
result = cli.run(project=project_dir,
silent=True,
args=["build", "sub-collect.bst"])
if project_policy == "fail":
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
else:
result.assert_success()
assert "WARNING [overlaps]" in result.stderr
def test_option_from_deep_junction(cli, tmpdir, datafiles):
project = os.path.join(str(datafiles), "junction_options_deep")
junction_repo_a = os.path.join(tmpdir, "a")
junction_repo_b = os.path.join(tmpdir, "b")
generate_junction(
junction_repo_a,
os.path.join(project, "subproject-2"),
os.path.join(project, "subproject-1", "junction-2.bst"),
store_ref=True,
options={"local_option": "set"},
)
generate_junction(
junction_repo_b,
os.path.join(project, "subproject-1"),
os.path.join(project, "junction-1.bst"),
store_ref=True,
)
result = cli.run(
project=project,
args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
assert not loaded.get_bool("is-default")
def test_push_cross_junction(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
result = cli.run(project=project,
args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
assert cli.get_element_state(project,
"junction.bst:import-etc.bst") == "cached"
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}],
}})
cli.run(project=project,
args=["artifact", "push", "junction.bst:import-etc.bst"])
cache_key = cli.get_element_key(project, "junction.bst:import-etc.bst")
assert share.get_artifact(
cli.get_artifact_name(project,
"subtest",
"import-etc.bst",
cache_key=cache_key))
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage,
workspaced):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
element = {
"kind": "stack",
"depends": [{
"junction": "junction.bst",
"filename": "import-etc.bst"
}]
}
_yaml.roundtrip_dump(element, element_path)
# Open a workspace if we're testing workspaced behavior
if workspaced:
result = cli.run(
project=project,
silent=True,
args=[
"workspace", "open", "--no-checkout", "--directory",
subproject_path, "junction.bst"
],
)
result.assert_success()
# Assert the correct error when trying to show the pipeline
dep_result = cli.run(project=project,
silent=True,
args=["show", "junction-dep.bst"])
# Assert the correct error when trying to show the pipeline
etc_result = cli.run(project=project,
silent=True,
args=["show", "junction.bst:import-etc.bst"])
# If a workspace is open, no ref is needed
if workspaced:
dep_result.assert_success()
etc_result.assert_success()
else:
# Assert that we have the expected provenance encoded into the error
element_node = _yaml.load(element_path, shortname="junction-dep.bst")
ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in dep_result.stderr
dep_result.assert_main_error(ErrorDomain.LOAD,
LoadErrorReason.SUBPROJECT_INCONSISTENT)
etc_result.assert_main_error(ErrorDomain.LOAD,
LoadErrorReason.SUBPROJECT_INCONSISTENT)
def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
element = {
"kind":
"stack",
"depends": [{
"filename": "junction.bst:import-etc.bst",
"junction": "junction.bst",
}]
}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should fail as filenames should not contain
# `:` when junction is explicity specified
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
def test_junction_build_remote(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
subproject_element_path = os.path.join(subproject_path, "elements")
amhello_files_path = os.path.join(subproject_path, "files")
element_path = os.path.join(project, "elements")
junction_path = os.path.join(element_path, "junction.bst")
# We need a repo for real trackable elements
repo = create_repo("git", str(tmpdir))
ref = repo.create(amhello_files_path)
# ensure that the correct project directory is also listed in the junction
subproject_conf = os.path.join(subproject_path, "project.conf")
with open(subproject_conf, encoding="utf-8") as f:
config = f.read()
config = config.format(project_dir=subproject_path)
with open(subproject_conf, "w", encoding="utf-8") as f:
f.write(config)
# Create a trackable element to depend on the cross junction element,
# this one has it's ref resolved already
create_element(repo, "sub-target.bst", subproject_element_path, ["autotools/amhello.bst"], ref=ref)
# Create a trackable element to depend on the cross junction element
create_element(repo, "target.bst", element_path, [{"junction": "junction.bst", "filename": "sub-target.bst"}])
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Now create a compose element at the top level
element = {"kind": "compose", "depends": [{"filename": "target.bst", "type": "build"}]}
_yaml.roundtrip_dump(element, os.path.join(element_path, "composed.bst"))
# We're doing remote execution so ensure services are available
services = cli.ensure_services()
assert set(services) == set(["action-cache", "execution", "storage"])
# track the junction first to ensure we have refs
result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
# track target to ensure we have refs
result = cli.run(project=project, args=["source", "track", "--deps", "all", "composed.bst"])
result.assert_success()
# build
result = cli.run(project=project, silent=True, args=["build", "composed.bst"])
result.assert_success()
# Assert that the main target is cached as a result
assert cli.get_element_state(project, "composed.bst") == "cached"
def test_include_junction_file(cli, tmpdir, datafiles):
project = os.path.join(str(datafiles), "junction")
generate_junction(tmpdir,
os.path.join(project, "subproject"),
os.path.join(project, "junction.bst"),
store_ref=True)
result = cli.run(
project=project,
args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
assert loaded.get_bool("included")
def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
project = os.path.join(str(datafiles), "overrides-junction")
generate_junction(tmpdir,
os.path.join(project, "subproject"),
os.path.join(project, "junction.bst"),
store_ref=True)
result = cli.run(
project=project,
args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
assert loaded.get_str("main_override", default=None) is not None
assert loaded.get_str("included_override", default=None) is None
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, workspaced):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
if ref_storage == "project.refs":
project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
_yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Open a workspace if we're testing workspaced behavior
if workspaced:
result = cli.run(
project=project,
silent=True,
args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"],
)
result.assert_success()
# Assert successful bst show (requires implicit subproject fetching)
result = cli.run(project=project, silent=True, args=["show", element_name])
result.assert_success()
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
if ref_storage == "project.refs":
project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
_yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
assert cli.get_element_state(project, "junction-dep.bst") == "cached"
def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst:foo.bst"]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should fail as recursive lookups for
# cross-junction elements is not allowed.
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
def test_build_checkout_cross_junction(datafiles, cli, tmpdir):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
checkout = os.path.join(cli.directory, "checkout")
generate_junction(tmpdir, subproject_path, junction_path)
result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
result = cli.run(
project=project, args=["artifact", "checkout", "junction.bst:import-etc.bst", "--directory", checkout]
)
result.assert_success()
filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
def test_push_pull_cross_junction(cli, tmpdir, datafiles):
project = str(datafiles)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
generate_junction(tmpdir,
subproject_path,
junction_path,
store_ref=True)
# First build the target element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project,
args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
assert cli.get_element_state(project,
"junction.bst:import-etc.bst") == "cached"
cache_dir = os.path.join(project, "cache", "cas")
shutil.rmtree(cache_dir)
artifact_dir = os.path.join(project, "cache", "artifacts")
shutil.rmtree(artifact_dir)
assert cli.get_element_state(
project, "junction.bst:import-etc.bst") == "buildable"
# Now try bst artifact pull
result = cli.run(
project=project,
args=["artifact", "pull", "junction.bst:import-etc.bst"])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert cli.get_element_state(project,
"junction.bst:import-etc.bst") == "cached"
def test_junction_element(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
element = {
"kind": "stack",
"depends": [{
"junction": "junction.bst",
"filename": "import-etc.bst"
}]
}
_yaml.roundtrip_dump(element, element_path)
# First demonstrate that showing the pipeline yields an error
result = cli.run(project=project, args=["show", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD,
LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
element_node = _yaml.load(element_path, shortname="junction-dep.bst")
ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in result.stderr
# Now track the junction itself
result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
# Now assert element state (via bst show under the hood) of the dep again
assert cli.get_element_state(project, "junction-dep.bst") == "waiting"
def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles):
project = str(datafiles)
subproject_path = os.path.join(project, "files", "sub-project")
junction_path = os.path.join(project, "elements", "junction.bst")
element_path = os.path.join(project, "elements", "junction-dep.bst")
workspace = os.path.join(cli.directory, "workspace")
checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element
#
element = {
"kind": "stack",
"depends": [{
"junction": "junction.bst",
"filename": "import-etc.bst"
}]
}
_yaml.roundtrip_dump(element, element_path)
# Now open a workspace on the junction
#
result = cli.run(
project=project,
args=["workspace", "open", "--directory", workspace, "junction.bst"])
result.assert_success()
filename = os.path.join(workspace, "files", "etc-files", "etc",
"animal.conf")
# Assert the content of /etc/animal.conf in the workspace
assert os.path.exists(filename)
with open(filename, "r") as f:
contents = f.read()
assert contents == "animal=Pony\n"
# Modify the content of the animal.conf in the workspace
with open(filename, "w") as f:
f.write("animal=Horsy\n")
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
result = cli.run(project=project,
args=[
"artifact", "checkout", "junction-dep.bst",
"--directory", checkout
])
result.assert_success()
# Assert the workspace modified content of /etc/animal.conf
filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
with open(filename, "r") as f:
contents = f.read()
assert contents == "animal=Horsy\n"
