def test_remote_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], )
def test_remote_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert element_name in result.get_pushed_elements() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # then remove it locally result = cli.run(project=project, args=["artifact", "delete", element_name]) result.assert_success() result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert element_name in result.get_pulled_elements()
def test_buildtree_remote(cli, tmpdir, datafiles): project = str(datafiles) element_name = "build-shell/buildtree.bst" share_path = os.path.join(str(tmpdir), "share") services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) with create_artifact_share(share_path) as share: cli.configure({ "artifacts": { "servers": [{ "url": share.repo, "push": True }] }, "cache": { "pull-buildtrees": False } }) res = cli.run( project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() # remove local cache shutil.rmtree(os.path.join(str(tmpdir), "cache", "cas")) shutil.rmtree(os.path.join(str(tmpdir), "cache", "artifacts")) # pull without buildtree res = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name]) res.assert_success() # check shell doesn't work res = cli.run( project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() # pull with buildtree res = cli.run(project=project, args=[ "--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name ]) res.assert_success() # check it works this time res = cli.run(project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "cat", "test" ]) res.assert_success() assert "Hi" in res.output
def test_junction_build_remote(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") subproject_element_path = os.path.join(subproject_path, "elements") amhello_files_path = os.path.join(subproject_path, "files") element_path = os.path.join(project, "elements") junction_path = os.path.join(element_path, "junction.bst") # We need a repo for real trackable elements repo = create_repo("git", str(tmpdir)) ref = repo.create(amhello_files_path) # ensure that the correct project directory is also listed in the junction subproject_conf = os.path.join(subproject_path, "project.conf") with open(subproject_conf, encoding="utf-8") as f: config = f.read() config = config.format(project_dir=subproject_path) with open(subproject_conf, "w", encoding="utf-8") as f: f.write(config) # Create a trackable element to depend on the cross junction element, # this one has it's ref resolved already create_element(repo, "sub-target.bst", subproject_element_path, ["autotools/amhello.bst"], ref=ref) # Create a trackable element to depend on the cross junction element create_element(repo, "target.bst", element_path, [{"junction": "junction.bst", "filename": "sub-target.bst"}]) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Now create a compose element at the top level element = {"kind": "compose", "depends": [{"filename": "target.bst", "type": "build"}]} _yaml.roundtrip_dump(element, os.path.join(element_path, "composed.bst")) # We're doing remote execution so ensure services are available services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # track the junction first to ensure we have refs result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # track target to ensure we have refs result = cli.run(project=project, args=["source", "track", "--deps", "all", "composed.bst"]) result.assert_success() # build result = cli.run(project=project, silent=True, args=["build", "composed.bst"]) result.assert_success() # Assert that the main target is cached as a result assert cli.get_element_state(project, "composed.bst") == "cached"
def test_remote_autotools_run(cli, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) services = cli.ensure_services() result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) result.assert_success() assert result.output == "Hello World!\nThis is amhello 1.0.\n"
def get_timemark(cli, project, element_name, marker): result = cli.run( project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", marker[1:]], ) result.assert_success() marker_time = int(result.output) return marker_time
def test_remote_autotools_build_no_cache(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" cli.configure({"artifacts": {"servers": [{"url": "http://fake.url.service", "push": True}]}}) result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert """WARNING Failed to initialize remote""" in result.stderr assert """Remote initialisation failed with status UNAVAILABLE: DNS resolution failed""" in result.stderr
def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files, build_all): project = str(datafiles) element_name = "no-runtime-deps.bst" builddep_element_name = "autotools/amhello.bst" checkout = os.path.join(cli.directory, "checkout") builddep_checkout = os.path.join(cli.directory, "builddep-checkout") services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # configure pull blobs if build_all: cli.configure({"build": {"dependencies": "all"}}) cli.config["remote-execution"]["pull-artifact-files"] = pull_artifact_files result = cli.run(project=project, args=["build", element_name]) result.assert_success() # Verify artifact is pulled bar files when ensure artifact files is set result = cli.run( project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) if pull_artifact_files: result.assert_success() assert_contains(checkout, ["/test"]) else: result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") # Verify build dependencies is pulled for ALL and BUILD result = cli.run(project=project, args=[ "artifact", "checkout", builddep_element_name, "--directory", builddep_checkout ]) if build_all and pull_artifact_files: result.assert_success() else: result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
def test_build_remote_failure(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") checkout_path = os.path.join(cli.directory, "checkout") # Write out our test target element = { "kind": "script", "depends": [ { "filename": "base.bst", "type": "build", }, ], "config": { "commands": [ "touch %{install-root}/foo", "false", ], }, } _yaml.roundtrip_dump(element, element_path) services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # Try to build it, this should result in a failure that contains the content result = cli.run(project=project, args=["build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result = cli.run(project=project, args=[ "artifact", "checkout", "element.bst", "--directory", checkout_path ]) result.assert_success() # check that the file created before the failure exists filename = os.path.join(checkout_path, "foo") assert os.path.isfile(filename)
def test_build_partial_push(cli, tmpdir, datafiles): project = str(datafiles) share_dir = os.path.join(str(tmpdir), "artifactshare") element_name = "no-runtime-deps.bst" builddep_element_name = "autotools/amhello.bst" with create_artifact_share(share_dir) as share: services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) cli.config["artifacts"] = { "url": share.repo, "push": True, } res = cli.run(project=project, args=["build", element_name]) res.assert_success() assert builddep_element_name in res.get_pushed_elements()
def check_buildtree( cli, project, element_name, input_files, generated_files, incremental=False, ): # check modified workspace dir was cached # - generated files are present # - generated files are newer than inputs # - check the date recorded in the marker file # - check that the touched file mtime is preserved from before assert cli and project and element_name and input_files and generated_files result = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1", "-exec", "stat", "-c", "%n::%Y", "{}", ";", ], ) result.assert_success() buildtree = {} output = result.output.splitlines() typ_inptime = None typ_gentime = None for line in output: assert "::" in line fname, mtime = line.split("::") # remove the symbolic dir fname = fname[1:] mtime = int(mtime) buildtree[fname] = mtime if incremental: # directory timestamps are not meaningful if fname in DIRS: continue if fname in input_files: if fname != os.sep + MAIN and not typ_inptime: typ_inptime = mtime if fname in generated_files: if fname != os.sep + MAINO and not typ_gentime: typ_gentime = mtime # all expected files should have been found for filename in input_files + generated_files: assert filename in buildtree if incremental: # the source file was changed so should be more recent than other input files # it should be older than the main object. # The main object should be more recent than generated files. assert buildtree[os.sep + MAIN] > typ_inptime assert buildtree[os.sep + MAINO] > buildtree[os.sep + MAIN] assert buildtree[os.sep + MAINO] > typ_gentime for fname in DIRS: del buildtree[fname] return buildtree
def test_workspace_build(cli, tmpdir, datafiles, modification): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") workspace = os.path.join(cli.directory, "workspace") element_name = "autotools/amhello.bst" # cli args artifact_checkout = ["artifact", "checkout", element_name, "--directory", checkout] build = ["--cache-buildtrees", "always", "build", element_name] input_files, generated_files, artifacts = files() services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # open a workspace for the element in the workspace directory result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) result.assert_success() # check that the workspace path exists assert os.path.exists(workspace) # add a file (asserting later that this is in the buildtree) newfile = "newfile.cfg" newfile_path = os.path.join(workspace, newfile) with open(newfile_path, "w", encoding="utf-8") as fdata: fdata.write("somestring") input_files.append(os.sep + newfile) # check that the workspace *only* contains the expected input files assert_contains(workspace, input_files, strict=True) # save the mtimes for later comparison ws_times = get_mtimes(workspace) # build the element and cache the buildtree result = cli.run(project=project, args=build) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" build_key = cli.get_element_key(project, element_name) # check that the local workspace is unchanged assert_contains(workspace, input_files, strict=True) assert ws_times == get_mtimes(workspace) # check modified workspace dir was cached and save the time # build was run. Incremental build conditions do not apply since the workspace # was initially opened using magic timestamps. build_times = check_buildtree(cli, project, element_name, input_files, generated_files, incremental=False) build_timemark = get_timemark(cli, project, element_name, (os.sep + BLDMARK)) # check that the artifacts are available result = cli.run(project=project, args=artifact_checkout) result.assert_success() assert_contains(checkout, artifacts) shutil.rmtree(checkout) # rebuild the element result = cli.run(project=project, args=build) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" rebuild_key = cli.get_element_key(project, element_name) assert rebuild_key == build_key rebuild_times = check_buildtree(cli, project, element_name, input_files, generated_files, incremental=False) rebuild_timemark = get_timemark(cli, project, element_name, (os.sep + BLDMARK)) # buildmark time should be the same assert build_timemark == rebuild_timemark assert all(rebuild_time == build_times[fname] for fname, rebuild_time in rebuild_times.items()), "{}\n{}".format( rebuild_times, build_times ) # modify the open workspace and rebuild main_path = os.path.join(workspace, MAIN) assert os.path.exists(main_path) if modification == "time": # touch a file in the workspace and save the mtime os.utime(main_path) touched_time = int(os.stat(main_path).st_mtime) elif modification == "content": # change a source file (there's a race here but it's not serious) with open(main_path, "r", encoding="utf-8") as fdata: data = fdata.readlines() with open(main_path, "w", encoding="utf-8") as fdata: for line in data: fdata.write(re.sub(r"Hello", "Goodbye", line)) touched_time = int(os.stat(main_path).st_mtime) # refresh input times ws_times = get_mtimes(workspace) # rebuild the element result = cli.run(project=project, args=build) result.assert_success() rebuild_times = check_buildtree(cli, project, element_name, input_files, generated_files, incremental=True) rebuild_timemark = get_timemark(cli, project, element_name, (os.sep + BLDMARK)) assert rebuild_timemark > build_timemark # check the times of the changed files assert rebuild_times[os.sep + MAIN] == touched_time del rebuild_times[os.sep + MAIN] del rebuild_times[os.sep + MAINO] del rebuild_times[os.sep + SRC + os.sep + "hello"] del rebuild_times[os.sep + DEPS + os.sep + "main.Po"] del rebuild_times[os.sep + BLDMARK] # check the times of the unmodified files assert all(rebuild_time == build_times[fname] for fname, rebuild_time in rebuild_times.items()), "{}\n{}".format( rebuild_times, build_times ) # Check workspace is unchanged assert_contains(workspace, input_files, strict=True) assert ws_times == get_mtimes(workspace)