def test_resolve_gomod_unused_dep(mock_run, mock_temp_dir, tmpdir): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"} # Mock the "subprocess.run" calls mock_run.side_effect = [ mock.Mock(returncode=0, stdout=None), # go mod edit -replace mock.Mock(returncode=0, stdout=None), # go mod download mock.Mock(returncode=0, stdout=None), # go mod tidy mock.Mock(returncode=0, stdout=_generate_mock_cmd_output()), # go list -m all ] expected_error = "The following gomod dependency replacements don't apply: pizza" with pytest.raises(CachitoError, match=expected_error): resolve_gomod( "/path/archive.tar.gz", request, [{ "name": "pizza", "type": "gomod", "version": "v1.0.0" }], )
def test_resolve_gomod_strict_mode_raise_error(mock_isdir, mock_gwc, mock_run, mock_temp_dir, tmpdir): mock_isdir.return_value = True # Mock the get_worker_config mock_config = mock.Mock() mock_config.cachito_gomod_strict_vendor = True mock_config.cachito_athens_url = "http://athens:3000" mock_gwc.return_value = mock_config # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" call mock_run.return_value = mock.Mock(returncode=0, stdout=None) # go mod edit -replace archive_path = "/this/is/path/to/archive.tar.gz" request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"} expected_error = ( 'The "gomod-vendor" flag must be set when your repository has vendored dependencies.' ) with pytest.raises(CachitoError, match=expected_error): resolve_gomod(archive_path, request, [{ "name": "pizza", "type": "gomod", "version": "v1.0.0" }])
def test_go_list_cmd_failure(mock_run, mock_temp_dir, tmpdir, go_mod_rc, go_list_rc): archive_path = "/this/is/path/to/archive.tar.gz" request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"} # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" calls mock_run.side_effect = [ mock.Mock(returncode=go_mod_rc, stdout=None), # go mod download mock.Mock(returncode=go_list_rc, stdout=_generate_mock_cmd_output()), # go list -m all ] with pytest.raises(CachitoError) as exc_info: resolve_gomod(archive_path, request) assert str(exc_info.value) == "Processing gomod dependencies failed"
def test_resolve_gomod_vendor_dependencies( mock_bundle_dir, mock_run, mock_temp_dir, mock_golang_version, tmpdir, sample_package ): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" calls mock_run.side_effect = [ # go mod vendor mock.Mock(returncode=0, stdout=None), # go list -m all mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2"), # go list -find ./... mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2"), # go list -deps mock.Mock(returncode=0, stdout=pkg_lvl_stdout), ] mock_golang_version.return_value = "v2.1.1" archive_path = "/this/is/path/to/archive.tar.gz" request = { "id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848", "flags": ["gomod-vendor"], } gomod = resolve_gomod(archive_path, request) assert mock_run.call_args_list[0][0][0] == ("go", "mod", "vendor") assert gomod["module"] == sample_package assert not gomod["module_deps"] # Ensure an empty directory is created at bundle_dir.gomod_download_dir mock_bundle_dir.return_value.gomod_download_dir.mkdir.assert_called_once_with( exist_ok=True, parents=True )
def test_resolve_gomod_no_deps( mock_exists, mock_makedirs, mock_run, mock_merge_tree, mock_temp_dir, mock_golang_version, tmpdir, sample_package, sample_pkg_lvl_pkg, ): # Ensure to create the gomod download cache directory mock_exists.return_value = False # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" calls mock_run.side_effect = [ # go mod download mock.Mock(returncode=0, stdout=None), # go list -m all mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2"), # go list -find ./... mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2"), # go list -deps mock.Mock(returncode=0, stdout=pkg_lvl_stdout), ] mock_golang_version.return_value = "v2.1.1" archive_path = "/this/is/path/to/archive.tar.gz" request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"} gomod = resolve_gomod(archive_path, request) assert gomod["module"] == sample_package assert not gomod["module_deps"] assert len(gomod["packages"]) == 1 assert gomod["packages"][0]["pkg"] == sample_pkg_lvl_pkg assert not gomod["packages"][0]["pkg_deps"] # The second one ensures the source cache directory exists mock_makedirs.assert_called_once_with(os.path.join( tmpdir, RequestBundleDir.go_mod_cache_download_part), exist_ok=True) bundle_dir = RequestBundleDir(request["id"]) mock_merge_tree.assert_called_once_with( os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part), str(bundle_dir.gomod_download_dir), )
def test_resolve_gomod_disallow_absolute_paths( mock_run, mock_bundle_dir, mock_temp_dir, mock_golang_version, mock_merge_tree, tmpdir, platform_specific_path, ): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" calls mock_run.side_effect = [ # go mod download mock.Mock(returncode=0, stdout=None), # go list -m all mock.Mock(returncode=0, stdout="k8s.io/kubectl"), # go list -find ./... mock.Mock(returncode=0, stdout="k8s.io/kubernetes/cmd/kubectl"), # go list -deps mock.Mock( returncode=0, stdout= f"k8s.io/kubectl/pkg/apps k8s.io/kubectl => {platform_specific_path}", ), ] archive_path = "/this/is/path/to/archive.tar.gz" request = { "id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848", } err_msg = re.escape( f"Absolute paths to gomod dependencies are not supported: {platform_specific_path}" ) with pytest.raises(CachitoError, match=err_msg): resolve_gomod(archive_path, request)
def fetch_gomod_source(request_id, dep_replacements=None, package_configs=None): """ Resolve and fetch gomod dependencies for a given request. :param int request_id: the Cachito request ID this is for :param list dep_replacements: dependency replacements with the keys "name" and "version"; only supported with a single path :param list package_configs: the list of optional package configurations submitted by the user :raises CachitoError: if the dependencies could not be retrieved """ version_output = run_cmd(["go", "version"], {}) log.info(f"Go version: {version_output.strip()}") config = get_worker_config() if package_configs is None: package_configs = [] bundle_dir: RequestBundleDir = RequestBundleDir(request_id) subpaths = [os.path.normpath(c["path"]) for c in package_configs if c.get("path")] if not subpaths: # Default to the root of the application source subpaths = [os.curdir] invalid_gomod_files = _find_missing_gomod_files(bundle_dir, subpaths) if invalid_gomod_files: invalid_files_print = "; ".join(invalid_gomod_files) file_suffix = "s" if len(invalid_gomod_files) > 1 else "" # missing gomod files is supported if there is only one path referenced if config.cachito_gomod_ignore_missing_gomod_file and len(subpaths) == 1: log.warning("go.mod file missing for request at %s", invalid_files_print) return raise CachitoError( "The {} file{} must be present for the gomod package manager".format( invalid_files_print.strip(), file_suffix ) ) if len(subpaths) > 1 and dep_replacements: raise CachitoError( "Dependency replacements are only supported for a single go module path." ) env_vars = { "GOCACHE": {"value": "deps/gomod", "kind": "path"}, "GOPATH": {"value": "deps/gomod", "kind": "path"}, "GOMODCACHE": {"value": "deps/gomod/pkg/mod", "kind": "path"}, } env_vars.update(config.cachito_default_environment_variables.get("gomod", {})) update_request_env_vars(request_id, env_vars) packages_json_data = PackagesData() for i, subpath in enumerate(subpaths): log.info( "Fetching the gomod dependencies for request %d in subpath %s", request_id, subpath ) set_request_state( request_id, "in_progress", f'Fetching the gomod dependencies at the "{subpath}" directory', ) request = get_request(request_id) gomod_source_path = str(bundle_dir.app_subpath(subpath).source_dir) try: gomod = resolve_gomod( gomod_source_path, request, dep_replacements, bundle_dir.source_dir ) except CachitoError: log.exception("Failed to fetch gomod dependencies for request %d", request_id) raise module_info = gomod["module"] packages_json_data.add_package(module_info, subpath, gomod["module_deps"]) # add package deps for package in gomod["packages"]: pkg_info = package["pkg"] package_subpath = _package_subpath(module_info["name"], pkg_info["name"], subpath) packages_json_data.add_package(pkg_info, package_subpath, package.get("pkg_deps", [])) packages_json_data.write_to_file(bundle_dir.gomod_packages_data)
def test_resolve_gomod( mock_run, mock_merge_tree, mock_temp_dir, mock_golang_version, dep_replacement, go_list_error_pkg, expected_replace, tmpdir, sample_deps, sample_deps_replace, sample_deps_replace_new_name, sample_package, ): mock_cmd_output = _generate_mock_cmd_output(go_list_error_pkg) # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" calls run_side_effects = [] if dep_replacement: run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod edit -replace run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod download if dep_replacement: run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod tidy run_side_effects.append(mock.Mock( returncode=0, stdout=mock_cmd_output)) # go list -m all run_side_effects.append(mock.Mock( returncode=0, stdout=mock_pkg_list)) # go list -find ./... run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_deps)) # go list -deps mock_run.side_effect = run_side_effects mock_golang_version.return_value = "v2.1.1" archive_path = "/this/is/path/to/archive.tar.gz" request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"} if dep_replacement is None: gomod = resolve_gomod(archive_path, request) expected_deps = sample_deps else: gomod = resolve_gomod(archive_path, request, [dep_replacement]) if dep_replacement.get("new_name"): expected_deps = sample_deps_replace_new_name else: expected_deps = sample_deps_replace if expected_replace: assert mock_run.call_args_list[0][0][0] == ( "go", "mod", "edit", "-replace", expected_replace, ) if dep_replacement: assert mock_run.call_args_list[2][0][0] == ("go", "mod", "tidy") assert gomod["module"] == sample_package assert gomod["module_deps"] == expected_deps mock_merge_tree.assert_called_once_with( os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part), str(RequestBundleDir(request["id"]).gomod_download_dir), )
def test_resolve_gomod( mock_run, mock_get_worker_config, mock_set_full_relpaths, mock_vet_local_deps, mock_get_allowed_local_deps, mock_merge_tree, mock_temp_dir, mock_golang_version, dep_replacement, go_list_error_pkg, expected_replace, cgo_disable, tmpdir, sample_deps, sample_deps_replace, sample_deps_replace_new_name, sample_package, sample_pkg_deps_without_replace, ): mock_cmd_output = _generate_mock_cmd_output(go_list_error_pkg) # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = str(tmpdir) # Mock the "subprocess.run" calls run_side_effects = [] if dep_replacement: run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod edit -replace run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod download if dep_replacement: run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod tidy run_side_effects.append( mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2") # go list -m ) run_side_effects.append(mock.Mock(returncode=0, stdout=mock_cmd_output)) # go list -m all run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_list)) # go list -find ./... run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_deps)) # go list -deps -json mock_run.side_effect = run_side_effects mock_golang_version.return_value = "v2.1.1" mock_get_allowed_local_deps.return_value = ["*"] archive_path = "/this/is/path/to/archive.tar.gz" request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"} if cgo_disable: request["flags"] = ["cgo-disable"] if dep_replacement is None: gomod = resolve_gomod(archive_path, request) expected_deps = sample_deps else: gomod = resolve_gomod(archive_path, request, [dep_replacement]) if dep_replacement.get("new_name"): expected_deps = sample_deps_replace_new_name else: expected_deps = sample_deps_replace if expected_replace: assert mock_run.call_args_list[0][0][0] == ( "go", "mod", "edit", "-replace", expected_replace, ) if dep_replacement: assert mock_run.call_args_list[2][0][0] == ("go", "mod", "tidy") for call in mock_run.call_args_list: env = call.kwargs["env"] if cgo_disable: assert env["CGO_ENABLED"] == "0" else: assert "CGO_ENABLED" not in env assert gomod["module"] == sample_package assert gomod["module_deps"] == expected_deps assert len(gomod["packages"]) == 1 if dep_replacement is None: assert ( sorted(gomod["packages"][0]["pkg_deps"], key=_package_sort_key) == sample_pkg_deps_without_replace ) mock_merge_tree.assert_called_once_with( os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part), str(RequestBundleDir(request["id"]).gomod_download_dir), ) expect_module_name = sample_package["name"] mock_get_allowed_local_deps.assert_called_once_with(expect_module_name) mock_vet_local_deps.assert_has_calls( [ mock.call(expected_deps, expect_module_name, ["*"]), mock.call(gomod["packages"][0]["pkg_deps"], expect_module_name, ["*"]), ], ) mock_set_full_relpaths.assert_called_once_with(gomod["packages"][0]["pkg_deps"], expected_deps)