def test_resolve_gomod_unused_dep(mock_run, mock_temp_dir, tmpdir):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
# Mock the "subprocess.run" calls
mock_run.side_effect = [
mock.Mock(returncode=0, stdout=None), # go mod edit -replace
mock.Mock(returncode=0, stdout=None), # go mod download
mock.Mock(returncode=0, stdout=None), # go mod tidy
mock.Mock(returncode=0,
stdout=_generate_mock_cmd_output()), # go list -m all
]
expected_error = "The following gomod dependency replacements don't apply: pizza"
with pytest.raises(CachitoError, match=expected_error):
resolve_gomod(
"/path/archive.tar.gz",
request,
[{
"name": "pizza",
"type": "gomod",
"version": "v1.0.0"
}],
)
def test_resolve_gomod_strict_mode_raise_error(mock_isdir, mock_gwc, mock_run,
mock_temp_dir, tmpdir):
mock_isdir.return_value = True
# Mock the get_worker_config
mock_config = mock.Mock()
mock_config.cachito_gomod_strict_vendor = True
mock_config.cachito_athens_url = "http://athens:3000"
mock_gwc.return_value = mock_config
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" call
mock_run.return_value = mock.Mock(returncode=0,
stdout=None) # go mod edit -replace
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
expected_error = (
'The "gomod-vendor" flag must be set when your repository has vendored dependencies.'
)
with pytest.raises(CachitoError, match=expected_error):
resolve_gomod(archive_path, request, [{
"name": "pizza",
"type": "gomod",
"version": "v1.0.0"
}])
def test_go_list_cmd_failure(mock_run, mock_temp_dir, tmpdir, go_mod_rc, go_list_rc):
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
mock_run.side_effect = [
mock.Mock(returncode=go_mod_rc, stdout=None), # go mod download
mock.Mock(returncode=go_list_rc, stdout=_generate_mock_cmd_output()), # go list -m all
]
with pytest.raises(CachitoError) as exc_info:
resolve_gomod(archive_path, request)
assert str(exc_info.value) == "Processing gomod dependencies failed"
def test_resolve_gomod_vendor_dependencies(
mock_bundle_dir, mock_run, mock_temp_dir, mock_golang_version, tmpdir, sample_package
):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
mock_run.side_effect = [
# go mod vendor
mock.Mock(returncode=0, stdout=None),
# go list -m all
mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2"),
# go list -find ./...
mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2"),
# go list -deps
mock.Mock(returncode=0, stdout=pkg_lvl_stdout),
]
mock_golang_version.return_value = "v2.1.1"
archive_path = "/this/is/path/to/archive.tar.gz"
request = {
"id": 3,
"ref": "c50b93a32df1c9d700e3e80996845bc2e13be848",
"flags": ["gomod-vendor"],
}
gomod = resolve_gomod(archive_path, request)
assert mock_run.call_args_list[0][0][0] == ("go", "mod", "vendor")
assert gomod["module"] == sample_package
assert not gomod["module_deps"]
# Ensure an empty directory is created at bundle_dir.gomod_download_dir
mock_bundle_dir.return_value.gomod_download_dir.mkdir.assert_called_once_with(
exist_ok=True, parents=True
)
def test_resolve_gomod_no_deps(
mock_exists,
mock_makedirs,
mock_run,
mock_merge_tree,
mock_temp_dir,
mock_golang_version,
tmpdir,
sample_package,
sample_pkg_lvl_pkg,
):
# Ensure to create the gomod download cache directory
mock_exists.return_value = False
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
mock_run.side_effect = [
# go mod download
mock.Mock(returncode=0, stdout=None),
# go list -m all
mock.Mock(returncode=0,
stdout="github.com/release-engineering/retrodep/v2"),
# go list -find ./...
mock.Mock(returncode=0,
stdout="github.com/release-engineering/retrodep/v2"),
# go list -deps
mock.Mock(returncode=0, stdout=pkg_lvl_stdout),
]
mock_golang_version.return_value = "v2.1.1"
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
gomod = resolve_gomod(archive_path, request)
assert gomod["module"] == sample_package
assert not gomod["module_deps"]
assert len(gomod["packages"]) == 1
assert gomod["packages"][0]["pkg"] == sample_pkg_lvl_pkg
assert not gomod["packages"][0]["pkg_deps"]
# The second one ensures the source cache directory exists
mock_makedirs.assert_called_once_with(os.path.join(
tmpdir, RequestBundleDir.go_mod_cache_download_part),
exist_ok=True)
bundle_dir = RequestBundleDir(request["id"])
mock_merge_tree.assert_called_once_with(
os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part),
str(bundle_dir.gomod_download_dir),
)
def test_resolve_gomod_disallow_absolute_paths(
mock_run,
mock_bundle_dir,
mock_temp_dir,
mock_golang_version,
mock_merge_tree,
tmpdir,
platform_specific_path,
):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
mock_run.side_effect = [
# go mod download
mock.Mock(returncode=0, stdout=None),
# go list -m all
mock.Mock(returncode=0, stdout="k8s.io/kubectl"),
# go list -find ./...
mock.Mock(returncode=0, stdout="k8s.io/kubernetes/cmd/kubectl"),
# go list -deps
mock.Mock(
returncode=0,
stdout=
f"k8s.io/kubectl/pkg/apps k8s.io/kubectl => {platform_specific_path}",
),
]
archive_path = "/this/is/path/to/archive.tar.gz"
request = {
"id": 3,
"ref": "c50b93a32df1c9d700e3e80996845bc2e13be848",
}
err_msg = re.escape(
f"Absolute paths to gomod dependencies are not supported: {platform_specific_path}"
)
with pytest.raises(CachitoError, match=err_msg):
resolve_gomod(archive_path, request)
def fetch_gomod_source(request_id, dep_replacements=None, package_configs=None):
"""
Resolve and fetch gomod dependencies for a given request.
:param int request_id: the Cachito request ID this is for
:param list dep_replacements: dependency replacements with the keys "name" and "version"; only
supported with a single path
:param list package_configs: the list of optional package configurations submitted by the user
:raises CachitoError: if the dependencies could not be retrieved
"""
version_output = run_cmd(["go", "version"], {})
log.info(f"Go version: {version_output.strip()}")
config = get_worker_config()
if package_configs is None:
package_configs = []
bundle_dir: RequestBundleDir = RequestBundleDir(request_id)
subpaths = [os.path.normpath(c["path"]) for c in package_configs if c.get("path")]
if not subpaths:
# Default to the root of the application source
subpaths = [os.curdir]
invalid_gomod_files = _find_missing_gomod_files(bundle_dir, subpaths)
if invalid_gomod_files:
invalid_files_print = "; ".join(invalid_gomod_files)
file_suffix = "s" if len(invalid_gomod_files) > 1 else ""
# missing gomod files is supported if there is only one path referenced
if config.cachito_gomod_ignore_missing_gomod_file and len(subpaths) == 1:
log.warning("go.mod file missing for request at %s", invalid_files_print)
return
raise CachitoError(
"The {} file{} must be present for the gomod package manager".format(
invalid_files_print.strip(), file_suffix
)
)
if len(subpaths) > 1 and dep_replacements:
raise CachitoError(
"Dependency replacements are only supported for a single go module path."
)
env_vars = {
"GOCACHE": {"value": "deps/gomod", "kind": "path"},
"GOPATH": {"value": "deps/gomod", "kind": "path"},
"GOMODCACHE": {"value": "deps/gomod/pkg/mod", "kind": "path"},
}
env_vars.update(config.cachito_default_environment_variables.get("gomod", {}))
update_request_env_vars(request_id, env_vars)
packages_json_data = PackagesData()
for i, subpath in enumerate(subpaths):
log.info(
"Fetching the gomod dependencies for request %d in subpath %s", request_id, subpath
)
set_request_state(
request_id,
"in_progress",
f'Fetching the gomod dependencies at the "{subpath}" directory',
)
request = get_request(request_id)
gomod_source_path = str(bundle_dir.app_subpath(subpath).source_dir)
try:
gomod = resolve_gomod(
gomod_source_path, request, dep_replacements, bundle_dir.source_dir
)
except CachitoError:
log.exception("Failed to fetch gomod dependencies for request %d", request_id)
raise
module_info = gomod["module"]
packages_json_data.add_package(module_info, subpath, gomod["module_deps"])
# add package deps
for package in gomod["packages"]:
pkg_info = package["pkg"]
package_subpath = _package_subpath(module_info["name"], pkg_info["name"], subpath)
packages_json_data.add_package(pkg_info, package_subpath, package.get("pkg_deps", []))
packages_json_data.write_to_file(bundle_dir.gomod_packages_data)
def test_resolve_gomod(
mock_run,
mock_merge_tree,
mock_temp_dir,
mock_golang_version,
dep_replacement,
go_list_error_pkg,
expected_replace,
tmpdir,
sample_deps,
sample_deps_replace,
sample_deps_replace_new_name,
sample_package,
):
mock_cmd_output = _generate_mock_cmd_output(go_list_error_pkg)
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
run_side_effects = []
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0,
stdout=None)) # go mod edit -replace
run_side_effects.append(mock.Mock(returncode=0,
stdout=None)) # go mod download
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0,
stdout=None)) # go mod tidy
run_side_effects.append(mock.Mock(
returncode=0, stdout=mock_cmd_output)) # go list -m all
run_side_effects.append(mock.Mock(
returncode=0, stdout=mock_pkg_list)) # go list -find ./...
run_side_effects.append(mock.Mock(returncode=0,
stdout=mock_pkg_deps)) # go list -deps
mock_run.side_effect = run_side_effects
mock_golang_version.return_value = "v2.1.1"
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
if dep_replacement is None:
gomod = resolve_gomod(archive_path, request)
expected_deps = sample_deps
else:
gomod = resolve_gomod(archive_path, request, [dep_replacement])
if dep_replacement.get("new_name"):
expected_deps = sample_deps_replace_new_name
else:
expected_deps = sample_deps_replace
if expected_replace:
assert mock_run.call_args_list[0][0][0] == (
"go",
"mod",
"edit",
"-replace",
expected_replace,
)
if dep_replacement:
assert mock_run.call_args_list[2][0][0] == ("go", "mod", "tidy")
assert gomod["module"] == sample_package
assert gomod["module_deps"] == expected_deps
mock_merge_tree.assert_called_once_with(
os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part),
str(RequestBundleDir(request["id"]).gomod_download_dir),
)
def test_resolve_gomod(
mock_run,
mock_get_worker_config,
mock_set_full_relpaths,
mock_vet_local_deps,
mock_get_allowed_local_deps,
mock_merge_tree,
mock_temp_dir,
mock_golang_version,
dep_replacement,
go_list_error_pkg,
expected_replace,
cgo_disable,
tmpdir,
sample_deps,
sample_deps_replace,
sample_deps_replace_new_name,
sample_package,
sample_pkg_deps_without_replace,
):
mock_cmd_output = _generate_mock_cmd_output(go_list_error_pkg)
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
run_side_effects = []
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod edit -replace
run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod download
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod tidy
run_side_effects.append(
mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2") # go list -m
)
run_side_effects.append(mock.Mock(returncode=0, stdout=mock_cmd_output)) # go list -m all
run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_list)) # go list -find ./...
run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_deps)) # go list -deps -json
mock_run.side_effect = run_side_effects
mock_golang_version.return_value = "v2.1.1"
mock_get_allowed_local_deps.return_value = ["*"]
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
if cgo_disable:
request["flags"] = ["cgo-disable"]
if dep_replacement is None:
gomod = resolve_gomod(archive_path, request)
expected_deps = sample_deps
else:
gomod = resolve_gomod(archive_path, request, [dep_replacement])
if dep_replacement.get("new_name"):
expected_deps = sample_deps_replace_new_name
else:
expected_deps = sample_deps_replace
if expected_replace:
assert mock_run.call_args_list[0][0][0] == (
"go",
"mod",
"edit",
"-replace",
expected_replace,
)
if dep_replacement:
assert mock_run.call_args_list[2][0][0] == ("go", "mod", "tidy")
for call in mock_run.call_args_list:
env = call.kwargs["env"]
if cgo_disable:
assert env["CGO_ENABLED"] == "0"
else:
assert "CGO_ENABLED" not in env
assert gomod["module"] == sample_package
assert gomod["module_deps"] == expected_deps
assert len(gomod["packages"]) == 1
if dep_replacement is None:
assert (
sorted(gomod["packages"][0]["pkg_deps"], key=_package_sort_key)
== sample_pkg_deps_without_replace
)
mock_merge_tree.assert_called_once_with(
os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part),
str(RequestBundleDir(request["id"]).gomod_download_dir),
)
expect_module_name = sample_package["name"]
mock_get_allowed_local_deps.assert_called_once_with(expect_module_name)
mock_vet_local_deps.assert_has_calls(
[
mock.call(expected_deps, expect_module_name, ["*"]),
mock.call(gomod["packages"][0]["pkg_deps"], expect_module_name, ["*"]),
],
)
mock_set_full_relpaths.assert_called_once_with(gomod["packages"][0]["pkg_deps"], expected_deps)
