def create_bundle_archive(request_id: int, flags: List[str]) -> None:
"""
Create the bundle archive to be downloaded by the user.
:param int request_id: the request the bundle is for
:param list[str] flags: the list of request flags.
"""
set_request_state(request_id, "in_progress",
"Assembling the bundle archive")
bundle_dir = RequestBundleDir(request_id)
log.debug("Using %s for creating the bundle for request %d", bundle_dir,
request_id)
log.info("Creating %s", bundle_dir.bundle_archive_file)
def filter_git_dir(tar_info):
return tar_info if os.path.basename(tar_info.name) != ".git" else None
tar_filter: Optional[Callable[[Any], Any]] = filter_git_dir
if "include-git-dir" in flags:
tar_filter = None
with tarfile.open(bundle_dir.bundle_archive_file,
mode="w:gz") as bundle_archive:
# Add the source to the bundle. This is done one file/directory at a time in the parent
# directory in order to exclude the app/.git folder.
for item in bundle_dir.source_dir.iterdir():
arc_name = os.path.join("app", item.name)
bundle_archive.add(str(item), arc_name, filter=tar_filter)
# Add the dependencies to the bundle
bundle_archive.add(str(bundle_dir.deps_dir), "deps")
def create_bundle_archive(request_id):
"""
Create the bundle archive to be downloaded by the user.
:param int request_id: the request the bundle is for
"""
request = set_request_state(request_id, "in_progress",
"Assembling the bundle archive")
bundle_dir = RequestBundleDir(request_id)
log.debug("Using %s for creating the bundle for request %d", bundle_dir,
request_id)
log.info("Creating %s", bundle_dir.bundle_archive_file)
def filter_git_dir(tar_info):
return tar_info if os.path.basename(tar_info.name) != ".git" else None
tar_filter = filter_git_dir
if "include-git-dir" in request.get("flags", []):
tar_filter = None
with tarfile.open(bundle_dir.bundle_archive_file,
mode="w:gz") as bundle_archive:
# Add the source to the bundle. This is done one file/directory at a time in the parent
# directory in order to exclude the app/.git folder.
for item in bundle_dir.source_dir.iterdir():
arc_name = os.path.join("app", item.name)
bundle_archive.add(str(item), arc_name, filter=tar_filter)
# Add the dependencies to the bundle
bundle_archive.add(str(bundle_dir.deps_dir), "deps")
set_request_state(request_id, "complete", "Completed successfully")
def test_add_git_submodules_as_package(get_worker_config, mock_repo,
task_passes_state_check, tmpdir):
get_worker_config.return_value = mock.Mock(cachito_bundles_dir=tmpdir)
submodule = mock.Mock()
submodule.name = "tour"
submodule.hexsha = "522fb816eec295ad58bc488c74b2b46748d471b2"
submodule.url = "https://github.com/user/tour.git"
submodule.path = "tour"
mock_repo.return_value.submodules = [submodule]
package = {
"type":
"git-submodule",
"name":
"tour",
"version":
"https://github.com/user/tour.git#522fb816eec295ad58bc488c74b2b46748d471b2",
}
gitsubmodule.add_git_submodules_as_package(3)
bundle_dir = RequestBundleDir(3)
expected = package.copy()
expected["path"] = "tour"
expected["dependencies"] = []
assert {
"packages": [expected]
} == json.loads(bundle_dir.git_submodule_packages_data.read_bytes())
def fetch_app_source(url, ref, request_id, gitsubmodule=False):
"""
Fetch the application source code that was requested and put it in long-term storage.
:param str url: the source control URL to pull the source from
:param str ref: the source control reference
:param int request_id: the Cachito request ID this is for
:param bool gitsubmodule: a bool to determine whether git submodules need to be processed.
"""
log.info('Fetching the source from "%s" at reference "%s"', url, ref)
set_request_state(request_id, "in_progress",
"Fetching the application source")
try:
# Default to Git for now
scm = Git(url, ref)
scm.fetch_source(gitsubmodule=gitsubmodule)
except requests.Timeout:
raise CachitoError(
"The connection timed out while downloading the source")
except CachitoError:
log.exception(
'Failed to fetch the source from the URL "%s" and reference "%s"',
url, ref)
raise
# Extract the archive contents to the temporary directory of where the bundle is being created.
# This will eventually end up in the bundle the user downloads. This is extracted now since
# some package managers may add dependency replacements, which require edits to source files.
bundle_dir = RequestBundleDir(request_id)
log.debug("Extracting %s to %s", scm.sources_dir.archive_path, bundle_dir)
shutil.unpack_archive(str(scm.sources_dir.archive_path), str(bundle_dir))
_enforce_sandbox(bundle_dir.source_root_dir)
def test_fetch_app_source(mock_set_request_state, fake_repo, gitsubmodule):
request_id = 1
repo_dir, repo_name = fake_repo
tasks.fetch_app_source(f"file://{repo_dir}", "master", request_id, gitsubmodule)
# Verify the archive file is created from fetched app source.
sources_dir = SourcesDir(repo_name, "master")
assert sources_dir.archive_path.name == "master.tar.gz"
# Verify the archive file is extracted into request bundle directory.
bundle_dir = RequestBundleDir(request_id)
assert bundle_dir.joinpath("app", "readme.rst").exists()
assert bundle_dir.joinpath("app", "main.py").exists()
# Clean up bundle dir after unpacking archive
shutil.rmtree(bundle_dir)
def resolve_yarn(app_source_path, request, skip_deps=None):
"""
Resolve and fetch npm dependencies for the given app source archive.
:param (str | Path) app_source_path: the full path to the application source code
:param dict request: the Cachito request this is for
:param set skip_deps: a set of dependency identifiers to not download because they've already
been downloaded for this request
:return: a dictionary that has the following keys:
``deps`` which is the list of dependencies,
``downloaded_deps`` which is a set of the dependency identifiers of the dependencies that
were downloaded as part of this function's execution,
``lock_file`` which is the lock file if it was modified,
``package`` which is the dictionary describing the main package, and
``package.json`` which is the package.json file if it was modified.
:rtype: dict
:raises CachitoError: if fetching the dependencies fails or required files are missing
"""
app_source_path = Path(app_source_path)
package_json_path = app_source_path / "package.json"
yarn_lock_path = app_source_path / "yarn.lock"
package_and_deps_info = _get_package_and_deps(package_json_path,
yarn_lock_path)
# By downloading the dependencies, it stores the tarballs in the bundle and also stages the
# content in the yarn repository for the request
proxy_repo_url = get_yarn_proxy_repo_url(request["id"])
bundle_dir = RequestBundleDir(request["id"])
bundle_dir.yarn_deps_dir.mkdir(exist_ok=True)
package_and_deps_info["downloaded_deps"] = download_dependencies(
bundle_dir.yarn_deps_dir,
package_and_deps_info["deps"],
proxy_repo_url,
skip_deps=skip_deps,
pkg_manager="yarn",
)
replacements = package_and_deps_info.pop("nexus_replacements")
pkg_json = _replace_deps_in_package_json(
package_and_deps_info["package.json"], replacements)
yarn_lock = _replace_deps_in_yarn_lock(package_and_deps_info["lock_file"],
replacements)
package_and_deps_info["package.json"] = pkg_json
if _set_proxy_resolved_urls(yarn_lock,
get_yarn_proxy_repo_name(request["id"])):
package_and_deps_info["lock_file"] = yarn_lock
else:
package_and_deps_info["lock_file"] = None
# Remove all the "bundled" and "version_in_nexus" keys since they are implementation details
for dep in package_and_deps_info["deps"]:
dep.pop("bundled")
dep.pop("version_in_nexus")
return package_and_deps_info
def test_resolve_npm(
get_worker_config,
mock_dd,
mock_gpad,
mock_exists,
shrink_wrap,
package_lock,
package_and_deps,
tmpdir,
):
get_worker_config.return_value = mock.Mock(cachito_bundles_dir=str(tmpdir))
package_json = True
mock_dd.return_value = {
"@angular-devkit/[email protected]",
"@angular/[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
}
# Note that the dictionary returned by the get_package_and_deps function is modified as part of
# the resolve_npm function. This is why a deep copy is necessary.
expected_deps_info = copy.deepcopy(package_and_deps)
expected_deps_info["downloaded_deps"] = {
"@angular-devkit/[email protected]",
"@angular/[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
}
if shrink_wrap:
expected_deps_info["lock_file_name"] = "npm-shrinkwrap.json"
mock_exists.side_effect = [shrink_wrap, package_json]
else:
expected_deps_info["lock_file_name"] = "package-lock.json"
mock_exists.side_effect = [shrink_wrap, package_lock, package_json]
mock_gpad.return_value = package_and_deps
# Remove the "bundled" key as does the resolve_npm function to get expected returned
# dependencies
for dep in expected_deps_info["deps"]:
dep.pop("bundled")
dep.pop("version_in_nexus")
src_path = "/tmp/cachito-bundles/temp/1/app"
deps_info = npm.resolve_npm(src_path, {"id": 1})
assert deps_info == expected_deps_info
package_json_path = os.path.join(src_path, "package.json")
if shrink_wrap:
lock_file_path = os.path.join(src_path, "npm-shrinkwrap.json")
elif package_lock:
lock_file_path = os.path.join(src_path, "package-lock.json")
mock_gpad.assert_called_once_with(package_json_path, lock_file_path)
# We can't verify the actual correct deps value was passed in since the deps that were passed
# in were mutated and mock does not keep a deepcopy of the function arguments.
mock_dd.assert_called_once_with(RequestBundleDir(1).npm_deps_dir, mock.ANY, mock.ANY, mock.ANY)
def save_bundle_archive_checksum(request_id: int) -> None:
"""Compute and store bundle archive's checksum.
:param int request_id: the request id.
"""
bundle_dir = RequestBundleDir(request_id)
archive_file = bundle_dir.bundle_archive_file
if not archive_file.exists():
raise CachitoError(f"Bundle archive {archive_file} does not exist.")
checksum = hash_file(archive_file).hexdigest()
bundle_dir.bundle_archive_checksum.write_text(checksum, encoding="utf-8")
def test_resolve_gomod_no_deps(
mock_exists,
mock_makedirs,
mock_run,
mock_merge_tree,
mock_temp_dir,
mock_golang_version,
tmpdir,
sample_package,
sample_pkg_lvl_pkg,
):
# Ensure to create the gomod download cache directory
mock_exists.return_value = False
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
mock_run.side_effect = [
# go mod download
mock.Mock(returncode=0, stdout=None),
# go list -m all
mock.Mock(returncode=0,
stdout="github.com/release-engineering/retrodep/v2"),
# go list -find ./...
mock.Mock(returncode=0,
stdout="github.com/release-engineering/retrodep/v2"),
# go list -deps
mock.Mock(returncode=0, stdout=pkg_lvl_stdout),
]
mock_golang_version.return_value = "v2.1.1"
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
gomod = resolve_gomod(archive_path, request)
assert gomod["module"] == sample_package
assert not gomod["module_deps"]
assert len(gomod["packages"]) == 1
assert gomod["packages"][0]["pkg"] == sample_pkg_lvl_pkg
assert not gomod["packages"][0]["pkg_deps"]
# The second one ensures the source cache directory exists
mock_makedirs.assert_called_once_with(os.path.join(
tmpdir, RequestBundleDir.go_mod_cache_download_part),
exist_ok=True)
bundle_dir = RequestBundleDir(request["id"])
mock_merge_tree.assert_called_once_with(
os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part),
str(bundle_dir.gomod_download_dir),
)
def test_aggregate_packages_data(get_worker_config, set_request_state,
packages_data, expected, tmpdir):
get_worker_config.return_value.cachito_bundles_dir = tmpdir
request_id = 1
bundle_dir: RequestBundleDir = RequestBundleDir(request_id)
for pkg_manager, data in packages_data.items():
data_file = getattr(bundle_dir,
f"{pkg_manager.replace('-', '_')}_packages_data")
with open(data_file, "w", encoding="utf-8") as f:
json.dump(data, f)
tasks.aggregate_packages_data(request_id, list(packages_data.keys()))
set_request_state.assert_called_once_with(request_id, "in_progress",
"Aggregating packages data")
with open(bundle_dir.packages_data, "r", encoding="utf-8") as f:
assert expected == json.load(f)
def test_save_bundle_archive_checksum(get_worker_config, bundle_archive_exists,
tmpdir):
request_id = 1
get_worker_config.return_value = mock.Mock(cachito_bundles_dir=str(tmpdir))
if bundle_archive_exists:
bundle_dir = RequestBundleDir(request_id)
file_content = b"1234"
bundle_dir.bundle_archive_file.write_bytes(file_content)
save_bundle_archive_checksum(request_id)
expected_checksum = hash_file(
bundle_dir.bundle_archive_file).hexdigest()
assert expected_checksum == bundle_dir.bundle_archive_checksum.read_text(
encoding="utf-8")
else:
with pytest.raises(CachitoError,
match=r"Bundle archive .+ does not exist"):
save_bundle_archive_checksum(request_id)
def add_git_submodules_as_package(request_id):
"""
Add git submodules as package to the Cachtio request.
:param int request_id: the Cachito request ID this is for
:raises CachitoError: if adding submodules as a package fail.
"""
bundle_dir = RequestBundleDir(request_id)
repo = git.Repo(str(bundle_dir.source_root_dir))
packages_json_data = PackagesData()
for sm in repo.submodules:
# Save package to db
package = {
"type": "git-submodule",
"name": sm.name,
"version": f"{sm.url}#{sm.hexsha}",
}
log.debug("Adding submodule '%s' as a package for Cachito request",
sm.name)
packages_json_data.add_package(package, sm.path, [])
packages_json_data.write_to_file(bundle_dir.git_submodule_packages_data)
def aggregate_packages_data(request_id: int,
pkg_managers: List[str]) -> PackagesData:
"""Aggregate packages data generated for each package manager into one unified data file.
:param int request_id: the request id.
"""
set_request_state(request_id, "in_progress", "Aggregating packages data")
bundle_dir = RequestBundleDir(request_id)
aggregated_data = PackagesData()
for pkg_manager in pkg_managers:
# Ensure git-submodule -> git_submodule
data_file = getattr(bundle_dir,
f"{pkg_manager.replace('-', '_')}_packages_data")
aggregated_data.load(data_file)
log.debug("Write request %s packages data into %s", request_id,
bundle_dir.packages_data)
aggregated_data.write_to_file(str(bundle_dir.packages_data))
return aggregated_data
def add_git_submodules_as_package(request_id):
"""
Add git submodules as package to the Cachtio request.
:param int request_id: the Cachito request ID this is for
:raises CachitoError: if adding submodules as a package fail.
"""
bundle_dir = RequestBundleDir(request_id)
repo = git.Repo(str(bundle_dir.source_root_dir))
for sm in repo.submodules:
# Save package to db
package = {
"type": "git-submodule",
"name": sm.name,
"version": f"{sm.url}#{sm.hexsha}",
}
log.debug("Adding submodule '%s' as a package for Cachito request",
sm.name)
update_request_with_package(request_id,
package,
package_subpath=sm.path)
def test_resolve_gomod(
mock_run,
mock_merge_tree,
mock_temp_dir,
mock_golang_version,
dep_replacement,
go_list_error_pkg,
expected_replace,
tmpdir,
sample_deps,
sample_deps_replace,
sample_deps_replace_new_name,
sample_package,
):
mock_cmd_output = _generate_mock_cmd_output(go_list_error_pkg)
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
run_side_effects = []
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0,
stdout=None)) # go mod edit -replace
run_side_effects.append(mock.Mock(returncode=0,
stdout=None)) # go mod download
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0,
stdout=None)) # go mod tidy
run_side_effects.append(mock.Mock(
returncode=0, stdout=mock_cmd_output)) # go list -m all
run_side_effects.append(mock.Mock(
returncode=0, stdout=mock_pkg_list)) # go list -find ./...
run_side_effects.append(mock.Mock(returncode=0,
stdout=mock_pkg_deps)) # go list -deps
mock_run.side_effect = run_side_effects
mock_golang_version.return_value = "v2.1.1"
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
if dep_replacement is None:
gomod = resolve_gomod(archive_path, request)
expected_deps = sample_deps
else:
gomod = resolve_gomod(archive_path, request, [dep_replacement])
if dep_replacement.get("new_name"):
expected_deps = sample_deps_replace_new_name
else:
expected_deps = sample_deps_replace
if expected_replace:
assert mock_run.call_args_list[0][0][0] == (
"go",
"mod",
"edit",
"-replace",
expected_replace,
)
if dep_replacement:
assert mock_run.call_args_list[2][0][0] == ("go", "mod", "tidy")
assert gomod["module"] == sample_package
assert gomod["module_deps"] == expected_deps
mock_merge_tree.assert_called_once_with(
os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part),
str(RequestBundleDir(request["id"]).gomod_download_dir),
)
def test_resolve_yarn(
get_worker_config,
mock_replace_yarnlock,
mock_replace_packjson,
mock_set_proxy_urls,
mock_get_repo_name,
mock_download_deps,
mock_get_repo_url,
mock_get_package_and_deps,
have_nexus_replacements,
any_urls_in_yarn_lock,
tmpdir,
):
get_worker_config.return_value = mock.Mock(cachito_bundles_dir=str(tmpdir))
n_pop_calls = 0
def dict_pop_mocker():
expected_keys = ["version_in_nexus", "bundled"]
def mock_pop(key):
nonlocal n_pop_calls
n_pop_calls += 1
if expected_keys:
assert key == expected_keys.pop()
else:
assert False
return mock_pop
mock_package = mock.Mock()
mock_deps = [mock.Mock()]
for mock_dep in mock_deps:
mock_dep.pop.side_effect = dict_pop_mocker()
mock_package_json = mock.Mock()
mock_yarn_lock = mock.Mock()
mock_nexus_replacements = {"foo": {}} if have_nexus_replacements else {}
mock_get_package_and_deps.return_value = {
"package": mock_package,
"deps": mock_deps,
"package.json": mock_package_json,
"lock_file": mock_yarn_lock,
"nexus_replacements": mock_nexus_replacements,
}
if any_urls_in_yarn_lock:
mock_set_proxy_urls.return_value = True
expect_yarn_lock = mock_replace_yarnlock.return_value
else:
mock_set_proxy_urls.return_value = False
expect_yarn_lock = None
rv = yarn.resolve_yarn("/some/path", {"id": 1}, skip_deps={"foobar"})
assert rv == {
"package": mock_package,
"deps": mock_deps,
"downloaded_deps": mock_download_deps.return_value,
"package.json": mock_replace_packjson.return_value,
"lock_file": expect_yarn_lock,
}
mock_get_package_and_deps.assert_called_once_with(
Path("/some/path/package.json"), Path("/some/path/yarn.lock")
)
mock_get_repo_url.assert_called_once_with(1)
mock_download_deps.assert_called_once_with(
RequestBundleDir(1).yarn_deps_dir,
mock_deps,
mock_get_repo_url.return_value,
skip_deps={"foobar"},
pkg_manager="yarn",
)
assert n_pop_calls == len(mock_deps) * 2
if have_nexus_replacements:
mock_get_repo_name.assert_called_once_with(1)
mock_replace_packjson.assert_called_once_with(mock_package_json, mock_nexus_replacements)
mock_replace_yarnlock.assert_called_once_with(mock_yarn_lock, mock_nexus_replacements)
mock_set_proxy_urls.assert_called_once_with(
mock_replace_yarnlock.return_value, mock_get_repo_name.return_value
)
def test_fetch_pip_source(
mock_get_raw_asset_url,
mock_read,
mock_cert,
mock_update_cfg,
mock_update_pkg,
mock_update_deps,
mock_get_request,
mock_set_state,
mock_prepare_nexus,
mock_finalize_nexus,
mock_resolve,
with_cert,
with_req,
package_subpath,
tmp_path,
task_passes_state_check,
):
pkg_data = {
"package": {
"name": "foo",
"version": "1",
"type": "pip"
},
"dependencies": [{
"name": "bar",
"version": "2.0",
"type": "pip",
"dev": True
}],
"requirements": [],
}
request = {"id": 1}
username = f"cachito-pip-{request['id']}"
password = "******"
repo_name = f"cachito-pip-hosted-{request['id']}"
config = get_worker_config()
nexus_url = config.cachito_nexus_url
index_base_url = nexus_url.replace("://", f"://{username}:{password}@")
env_vars = {
"PIP_INDEX_URL": {
"value": f"{index_base_url}/repository/{repo_name}/simple",
"kind": "literal",
}
}
mock_cert.return_value = None
cert_contents = "stub_cert"
cfg_contents = []
if with_req:
requirements_path = (RequestBundleDir(1).app_subpath(
package_subpath or ".").source_dir / "requirements.txt")
pkg_data["requirements"].append(str(requirements_path))
mock_get_raw_asset_url.return_value = "http://fake-raw-asset-url.dev"
req_contents = f"mypkg @ git+https://www.github.com/cachito/mypkg.git@{'f'*40}?egg=mypkg\n"
mock_read.return_value = [req_contents]
b64_req_contents = base64.b64encode(
f"mypkg @ http://{username}:{password}@fake-raw-asset-url.dev".
encode()).decode()
requirements_relpath = requirements_path.relative_to(
RequestBundleDir(1))
cfg_contents.append({
"content": b64_req_contents,
"path": str(requirements_relpath),
"type": "base64"
})
if with_cert:
mock_cert.return_value = cert_contents
env_vars["PIP_CERT"] = {
"value": "app/package-index-ca.pem",
"kind": "path"
}
b64_cert_contents = base64.b64encode(cert_contents.encode()).decode()
cfg_contents.append({
"content": b64_cert_contents,
"path": "app/package-index-ca.pem",
"type": "base64"
})
mock_resolve.return_value = pkg_data
mock_finalize_nexus.return_value = password
mock_get_request.return_value = request
if package_subpath:
package_configs = [{"path": package_subpath}]
else:
package_configs = None
pip.fetch_pip_source(request["id"], package_configs=package_configs)
mock_update_pkg.assert_called_once_with(request["id"],
pkg_data["package"],
env_vars,
package_subpath=package_subpath
or ".")
mock_update_deps.assert_called_once_with(
request["id"],
pkg_data["package"],
[{
"name": "bar",
"version": "2.0",
"type": "pip",
"dev": True
}],
)
if cfg_contents:
mock_update_cfg.assert_called_once_with(
request["id"],
cfg_contents,
)
def resolve_gomod(app_source_path,
request,
dep_replacements=None,
git_dir_path=None):
"""
Resolve and fetch gomod dependencies for given app source archive.
:param str app_source_path: the full path to the application source code
:param dict request: the Cachito request this is for
:param list dep_replacements: dependency replacements with the keys "name" and "version"; this
results in a series of `go mod edit -replace` commands
:param dict git_dir_path: the full path to the application's git repository
:return: a dict containing the Go module itself ("module" key), the list of dictionaries
representing the dependencies ("module_deps" key), the top package level dependency
("pkg" key), and a list of dictionaries representing the package level dependencies
("pkg_deps" key)
:rtype: dict
:raises CachitoError: if fetching dependencies fails
"""
if git_dir_path is None:
git_dir_path = app_source_path
if not dep_replacements:
dep_replacements = []
worker_config = get_worker_config()
athens_url = worker_config.cachito_athens_url
with GoCacheTemporaryDirectory(prefix="cachito-") as temp_dir:
env = {
"GOPATH": temp_dir,
"GO111MODULE": "on",
"GOCACHE": temp_dir,
"GOPROXY": f"{athens_url}|{athens_url}",
"PATH": os.environ.get("PATH", ""),
"GOMODCACHE": "{}/pkg/mod".format(temp_dir),
}
if "cgo-disable" in request.get("flags", []):
env["CGO_ENABLED"] = "0"
run_params = {"env": env, "cwd": app_source_path}
# Collect all the dependency names that are being replaced to later verify if they were
# all used
replaced_dep_names = set()
for dep_replacement in dep_replacements:
name = dep_replacement["name"]
replaced_dep_names.add(name)
new_name = dep_replacement.get("new_name", name)
version = dep_replacement["version"]
log.info("Applying the gomod replacement %s => %s@%s", name,
new_name, version)
run_gomod_cmd(("go", "mod", "edit", "-replace",
f"{name}={new_name}@{version}"), run_params)
# Vendor dependencies if the gomod-vendor flag is set
flags = request.get("flags", [])
should_vendor, can_make_changes = _should_vendor_deps(
flags, app_source_path, worker_config.cachito_gomod_strict_vendor)
if should_vendor:
_vendor_deps(run_params, can_make_changes, git_dir_path)
else:
log.info("Downloading the gomod dependencies")
run_download_cmd(("go", "mod", "download"), run_params)
if dep_replacements:
run_gomod_cmd(("go", "mod", "tidy"), run_params)
# main module
module_name = run_gomod_cmd(["go", "list", "-m"], run_params).rstrip()
# module level dependencies
if should_vendor:
module_lines = _module_lines_from_modules_txt(app_source_path)
else:
# .String formats the module as <name> <version> [=> <replace>],
# where <replace> is <name> <version> or <path>
output_format = "{{ if not .Main }}{{ .String }}{{ end }}"
go_list_output = run_gomod_cmd(("go", "list", "-mod", "readonly",
"-m", "-f", output_format, "all"),
run_params)
module_lines = go_list_output.splitlines()
module_level_deps = []
# Keep track of which dependency replacements were actually applied to verify they were all
# used later
used_replaced_dep_names = set()
for line in module_lines:
parts = line.split(" ")
replaces = None
if len(parts) == 4 and parts[2] == "=>":
# If a Go module uses a "replace" directive to a local path, it will be shown as:
# k8s.io/metrics v0.0.0 => ./staging/src/k8s.io/metrics
# In this case, take the module name and the relative path, since that is the
# actual dependency being used.
parts = [parts[0], parts[-1]]
elif len(parts) == 5 and parts[2] == "=>":
# If a Go module uses a "replace" directive, then it will be in the format:
# github.com/pkg/errors v0.8.0 => github.com/pkg/errors v0.8.1
# In this case, just take the right side since that is the actual
# dependency being used
old_name, old_version = parts[:2]
# Only keep track of user provided replaces. There could be existing "replace"
# directives in the go.mod file, but they are an implementation detail specific to
# Go and they don't need to be recorded in Cachito.
if old_name in replaced_dep_names:
used_replaced_dep_names.add(old_name)
replaces = {
"type": "gomod",
"name": old_name,
"version": old_version
}
parts = parts[3:]
if len(parts) == 2:
module_level_deps.append({
"name": parts[0],
"replaces": replaces,
"type": "gomod",
"version": parts[1]
})
else:
log.warning("Unexpected go module output: %s", line)
unused_dep_replacements = replaced_dep_names - used_replaced_dep_names
if unused_dep_replacements:
raise CachitoError(
"The following gomod dependency replacements don't apply: "
f'{", ".join(unused_dep_replacements)}')
# NOTE: If there are multiple go modules in a single git repo, they will
# all be versioned identically.
module_version = get_golang_version(module_name,
git_dir_path,
request["ref"],
update_tags=True)
module = {
"name": module_name,
"type": "gomod",
"version": module_version
}
bundle_dir = RequestBundleDir(request["id"])
if "gomod-vendor" in flags:
# Create an empty gomod cache in the bundle directory so that any Cachito
# user does not have to guard against this directory not existing
bundle_dir.gomod_download_dir.mkdir(exist_ok=True, parents=True)
else:
# Add the gomod cache to the bundle the user will later download
tmp_download_cache_dir = os.path.join(
temp_dir, RequestBundleDir.go_mod_cache_download_part)
if not os.path.exists(tmp_download_cache_dir):
os.makedirs(tmp_download_cache_dir, exist_ok=True)
log.debug(
"Adding dependencies from %s to %s",
tmp_download_cache_dir,
bundle_dir.gomod_download_dir,
)
_merge_bundle_dirs(tmp_download_cache_dir,
str(bundle_dir.gomod_download_dir))
log.info("Retrieving the list of packages")
package_list = run_gomod_cmd(["go", "list", "-find", "./..."],
run_params).splitlines()
log.info("Retrieving the list of package level dependencies")
package_info = _load_list_deps(
run_gomod_cmd(["go", "list", "-e", "-deps", "-json", "./..."],
run_params))
packages = []
processed_pkg_deps = set()
for pkg_name in package_list:
if pkg_name in processed_pkg_deps:
# Go searches for packages in directories through a top-down approach. If a toplevel
# package is already listed as a dependency, we do not list it here, since its
# dependencies would also be listed in the parent package
log.debug(
"Package %s is already listed as a package dependency. Skipping...",
pkg_name)
continue
pkg_level_deps = []
for dep_name in package_info[pkg_name].get("Deps", []):
dep_info = package_info.get(dep_name)
if dep_info is None: # dependency is from the standard library
continue
processed_pkg_deps.add(dep_name)
# If the dependency does not have a version, we'll use the module version
version = _get_dep_version(dep_info) or module_version
pkg_level_deps.append({
"name": dep_name,
"type": "go-package",
"version": version
})
# Top-level packages always use the module version
pkg = {
"name": pkg_name,
"type": "go-package",
"version": module_version
}
packages.append({"pkg": pkg, "pkg_deps": pkg_level_deps})
allowlist = _get_allowed_local_deps(module_name)
log.debug("Allowed local dependencies for %s: %s", module_name,
allowlist)
_vet_local_deps(module_level_deps, module_name, allowlist)
for pkg in packages:
# Local dependencies are always relative to the main module, even for subpackages
_vet_local_deps(pkg["pkg_deps"], module_name, allowlist)
_set_full_local_dep_relpaths(pkg["pkg_deps"], module_level_deps)
return {
"module": module,
"module_deps": module_level_deps,
"packages": packages
}
def test_fetch_npm_source(
mock_update_request_env_vars,
mock_urwcf,
mock_gnc,
mock_gcc,
mock_fnfjr,
mock_rn,
mock_pnfjr,
mock_get_request,
mock_srs,
mock_vnf,
get_worker_config,
ca_file,
lock_file,
package_json,
package_subpath,
subpath_as_path_component,
reverse_path_component,
task_passes_state_check,
tmpdir,
):
get_worker_config.return_value = mock.Mock(cachito_bundles_dir=tmpdir)
request_id = 6
request = {"id": request_id}
mock_get_request.return_value = request
package = {"name": "han-solo", "type": "npm", "version": "5.0.0"}
deps = [
{"dev": False, "name": "@angular/animations", "type": "npm", "version": "8.2.14"},
{"dev": False, "name": "tslib", "type": "npm", "version": "1.11.1"},
]
mock_rn.return_value = {
"deps": deps,
"downloaded_deps": {"@angular/[email protected]", "[email protected]"},
"lock_file": lock_file,
"lock_file_name": "package-lock.json",
"package": package,
"package.json": package_json,
}
username = f"cachito-npm-{request_id}"
password = "******"
mock_fnfjr.return_value = password
mock_gcc.return_value = ca_file
mock_gnc.return_value = "some npmrc"
if package_subpath:
package_configs = [{"path": package_subpath}]
else:
package_configs = None
npm.fetch_npm_source(request_id, package_configs=package_configs)
bundle_dir = RequestBundleDir(request_id)
mock_vnf.assert_called_once_with(bundle_dir, [package_subpath or "."])
assert mock_srs.call_count == 3
assert mock_get_request.called_once_with(request_id)
mock_pnfjr.assert_called_once_with("cachito-npm-6")
lock_file_path = str(bundle_dir.app_subpath(package_subpath or ".").source_dir)
mock_rn.assert_called_once_with(lock_file_path, request, skip_deps=set())
if ca_file:
mock_gnc.assert_called_once_with(
"http://nexus:8081/repository/cachito-npm-6/",
username,
password,
custom_ca_path=f"{reverse_path_component}registry-ca.pem",
)
else:
mock_gnc.assert_called_once_with(
"http://nexus:8081/repository/cachito-npm-6/", username, password, custom_ca_path=None
)
expected_config_files = []
if package_json:
expected_config_files.append(
{
"content": "ewogICJuYW1lIjogImhhbi1zb2xvIgp9",
"path": f"app/{subpath_as_path_component}package.json",
"type": "base64",
}
)
if lock_file:
expected_config_files.append(
{
"content": "ewogICJkZXBlbmRlbmNpZXMiOiBbXQp9",
"path": f"app/{subpath_as_path_component}package-lock.json",
"type": "base64",
}
)
if ca_file:
expected_config_files.append(
{
"content": "c29tZSBDQSBmaWxlIGNvbnRlbnRz",
"path": "app/registry-ca.pem",
"type": "base64",
}
)
expected_config_files.append(
{
"content": "c29tZSBucG1yYw==",
"path": f"app/{subpath_as_path_component}.npmrc",
"type": "base64",
}
)
mock_urwcf.assert_called_once_with(request_id, expected_config_files)
mock_update_request_env_vars.assert_called_once_with(
request_id,
{
"CHROMEDRIVER_SKIP_DOWNLOAD": {"value": "true", "kind": "literal"},
"SKIP_SASS_BINARY_DOWNLOAD_FOR_CI": {"value": "true", "kind": "literal"},
},
)
pkg_info = package.copy()
pkg_info["dependencies"] = deps
if package_subpath and package_subpath != os.curdir:
pkg_info["path"] = package_subpath
assert {"packages": [pkg_info]} == json.loads(bundle_dir.npm_packages_data.read_bytes())
def fetch_npm_source(request_id, package_configs=None):
"""
Resolve and fetch npm dependencies for a given request.
This function uses the Python ``os.path`` library to manipulate paths, so the path to the
configuration files may differ in format based on the system the Cachito worker is deployed on
(i.e. Linux vs Windows).
:param int request_id: the Cachito request ID this is for
:param list package_configs: the list of optional package configurations submitted by the user
:raise CachitoError: if the task fails
"""
if package_configs is None:
package_configs = []
validate_npm_config()
bundle_dir = RequestBundleDir(request_id)
log.debug("Checking if the application source uses npm")
subpaths = [
os.path.normpath(c["path"]) for c in package_configs if c.get("path")
]
if not subpaths:
# Default to the root of the application source
subpaths = [os.curdir]
_verify_npm_files(bundle_dir, subpaths)
log.info("Configuring Nexus for npm for the request %d", request_id)
set_request_state(request_id, "in_progress", "Configuring Nexus for npm")
repo_name = get_npm_proxy_repo_name(request_id)
prepare_nexus_for_js_request(repo_name)
npm_config_files = []
downloaded_deps = set()
for i, subpath in enumerate(subpaths):
log.info("Fetching the npm dependencies for request %d in subpath %s",
request_id, subpath)
request = set_request_state(
request_id,
"in_progress",
f'Fetching the npm dependencies at the "{subpath}" directory"',
)
package_source_path = str(bundle_dir.app_subpath(subpath).source_dir)
try:
package_and_deps_info = resolve_npm(package_source_path,
request,
skip_deps=downloaded_deps)
except CachitoError:
log.exception("Failed to fetch npm dependencies for request %d",
request_id)
raise
downloaded_deps = downloaded_deps | package_and_deps_info[
"downloaded_deps"]
log.info(
"Generating the npm configuration files for request %d in subpath %s",
request_id,
subpath,
)
remote_package_source_path = os.path.normpath(
os.path.join("app", subpath))
if package_and_deps_info["package.json"]:
package_json_str = json.dumps(
package_and_deps_info["package.json"], indent=2)
npm_config_files.append({
"content":
base64.b64encode(
package_json_str.encode("utf-8")).decode("utf-8"),
"path":
os.path.join(remote_package_source_path, "package.json"),
"type":
"base64",
})
if package_and_deps_info["lock_file"]:
package_lock_str = json.dumps(package_and_deps_info["lock_file"],
indent=2)
lock_file_name = package_and_deps_info["lock_file_name"]
npm_config_files.append({
"content":
base64.b64encode(
package_lock_str.encode("utf-8")).decode("utf-8"),
"path":
os.path.join(remote_package_source_path, lock_file_name),
"type":
"base64",
})
if i == 0:
env_vars = get_worker_config(
).cachito_default_environment_variables.get("npm", {})
else:
env_vars = None
package = package_and_deps_info["package"]
update_request_with_package(request_id,
package,
env_vars,
package_subpath=subpath)
update_request_with_deps(request_id, package,
package_and_deps_info["deps"])
log.info("Finalizing the Nexus configuration for npm for the request %d",
request_id)
set_request_state(request_id, "in_progress",
"Finalizing the Nexus configuration for npm")
username = get_npm_proxy_username(request_id)
password = finalize_nexus_for_js_request(username, repo_name)
log.info("Generating the .npmrc file(s)")
ca_cert = nexus.get_ca_cert()
if ca_cert:
# The custom CA will be called registry-ca.pem in the "app" directory
npm_config_files.append({
"content":
base64.b64encode(ca_cert.encode("utf-8")).decode("utf-8"),
"path":
os.path.join("app", "registry-ca.pem"),
"type":
"base64",
})
for subpath in subpaths:
proxy_repo_url = get_npm_proxy_repo_url(request_id)
if ca_cert:
# Determine the relative path to the registry-ca.pem file
custom_ca_path = os.path.relpath("registry-ca.pem", start=subpath)
else:
custom_ca_path = None
npm_rc = generate_npmrc_content(proxy_repo_url,
username,
password,
custom_ca_path=custom_ca_path)
npm_config_files.append({
"content":
base64.b64encode(npm_rc.encode("utf-8")).decode("utf-8"),
"path":
os.path.normpath(os.path.join("app", subpath, ".npmrc")),
"type":
"base64",
})
update_request_with_config_files(request_id, npm_config_files)
def resolve_npm(app_source_path, request, skip_deps=None):
"""
Resolve and fetch npm dependencies for the given app source archive.
:param str app_source_path: the full path to the application source code
:param dict request: the Cachito request this is for
:param set skip_deps: a set of dependency identifiers to not download because they've already
been downloaded for this request
:return: a dictionary that has the following keys:
``deps`` which is the list of dependencies,
``downloaded_deps`` which is a set of the dependency identifiers of the dependencies that
were downloaded as part of this function's execution,
``lock_file`` which is the lock file if it was modified,
``lock_file_name`` is the name of the lock file that was used,
``package`` which is the dictionary describing the main package, and
``package.json`` which is the package.json file if it was modified.
:rtype: dict
:raises CachitoError: if fetching the dependencies fails or required files are missing
:raises ValidationError: if lock file does not have the correct format
"""
# npm-shrinkwrap.json and package-lock.json share the same format but serve slightly
# different purposes. See the following documentation for more information:
# https://docs.npmjs.com/files/package-lock.json.
for lock_file in ("npm-shrinkwrap.json", "package-lock.json"):
package_lock_path = os.path.join(app_source_path, lock_file)
if os.path.exists(package_lock_path):
break
else:
raise CachitoError(
"The npm-shrinkwrap.json or package-lock.json file must be present for the npm "
"package manager"
)
package_json_path = os.path.join(app_source_path, "package.json")
if not os.path.exists(package_json_path):
raise CachitoError("The package.json file must be present for the npm package manager")
try:
package_and_deps_info = get_package_and_deps(package_json_path, package_lock_path)
except KeyError as e:
msg = f"The lock file {lock_file} has an unexpected format (missing key: {e})"
log.exception(msg)
raise ValidationError(msg)
package_and_deps_info["lock_file_name"] = lock_file
# By downloading the dependencies, it stores the tarballs in the bundle and also stages the
# content in the npm repository for the request
proxy_repo_url = get_npm_proxy_repo_url(request["id"])
bundle_dir = RequestBundleDir(request["id"])
bundle_dir.npm_deps_dir.mkdir(exist_ok=True)
package_and_deps_info["downloaded_deps"] = download_dependencies(
bundle_dir.npm_deps_dir, package_and_deps_info["deps"], proxy_repo_url, skip_deps,
)
# Remove all the "bundled" keys since that is an implementation detail that should not be
# exposed outside of this function
for dep in package_and_deps_info["deps"]:
dep.pop("bundled")
dep.pop("version_in_nexus")
return package_and_deps_info
def fetch_yarn_source(request_id: int, package_configs: List[dict] = None):
"""
Resolve and fetch yarn dependencies for a given request.
This function uses the Python ``os.path`` library to manipulate paths, so the path to the
configuration files may differ in format based on the system the Cachito worker is deployed on
(i.e. Linux vs Windows).
:param int request_id: the Cachito request ID this is for
:param list package_configs: the list of optional package configurations submitted by the user
:raise CachitoError: if the task fails
"""
if package_configs is None:
package_configs = []
validate_yarn_config()
bundle_dir = RequestBundleDir(request_id)
subpaths = [
os.path.normpath(c["path"]) for c in package_configs if c.get("path")
]
if not subpaths:
# Default to the root of the application source
subpaths = [os.curdir]
_verify_yarn_files(bundle_dir, subpaths)
log.info("Configuring Nexus for yarn for the request %d", request_id)
set_request_state(request_id, "in_progress", "Configuring Nexus for yarn")
repo_name = get_yarn_proxy_repo_name(request_id)
prepare_nexus_for_js_request(repo_name)
yarn_config_files = []
downloaded_deps = set()
for i, subpath in enumerate(subpaths):
log.info("Fetching the yarn dependencies for request %d in subpath %s",
request_id, subpath)
set_request_state(
request_id,
"in_progress",
f'Fetching the yarn dependencies at the "{subpath}" directory',
)
request = get_request(request_id)
package_source_path = str(bundle_dir.app_subpath(subpath).source_dir)
try:
package_and_deps_info = resolve_yarn(package_source_path,
request,
skip_deps=downloaded_deps)
except CachitoError:
log.exception("Failed to fetch yarn dependencies for request %d",
request_id)
raise
downloaded_deps = downloaded_deps | package_and_deps_info[
"downloaded_deps"]
log.info(
"Generating the yarn configuration files for request %d in subpath %s",
request_id,
subpath,
)
remote_package_source_path = os.path.normpath(
os.path.join("app", subpath))
if package_and_deps_info["package.json"]:
package_json_str = json.dumps(
package_and_deps_info["package.json"], indent=2)
package_json_path = os.path.join(remote_package_source_path,
"package.json")
yarn_config_files.append(
make_base64_config_file(package_json_str, package_json_path))
if package_and_deps_info["lock_file"]:
yarn_lock_str = _yarn_lock_to_str(
package_and_deps_info["lock_file"])
yarn_lock_path = os.path.join(remote_package_source_path,
"yarn.lock")
yarn_config_files.append(
make_base64_config_file(yarn_lock_str, yarn_lock_path))
if i == 0:
default_env = get_worker_config(
).cachito_default_environment_variables
env_vars = {
**default_env.get("npm", {}),
**default_env.get("yarn", {})
}
else:
env_vars = None
package = package_and_deps_info["package"]
update_request_with_package(request_id,
package,
env_vars,
package_subpath=subpath)
update_request_with_deps(request_id, package,
package_and_deps_info["deps"])
log.info("Finalizing the Nexus configuration for yarn for the request %d",
request_id)
set_request_state(request_id, "in_progress",
"Finalizing the Nexus configuration for yarn")
username = get_yarn_proxy_repo_username(request_id)
password = finalize_nexus_for_js_request(username, repo_name)
log.info("Generating the .npmrc file(s)")
proxy_repo_url = get_yarn_proxy_repo_url(request_id)
yarn_config_files.extend(
generate_npmrc_config_files(proxy_repo_url, username, password,
subpaths))
log.info("Adding empty .yarnrc file(s)")
for subpath in subpaths:
yarnrc_path = os.path.normpath(os.path.join("app", subpath, ".yarnrc"))
yarn_config_files.append(make_base64_config_file("", yarnrc_path))
update_request_with_config_files(request_id, yarn_config_files)
def test_download_dependencies(
mock_gwc,
mock_move,
mock_run_cmd,
mock_gawnf,
mock_exists,
mock_td,
nexus_ca_cert_exists,
pkg_manager,
tmpdir,
):
bundles_dir = tmpdir.mkdir("bundles")
mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir)
mock_gwc.return_value.cachito_nexus_ca_cert = "/etc/cachito/nexus_ca.pem"
mock_td_path = tmpdir.mkdir("cachito-agfdsk")
mock_td.return_value.__enter__.return_value = str(mock_td_path)
mock_exists.return_value = nexus_ca_cert_exists
mock_run_cmd.return_value = textwrap.dedent("""\
angular-devkit-architect-0.803.26.tgz
angular-animations-8.2.14.tgz
rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz
exsp-2.10.2-external-sha512-abcdefg.tar.gz
""")
deps = [
{
"bundled": False,
"dev": True,
"name": "@angular-devkit/architect",
"version": "0.803.26",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "@angular/animations",
"version": "8.2.14",
"version_in_nexus": None,
},
{
"bundled": True,
"dev": True,
"name": "object-assign",
"version": "4.1.1",
"version_in_nexus": None,
},
{
"bundled":
False,
"dev":
False,
"name":
"rxjs",
"version":
"github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30",
"version_in_nexus":
"6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
},
{
"bundled": False,
"dev": False,
"name": "jsplumb",
"version": "file:../jsplumb-2.10.2.tgz",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "exsp",
"version": "https://github.com/exsp/exsp/archive/2.10.2.tar.gz",
"version_in_nexus": "2.10.2-external-sha512-abcdefg",
},
]
request_id = 1
request_bundle_dir = bundles_dir.mkdir("temp").mkdir(str(request_id))
deps_path = os.path.join(request_bundle_dir, f"deps/{pkg_manager}")
proxy_repo_url = npm.get_npm_proxy_repo_url(request_id)
general_js.download_dependencies(request_id,
deps,
proxy_repo_url,
pkg_manager=pkg_manager)
mock_npm_rc_path = str(mock_td_path.join(".npmrc"))
if nexus_ca_cert_exists:
mock_gawnf.assert_called_once_with(
mock_npm_rc_path,
"http://*****:*****@angular-devkit/[email protected]",
"@angular/[email protected]",
"rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
"[email protected]",
]
assert mock_run_cmd.call_args[0][0] == expected_npm_pack
run_cmd_env_vars = mock_run_cmd.call_args[0][1]["env"]
assert run_cmd_env_vars["NPM_CONFIG_CACHE"] == str(
mock_td_path.join("cache"))
assert run_cmd_env_vars["NPM_CONFIG_USERCONFIG"] == mock_npm_rc_path
assert mock_run_cmd.call_args[0][1]["cwd"] == f"{deps_path}"
dep1_source_path = RequestBundleDir(
f"{deps_path}/angular-devkit-architect-0.803.26.tgz")
dep1_dest_path = RequestBundleDir(
f"{deps_path}/@angular-devkit/architect/angular-devkit-architect-0.803.26.tgz"
)
dep2_source_path = RequestBundleDir(
f"{deps_path}/angular-animations-8.2.14.tgz")
dep2_dest_path = RequestBundleDir(
f"{deps_path}/@angular/animations/angular-animations-8.2.14.tgz")
dep3_source_path = RequestBundleDir(
f"{deps_path}/rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz"
)
dep3_dest_path = RequestBundleDir(
f"{deps_path}/github/ReactiveX/rxjs/rxjs-6.5.5-external-gitcommit-"
"78032157f5c1655436829017bbda787565b48c30.tgz")
dep4_source_path = RequestBundleDir(
f"{deps_path}/exsp-2.10.2-external-sha512-abcdefg.tar.gz")
dep4_dest_path = RequestBundleDir(
f"{deps_path}/external-exsp/exsp-2.10.2-external-sha512-abcdefg.tar.gz"
)
mock_move.assert_has_calls([
mock.call(dep1_source_path, dep1_dest_path),
mock.call(dep2_source_path, dep2_dest_path),
mock.call(dep3_source_path, dep3_dest_path),
mock.call(dep4_source_path, dep4_dest_path),
])
def download_dependencies(request_id, deps, proxy_repo_url, skip_deps=None, pkg_manager="npm"):
"""
Download the list of npm dependencies using npm pack to the deps bundle directory.
By downloading the dependencies, this stages the content in the request specific npm proxy.
Any dependency that has the key "bundled" set to ``True`` will not be downloaded. This is
because the dependency is bundled as part of another dependency, and thus already present in
the tarball of the dependency that bundles it.
:param int request_id: the ID of the request these dependencies are being downloaded for
:param list deps: a list of dependencies where each dependency has the keys: bundled, name,
version, and version_in_nexus
:param str proxy_repo_url: the Nexus proxy repository URL to use as the registry
:param set skip_deps: a set of dependency identifiers to not download because they've already
been downloaded for this request
:param str pkg_manager: the name of the package manager to download dependencies for, affects
destination directory and logging output (npm is used to do the actual download regardless)
:return: a set of dependency identifiers that were downloaded
:rtype: set
:raises CachitoError: if any of the downloads fail
"""
if skip_deps is None:
skip_deps = set()
conf = get_worker_config()
with tempfile.TemporaryDirectory(prefix="cachito-") as temp_dir:
npm_rc_file = os.path.join(temp_dir, ".npmrc")
if conf.cachito_nexus_ca_cert and os.path.exists(conf.cachito_nexus_ca_cert):
nexus_ca = conf.cachito_nexus_ca_cert
else:
nexus_ca = None
# The token must be privileged so that it has access to the cachito-js repository
generate_and_write_npmrc_file(
npm_rc_file,
proxy_repo_url,
conf.cachito_nexus_username,
conf.cachito_nexus_password,
custom_ca_path=nexus_ca,
)
env = {
# This is set since the home directory must be determined by the HOME environment
# variable or by looking at the /etc/passwd file. The latter does not always work
# since some deployments (e.g. OpenShift) don't have an entry for the running user
# in /etc/passwd.
"HOME": os.environ.get("HOME", ""),
"NPM_CONFIG_CACHE": os.path.join(temp_dir, "cache"),
# This should not be necessary since all the dependencies come from Nexus, but it's an
# extra precaution
"NPM_CONFIG_IGNORE_SCRIPTS": "true",
"NPM_CONFIG_USERCONFIG": npm_rc_file,
"PATH": os.environ.get("PATH", ""),
}
bundle_dir = RequestBundleDir(request_id)
if pkg_manager == "npm":
deps_download_dir = bundle_dir.npm_deps_dir
elif pkg_manager == "yarn":
deps_download_dir = bundle_dir.yarn_deps_dir
else:
raise ValueError(f"Invalid package manager: {pkg_manager!r}")
deps_download_dir.mkdir(exist_ok=True)
# Download the dependencies directly in the bundle directory
run_params = {"env": env, "cwd": str(deps_download_dir)}
log.info("Processing %d %s dependencies to stage in Nexus", len(deps), pkg_manager)
downloaded_deps = set()
# This must be done in batches to prevent Nexus from erroring with "Header is too large"
deps_batches = []
counter = 0
batch_size = get_worker_config().cachito_js_download_batch_size
for dep in deps:
external_dep_version = None
if dep.get("version_in_nexus"):
version = dep["version_in_nexus"]
external_dep_version = dep["version"]
else:
version = dep["version"]
dep_identifier = f"{dep['name']}@{version}"
if dep["bundled"]:
log.debug("Not downloading %s since it is a bundled dependency", dep_identifier)
continue
elif dep["version"].startswith("file:"):
log.debug("Not downloading %s since it is a file dependency", dep_identifier)
continue
elif dep_identifier in skip_deps:
log.debug(
"Not downloading %s since it was already downloaded previously", dep_identifier
)
continue
if counter % batch_size == 0:
deps_batches.append([])
deps_batches[-1].append((dep_identifier, external_dep_version))
downloaded_deps.add(dep_identifier)
counter += 1
for dep_batch in deps_batches:
# Create a list of dependencies to be downloaded. Excluding 'external_dep_version'
# from the list of tuples
dep_batch_download = [i[0] for i in dep_batch]
log.debug(
"Downloading the following %s dependencies: %s",
pkg_manager,
", ".join(dep_batch_download),
)
npm_pack_args = ["npm", "pack"] + dep_batch_download
output = run_cmd(
npm_pack_args, run_params, f"Failed to download the {pkg_manager} dependencies"
)
# Move dependencies to their respective folders
# Iterate through the tuples made of dependency tarball and dep_identifier
# e.g. ('ab-2.10.2-external-sha512-ab.tar.gz', ('[email protected]',
# 'https://github.com/ab/2.10.2.tar.gz'))
for dep_pair in list(zip(output.split("\n"), dep_batch)):
tarball = dep_pair[0] # e.g. ab-2.10.2-external-sha512-ab.tar.gz
dep_indentifer = dep_pair[1][0] # [email protected]
dir_path = dep_indentifer.rsplit("@", 1)[0] # ab
external_dep_version = dep_pair[1][1] # https://github.com/ab/2.10.2.tar.gz
# In case of external dependencies, create additional intermediate
# parent e.g. github/<org>/<repo> or external-<repo>
if external_dep_version:
known_git_host_match = re.match(
r"^(?P<host>.+)(?::)(?!//)(?P<repo_path>.+)(?:#.+)$", external_dep_version
)
if known_git_host_match:
# This means external_dep_version is in the format of
# <git-host>:<namespace>/<repo>#<commit>
groups = known_git_host_match.groupdict()
dir_path = os.path.join(groups["host"], *groups["repo_path"].split("/"))
else:
dir_path = f"external-{dir_path}"
# Create the target directory for the dependency
dep_dir = deps_download_dir.joinpath(*dir_path.split("/", 1))
dep_dir.mkdir(exist_ok=True, parents=True)
# Move the dependency into the target directory
shutil.move(deps_download_dir.joinpath(tarball), dep_dir.joinpath(tarball))
return downloaded_deps
def test_resolve_gomod(
mock_run,
mock_get_worker_config,
mock_set_full_relpaths,
mock_vet_local_deps,
mock_get_allowed_local_deps,
mock_merge_tree,
mock_temp_dir,
mock_golang_version,
dep_replacement,
go_list_error_pkg,
expected_replace,
cgo_disable,
tmpdir,
sample_deps,
sample_deps_replace,
sample_deps_replace_new_name,
sample_package,
sample_pkg_deps_without_replace,
):
mock_cmd_output = _generate_mock_cmd_output(go_list_error_pkg)
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = str(tmpdir)
# Mock the "subprocess.run" calls
run_side_effects = []
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod edit -replace
run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod download
if dep_replacement:
run_side_effects.append(mock.Mock(returncode=0, stdout=None)) # go mod tidy
run_side_effects.append(
mock.Mock(returncode=0, stdout="github.com/release-engineering/retrodep/v2") # go list -m
)
run_side_effects.append(mock.Mock(returncode=0, stdout=mock_cmd_output)) # go list -m all
run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_list)) # go list -find ./...
run_side_effects.append(mock.Mock(returncode=0, stdout=mock_pkg_deps)) # go list -deps -json
mock_run.side_effect = run_side_effects
mock_golang_version.return_value = "v2.1.1"
mock_get_allowed_local_deps.return_value = ["*"]
archive_path = "/this/is/path/to/archive.tar.gz"
request = {"id": 3, "ref": "c50b93a32df1c9d700e3e80996845bc2e13be848"}
if cgo_disable:
request["flags"] = ["cgo-disable"]
if dep_replacement is None:
gomod = resolve_gomod(archive_path, request)
expected_deps = sample_deps
else:
gomod = resolve_gomod(archive_path, request, [dep_replacement])
if dep_replacement.get("new_name"):
expected_deps = sample_deps_replace_new_name
else:
expected_deps = sample_deps_replace
if expected_replace:
assert mock_run.call_args_list[0][0][0] == (
"go",
"mod",
"edit",
"-replace",
expected_replace,
)
if dep_replacement:
assert mock_run.call_args_list[2][0][0] == ("go", "mod", "tidy")
for call in mock_run.call_args_list:
env = call.kwargs["env"]
if cgo_disable:
assert env["CGO_ENABLED"] == "0"
else:
assert "CGO_ENABLED" not in env
assert gomod["module"] == sample_package
assert gomod["module_deps"] == expected_deps
assert len(gomod["packages"]) == 1
if dep_replacement is None:
assert (
sorted(gomod["packages"][0]["pkg_deps"], key=_package_sort_key)
== sample_pkg_deps_without_replace
)
mock_merge_tree.assert_called_once_with(
os.path.join(tmpdir, RequestBundleDir.go_mod_cache_download_part),
str(RequestBundleDir(request["id"]).gomod_download_dir),
)
expect_module_name = sample_package["name"]
mock_get_allowed_local_deps.assert_called_once_with(expect_module_name)
mock_vet_local_deps.assert_has_calls(
[
mock.call(expected_deps, expect_module_name, ["*"]),
mock.call(gomod["packages"][0]["pkg_deps"], expect_module_name, ["*"]),
],
)
mock_set_full_relpaths.assert_called_once_with(gomod["packages"][0]["pkg_deps"], expected_deps)
def resolve_gomod(app_source_path,
request,
dep_replacements=None,
git_dir_path=None):
"""
Resolve and fetch gomod dependencies for given app source archive.
:param str app_source_path: the full path to the application source code
:param dict request: the Cachito request this is for
:param list dep_replacements: dependency replacements with the keys "name" and "version"; this
results in a series of `go mod edit -replace` commands
:param dict git_dir_path: the full path to the application's git repository
:return: a dict containing the Go module itself ("module" key), the list of dictionaries
representing the dependencies ("module_deps" key), the top package level dependency
("pkg" key), and a list of dictionaries representing the package level dependencies
("pkg_deps" key)
:rtype: dict
:raises CachitoError: if fetching dependencies fails
"""
if git_dir_path is None:
git_dir_path = app_source_path
if not dep_replacements:
dep_replacements = []
worker_config = get_worker_config()
with GoCacheTemporaryDirectory(prefix="cachito-") as temp_dir:
env = {
"GOPATH": temp_dir,
"GO111MODULE": "on",
"GOCACHE": temp_dir,
"GOPROXY": worker_config.cachito_athens_url,
"PATH": os.environ.get("PATH", ""),
"GOMODCACHE": "{}/pkg/mod".format(temp_dir),
}
run_params = {"env": env, "cwd": app_source_path}
# Collect all the dependency names that are being replaced to later verify if they were
# all used
replaced_dep_names = set()
for dep_replacement in dep_replacements:
name = dep_replacement["name"]
replaced_dep_names.add(name)
new_name = dep_replacement.get("new_name", name)
version = dep_replacement["version"]
log.info("Applying the gomod replacement %s => %s@%s", name,
new_name, version)
run_gomod_cmd(("go", "mod", "edit", "-replace",
f"{name}={new_name}@{version}"), run_params)
# Vendor dependencies if the gomod-vendor flag is set
flags = request.get("flags", [])
if "gomod-vendor" in flags:
log.info("Vendoring the gomod dependencies")
run_gomod_cmd(("go", "mod", "vendor"), run_params)
elif worker_config.cachito_gomod_strict_vendor and os.path.isdir(
os.path.join(app_source_path, "vendor")):
raise CachitoError(
'The "gomod-vendor" flag must be set when your repository has vendored'
" dependencies.")
else:
log.info("Downloading the gomod dependencies")
run_gomod_cmd(("go", "mod", "download"), run_params)
if dep_replacements:
run_gomod_cmd(("go", "mod", "tidy"), run_params)
# module level dependencies
output_format = "{{.Path}} {{.Version}} {{.Replace}}"
go_list_output = run_gomod_cmd(("go", "list", "-mod", "readonly", "-m",
"-f", output_format, "all"),
run_params)
module_level_deps = []
module_name = None
# Keep track of which dependency replacements were actually applied to verify they were all
# used later
used_replaced_dep_names = set()
go_module_name_error = "The Go module name could not be determined"
for line in go_list_output.splitlines():
# If there is no "replace" directive used on the dependency, then the last column will
# be "<nil>"
parts = [
part for part in line.split(" ") if part not in ("", "<nil>")
]
if len(parts) == 1:
# This is the application itself, not a dependency
if module_name is not None:
log.error(
'go list produced two lines which look like module names: "%s" and "%s"',
module_name,
parts[0],
)
raise CachitoError(go_module_name_error)
module_name = parts[0]
continue
replaces = None
if len(parts) == 3:
# If a Go module uses a "replace" directive to a local path, it will be shown as:
# k8s.io/metrics v0.0.0 ./staging/src/k8s.io/metrics
# In this case, just take the left side.
parts = parts[0:2]
elif len(parts) == 4:
# If a Go module uses a "replace" directive, then it will be in the format:
# github.com/pkg/errors v0.8.0 github.com/pkg/errors v0.8.1
# In this case, just take the right side since that is the actual
# dependency being used
old_name, old_version = parts[0], parts[1]
# Only keep track of user provided replaces. There could be existing "replace"
# directives in the go.mod file, but they are an implementation detail specific to
# Go and they don't need to be recorded in Cachito.
if old_name in replaced_dep_names:
used_replaced_dep_names.add(old_name)
replaces = {
"type": "gomod",
"name": old_name,
"version": old_version
}
parts = parts[2:]
if len(parts) == 2:
module_level_deps.append({
"name": parts[0],
"replaces": replaces,
"type": "gomod",
"version": parts[1]
})
else:
log.warning("Unexpected go module output: %s", line)
unused_dep_replacements = replaced_dep_names - used_replaced_dep_names
if unused_dep_replacements:
raise CachitoError(
"The following gomod dependency replacements don't apply: "
f'{", ".join(unused_dep_replacements)}')
if not module_name:
# This should never occur, but it's here as a precaution
raise CachitoError(go_module_name_error)
# NOTE: If there are multiple go modules in a single git repo, they will
# all be versioned identically.
module_version = get_golang_version(module_name,
git_dir_path,
request["ref"],
update_tags=True)
module = {
"name": module_name,
"type": "gomod",
"version": module_version
}
bundle_dir = RequestBundleDir(request["id"])
if "gomod-vendor" in flags:
# Create an empty gomod cache in the bundle directory so that any Cachito
# user does not have to guard against this directory not existing
bundle_dir.gomod_download_dir.mkdir(exist_ok=True, parents=True)
else:
# Add the gomod cache to the bundle the user will later download
tmp_download_cache_dir = os.path.join(
temp_dir, RequestBundleDir.go_mod_cache_download_part)
if not os.path.exists(tmp_download_cache_dir):
os.makedirs(tmp_download_cache_dir, exist_ok=True)
log.debug(
"Adding dependencies from %s to %s",
tmp_download_cache_dir,
bundle_dir.gomod_download_dir,
)
_merge_bundle_dirs(tmp_download_cache_dir,
str(bundle_dir.gomod_download_dir))
log.info("Retrieving the list of package level dependencies")
list_pkgs_cmd = ("go", "list", "-find", "./...")
go_list_pkgs_output = run_gomod_cmd(list_pkgs_cmd, run_params)
packages = []
processed_pkg_deps = set()
for package in go_list_pkgs_output.splitlines():
if package in processed_pkg_deps:
# Go searches for packages in directories through a top-down approach. If a toplevel
# package is already listed as a dependency, we do not list it here, since its
# dependencies would also be listed in the parent package
log.debug(
"Package %s is already listed as a package dependency. Skipping...",
package)
continue
list_deps_cmd = (
"go",
"list",
"-deps",
"-f",
"{{if not .Standard}}{{.ImportPath}} {{.Module}}{{end}}",
package,
)
go_list_deps_output = run_gomod_cmd(list_deps_cmd, run_params)
pkg_level_deps = []
for line in go_list_deps_output.splitlines():
name, version = _parse_name_and_version(line)
# If the line did not contain a version, we'll use the module version
version = version or module_version
if version.startswith("."):
raise CachitoError(
f"Local gomod dependencies are not yet supported: {version}"
)
elif version.startswith("/") or PureWindowsPath(version).root:
# This will disallow paths starting with '/', '\' or '<drive letter>:\'
raise CachitoError(
f"Absolute paths to gomod dependencies are not supported: {version}"
)
pkg = {
"name": name,
"type": "go-package",
"version": version,
}
processed_pkg_deps.add(name)
pkg_level_deps.append(pkg)
# The last item on `go list -deps` is the main package being evaluated
pkg = pkg_level_deps.pop()
packages.append({"pkg": pkg, "pkg_deps": pkg_level_deps})
return {
"module": module,
"module_deps": module_level_deps,
"packages": packages
}
def fetch_gomod_source(request_id, dep_replacements=None, package_configs=None):
"""
Resolve and fetch gomod dependencies for a given request.
:param int request_id: the Cachito request ID this is for
:param list dep_replacements: dependency replacements with the keys "name" and "version"; only
supported with a single path
:param list package_configs: the list of optional package configurations submitted by the user
:raises CachitoError: if the dependencies could not be retrieved
"""
version_output = run_cmd(["go", "version"], {})
log.info(f"Go version: {version_output.strip()}")
config = get_worker_config()
if package_configs is None:
package_configs = []
bundle_dir: RequestBundleDir = RequestBundleDir(request_id)
subpaths = [os.path.normpath(c["path"]) for c in package_configs if c.get("path")]
if not subpaths:
# Default to the root of the application source
subpaths = [os.curdir]
invalid_gomod_files = _find_missing_gomod_files(bundle_dir, subpaths)
if invalid_gomod_files:
invalid_files_print = "; ".join(invalid_gomod_files)
file_suffix = "s" if len(invalid_gomod_files) > 1 else ""
# missing gomod files is supported if there is only one path referenced
if config.cachito_gomod_ignore_missing_gomod_file and len(subpaths) == 1:
log.warning("go.mod file missing for request at %s", invalid_files_print)
return
raise CachitoError(
"The {} file{} must be present for the gomod package manager".format(
invalid_files_print.strip(), file_suffix
)
)
if len(subpaths) > 1 and dep_replacements:
raise CachitoError(
"Dependency replacements are only supported for a single go module path."
)
env_vars = {
"GOCACHE": {"value": "deps/gomod", "kind": "path"},
"GOPATH": {"value": "deps/gomod", "kind": "path"},
"GOMODCACHE": {"value": "deps/gomod/pkg/mod", "kind": "path"},
}
env_vars.update(config.cachito_default_environment_variables.get("gomod", {}))
update_request_env_vars(request_id, env_vars)
packages_json_data = PackagesData()
for i, subpath in enumerate(subpaths):
log.info(
"Fetching the gomod dependencies for request %d in subpath %s", request_id, subpath
)
set_request_state(
request_id,
"in_progress",
f'Fetching the gomod dependencies at the "{subpath}" directory',
)
request = get_request(request_id)
gomod_source_path = str(bundle_dir.app_subpath(subpath).source_dir)
try:
gomod = resolve_gomod(
gomod_source_path, request, dep_replacements, bundle_dir.source_dir
)
except CachitoError:
log.exception("Failed to fetch gomod dependencies for request %d", request_id)
raise
module_info = gomod["module"]
packages_json_data.add_package(module_info, subpath, gomod["module_deps"])
# add package deps
for package in gomod["packages"]:
pkg_info = package["pkg"]
package_subpath = _package_subpath(module_info["name"], pkg_info["name"], subpath)
packages_json_data.add_package(pkg_info, package_subpath, package.get("pkg_deps", []))
packages_json_data.write_to_file(bundle_dir.gomod_packages_data)
def fetch_pip_source(request_id, package_configs=None):
"""
Resolve and fetch pip dependencies for a given request.
:param int request_id: the Cachito request ID this is for
:param list package_configs: the list of optional package configurations submitted by the user
"""
version_output = run_cmd(["pip", "--version"], {})
log.info(f"pip version: {version_output.strip()}")
validate_pip_config()
bundle_dir: RequestBundleDir = RequestBundleDir(request_id)
log.info("Configuring Nexus for pip for the request %d", request_id)
set_request_state(request_id, "in_progress", "Configuring Nexus for pip")
pip_repo_name = get_pypi_hosted_repo_name(request_id)
raw_repo_name = get_raw_hosted_repo_name(request_id)
prepare_nexus_for_pip_request(pip_repo_name, raw_repo_name)
log.info("Fetching dependencies for request %d", request_id)
package_configs = package_configs or [{}]
packages_data = []
requirement_file_paths = []
for pkg_cfg in package_configs:
pkg_path = pkg_cfg.get("path", ".")
source_dir = bundle_dir.app_subpath(pkg_path).source_dir
set_request_state(
request_id,
"in_progress",
f"Fetching dependencies at the {pkg_path!r} directory",
)
request = get_request(request_id)
pkg_and_deps_info = resolve_pip(
source_dir,
request,
requirement_files=pkg_cfg.get("requirements_files"),
build_requirement_files=pkg_cfg.get("requirements_build_files"),
)
# defer custom requirement files creation to use the Nexus password in the URLs
for requirement_file_path in pkg_and_deps_info.pop("requirements"):
requirement_file_paths.append(requirement_file_path)
# defer DB operations to use the Nexus password in the env vars
packages_data.append(pkg_and_deps_info)
log.info("Finalizing the Nexus configuration for pip for the request %d",
request_id)
set_request_state(request_id, "in_progress",
"Finalizing the Nexus configuration for pip")
username = get_hosted_repositories_username(request_id)
password = finalize_nexus_for_pip_request(pip_repo_name, raw_repo_name,
username)
# Set environment variables and config files
pip_config_files = []
for requirement_file_path in requirement_file_paths:
custom_requirement_file = _get_custom_requirement_config_file(
requirement_file_path, bundle_dir.source_root_dir, raw_repo_name,
username, password)
if custom_requirement_file:
pip_config_files.append(custom_requirement_file)
raw_url = get_pypi_hosted_repo_url(request_id)
pip_index_url = get_index_url(raw_url, username, password)
env_vars = {"PIP_INDEX_URL": {"value": pip_index_url, "kind": "literal"}}
ca_cert = nexus.get_ca_cert()
if ca_cert:
ca_cert_path = os.path.join("app", "package-index-ca.pem")
env_vars["PIP_CERT"] = {"value": ca_cert_path, "kind": "path"}
pip_config_files.append(make_base64_config_file(ca_cert, ca_cert_path))
worker_config = get_worker_config()
env_vars.update(
worker_config.cachito_default_environment_variables.get("pip", {}))
update_request_env_vars(request_id, env_vars)
packages_json_data = PackagesData()
for pkg_cfg, pkg_data in zip(package_configs, packages_data):
pkg_subpath = os.path.normpath(pkg_cfg.get("path", "."))
pkg_info = pkg_data["package"]
pkg_deps = pkg_data["dependencies"]
packages_json_data.add_package(pkg_info, pkg_subpath, pkg_deps)
packages_json_data.write_to_file(bundle_dir.pip_packages_data)
if pip_config_files:
update_request_with_config_files(request_id, pip_config_files)
def fetch_npm_source(request_id, package_configs=None):
"""
Resolve and fetch npm dependencies for a given request.
This function uses the Python ``os.path`` library to manipulate paths, so the path to the
configuration files may differ in format based on the system the Cachito worker is deployed on
(i.e. Linux vs Windows).
:param int request_id: the Cachito request ID this is for
:param list package_configs: the list of optional package configurations submitted by the user
:raise CachitoError: if the task fails
"""
version_output = run_cmd(["npm", "--version"], {})
log.info(f"npm version: {version_output.strip()}")
version_output = run_cmd(["node", "--version"], {})
log.info(f"Node.js version: {version_output.strip()}")
if package_configs is None:
package_configs = []
validate_npm_config()
bundle_dir: RequestBundleDir = RequestBundleDir(request_id)
log.debug("Checking if the application source uses npm")
subpaths = [
os.path.normpath(c["path"]) for c in package_configs if c.get("path")
]
if not subpaths:
# Default to the root of the application source
subpaths = [os.curdir]
_verify_npm_files(bundle_dir, subpaths)
log.info("Configuring Nexus for npm for the request %d", request_id)
set_request_state(request_id, "in_progress", "Configuring Nexus for npm")
repo_name = get_npm_proxy_repo_name(request_id)
prepare_nexus_for_js_request(repo_name)
npm_config_files = []
downloaded_deps = set()
packages_json_data = PackagesData()
for i, subpath in enumerate(subpaths):
log.info("Fetching the npm dependencies for request %d in subpath %s",
request_id, subpath)
set_request_state(
request_id,
"in_progress",
f'Fetching the npm dependencies at the "{subpath}" directory"',
)
request = get_request(request_id)
package_source_path = str(bundle_dir.app_subpath(subpath).source_dir)
try:
package_and_deps_info = resolve_npm(package_source_path,
request,
skip_deps=downloaded_deps)
except CachitoError:
log.exception("Failed to fetch npm dependencies for request %d",
request_id)
raise
downloaded_deps = downloaded_deps | package_and_deps_info[
"downloaded_deps"]
log.info(
"Generating the npm configuration files for request %d in subpath %s",
request_id,
subpath,
)
remote_package_source_path = os.path.normpath(
os.path.join("app", subpath))
if package_and_deps_info["package.json"]:
package_json_str = json.dumps(
package_and_deps_info["package.json"], indent=2)
package_json_path = os.path.join(remote_package_source_path,
"package.json")
npm_config_files.append(
make_base64_config_file(package_json_str, package_json_path))
if package_and_deps_info["lock_file"]:
package_lock_str = json.dumps(package_and_deps_info["lock_file"],
indent=2)
lock_file_name = package_and_deps_info["lock_file_name"]
lock_file_path = os.path.join(remote_package_source_path,
lock_file_name)
npm_config_files.append(
make_base64_config_file(package_lock_str, lock_file_path))
if i == 0:
env_vars = get_worker_config(
).cachito_default_environment_variables.get("npm", {})
update_request_env_vars(request_id, env_vars)
pkg_info = package_and_deps_info["package"]
pkg_deps = package_and_deps_info["deps"]
packages_json_data.add_package(pkg_info, subpath, pkg_deps)
packages_json_data.write_to_file(bundle_dir.npm_packages_data)
log.info("Finalizing the Nexus configuration for npm for the request %d",
request_id)
set_request_state(request_id, "in_progress",
"Finalizing the Nexus configuration for npm")
username = get_npm_proxy_username(request_id)
password = finalize_nexus_for_js_request(username, repo_name)
log.info("Generating the .npmrc file(s)")
proxy_repo_url = get_npm_proxy_repo_url(request_id)
npm_config_files.extend(
generate_npmrc_config_files(proxy_repo_url, username, password,
subpaths))
update_request_with_config_files(request_id, npm_config_files)