def test_download_dependencies_skip_deps(
mock_gwc,
mock_move,
mock_run_cmd,
mock_gawnf,
mock_exists,
mock_td,
tmpdir,
):
bundles_dir = tmpdir.mkdir("bundles")
mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir)
mock_td.return_value.__enter__.return_value = str(
tmpdir.mkdir("cachito-agfdsk"))
mock_exists.return_value = False
mock_run_cmd.return_value = textwrap.dedent("""\
angular-devkit-architect-0.803.26.tgz
rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz
""")
deps = [
{
"bundled": False,
"dev": True,
"name": "@angular-devkit/architect",
"version": "0.803.26",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "@angular/animations",
"version": "8.2.14",
"version_in_nexus": None,
},
{
"bundled":
False,
"dev":
False,
"name":
"rxjs",
"version":
"github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30",
"version_in_nexus":
"6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
},
]
proxy_repo_url = npm.get_npm_proxy_repo_url(1)
general_js.download_dependencies(1, deps, proxy_repo_url,
{"@angular/[email protected]"})
mock_run_cmd.assert_called_once()
# This ensures that the skipped dependency is not downloaded
expected_npm_pack = [
"npm",
"pack",
"@angular-devkit/[email protected]",
"rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
]
assert mock_run_cmd.call_args[0][0] == expected_npm_pack
def resolve_npm(app_source_path, request, skip_deps=None):
"""
Resolve and fetch npm dependencies for the given app source archive.
:param str app_source_path: the full path to the application source code
:param dict request: the Cachito request this is for
:param set skip_deps: a set of dependency identifiers to not download because they've already
been downloaded for this request
:return: a dictionary that has the following keys:
``deps`` which is the list of dependencies,
``downloaded_deps`` which is a set of the dependency identifiers of the dependencies that
were downloaded as part of this function's execution,
``lock_file`` which is the lock file if it was modified,
``lock_file_name`` is the name of the lock file that was used,
``package`` which is the dictionary describing the main package, and
``package.json`` which is the package.json file if it was modified.
:rtype: dict
:raises CachitoError: if fetching the dependencies fails or required files are missing
"""
# npm-shrinkwrap.json and package-lock.json share the same format but serve slightly
# different purposes. See the following documentation for more information:
# https://docs.npmjs.com/files/package-lock.json.
for lock_file in ("npm-shrinkwrap.json", "package-lock.json"):
package_lock_path = os.path.join(app_source_path, lock_file)
if os.path.exists(package_lock_path):
break
else:
raise CachitoError(
"The npm-shrinkwrap.json or package-lock.json file must be present for the npm "
"package manager")
package_json_path = os.path.join(app_source_path, "package.json")
if not os.path.exists(package_json_path):
raise CachitoError(
"The package.json file must be present for the npm package manager"
)
try:
package_and_deps_info = get_package_and_deps(package_json_path,
package_lock_path)
except KeyError:
msg = f"The lock file {lock_file} has an unexpected format"
log.exception(msg)
raise CachitoError(msg)
package_and_deps_info["lock_file_name"] = lock_file
# By downloading the dependencies, it stores the tarballs in the bundle and also stages the
# content in the npm repository for the request
proxy_repo_url = get_npm_proxy_repo_url(request["id"])
package_and_deps_info["downloaded_deps"] = download_dependencies(
request["id"], package_and_deps_info["deps"], proxy_repo_url,
skip_deps)
# Remove all the "bundled" keys since that is an implementation detail that should not be
# exposed outside of this function
for dep in package_and_deps_info["deps"]:
dep.pop("bundled")
dep.pop("version_in_nexus")
return package_and_deps_info
def resolve_yarn(app_source_path, request, skip_deps=None):
"""
Resolve and fetch npm dependencies for the given app source archive.
:param (str | Path) app_source_path: the full path to the application source code
:param dict request: the Cachito request this is for
:param set skip_deps: a set of dependency identifiers to not download because they've already
been downloaded for this request
:return: a dictionary that has the following keys:
``deps`` which is the list of dependencies,
``downloaded_deps`` which is a set of the dependency identifiers of the dependencies that
were downloaded as part of this function's execution,
``lock_file`` which is the lock file if it was modified,
``package`` which is the dictionary describing the main package, and
``package.json`` which is the package.json file if it was modified.
:rtype: dict
:raises CachitoError: if fetching the dependencies fails or required files are missing
"""
app_source_path = Path(app_source_path)
package_json_path = app_source_path / "package.json"
yarn_lock_path = app_source_path / "yarn.lock"
package_and_deps_info = _get_package_and_deps(package_json_path,
yarn_lock_path)
# By downloading the dependencies, it stores the tarballs in the bundle and also stages the
# content in the yarn repository for the request
proxy_repo_url = get_yarn_proxy_repo_url(request["id"])
bundle_dir = RequestBundleDir(request["id"])
bundle_dir.yarn_deps_dir.mkdir(exist_ok=True)
package_and_deps_info["downloaded_deps"] = download_dependencies(
bundle_dir.yarn_deps_dir,
package_and_deps_info["deps"],
proxy_repo_url,
skip_deps=skip_deps,
pkg_manager="yarn",
)
replacements = package_and_deps_info.pop("nexus_replacements")
pkg_json = _replace_deps_in_package_json(
package_and_deps_info["package.json"], replacements)
yarn_lock = _replace_deps_in_yarn_lock(package_and_deps_info["lock_file"],
replacements)
package_and_deps_info["package.json"] = pkg_json
if _set_proxy_resolved_urls(yarn_lock,
get_yarn_proxy_repo_name(request["id"])):
package_and_deps_info["lock_file"] = yarn_lock
else:
package_and_deps_info["lock_file"] = None
# Remove all the "bundled" and "version_in_nexus" keys since they are implementation details
for dep in package_and_deps_info["deps"]:
dep.pop("bundled")
dep.pop("version_in_nexus")
return package_and_deps_info
def test_download_dependencies(
mock_gwc,
mock_move,
mock_run_cmd,
mock_gawnf,
mock_exists,
mock_td,
nexus_ca_cert_exists,
pkg_manager,
tmpdir,
):
bundles_dir = tmpdir.mkdir("bundles")
mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir)
mock_gwc.return_value.cachito_nexus_ca_cert = "/etc/cachito/nexus_ca.pem"
mock_td_path = tmpdir.mkdir("cachito-agfdsk")
mock_td.return_value.__enter__.return_value = str(mock_td_path)
mock_exists.return_value = nexus_ca_cert_exists
mock_run_cmd.return_value = textwrap.dedent("""\
angular-devkit-architect-0.803.26.tgz
angular-animations-8.2.14.tgz
rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz
exsp-2.10.2-external-sha512-abcdefg.tar.gz
""")
deps = [
{
"bundled": False,
"dev": True,
"name": "@angular-devkit/architect",
"version": "0.803.26",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "@angular/animations",
"version": "8.2.14",
"version_in_nexus": None,
},
{
"bundled": True,
"dev": True,
"name": "object-assign",
"version": "4.1.1",
"version_in_nexus": None,
},
{
"bundled":
False,
"dev":
False,
"name":
"rxjs",
"version":
"github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30",
"version_in_nexus":
"6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
},
{
"bundled": False,
"dev": False,
"name": "jsplumb",
"version": "file:../jsplumb-2.10.2.tgz",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "exsp",
"version": "https://github.com/exsp/exsp/archive/2.10.2.tar.gz",
"version_in_nexus": "2.10.2-external-sha512-abcdefg",
},
]
request_id = 1
request_bundle_dir = bundles_dir.mkdir("temp").mkdir(str(request_id))
deps_path = os.path.join(request_bundle_dir, f"deps/{pkg_manager}")
proxy_repo_url = npm.get_npm_proxy_repo_url(request_id)
general_js.download_dependencies(request_id,
deps,
proxy_repo_url,
pkg_manager=pkg_manager)
mock_npm_rc_path = str(mock_td_path.join(".npmrc"))
if nexus_ca_cert_exists:
mock_gawnf.assert_called_once_with(
mock_npm_rc_path,
"http://*****:*****@angular-devkit/[email protected]",
"@angular/[email protected]",
"rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
"[email protected]",
]
assert mock_run_cmd.call_args[0][0] == expected_npm_pack
run_cmd_env_vars = mock_run_cmd.call_args[0][1]["env"]
assert run_cmd_env_vars["NPM_CONFIG_CACHE"] == str(
mock_td_path.join("cache"))
assert run_cmd_env_vars["NPM_CONFIG_USERCONFIG"] == mock_npm_rc_path
assert mock_run_cmd.call_args[0][1]["cwd"] == f"{deps_path}"
dep1_source_path = RequestBundleDir(
f"{deps_path}/angular-devkit-architect-0.803.26.tgz")
dep1_dest_path = RequestBundleDir(
f"{deps_path}/@angular-devkit/architect/angular-devkit-architect-0.803.26.tgz"
)
dep2_source_path = RequestBundleDir(
f"{deps_path}/angular-animations-8.2.14.tgz")
dep2_dest_path = RequestBundleDir(
f"{deps_path}/@angular/animations/angular-animations-8.2.14.tgz")
dep3_source_path = RequestBundleDir(
f"{deps_path}/rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz"
)
dep3_dest_path = RequestBundleDir(
f"{deps_path}/github/ReactiveX/rxjs/rxjs-6.5.5-external-gitcommit-"
"78032157f5c1655436829017bbda787565b48c30.tgz")
dep4_source_path = RequestBundleDir(
f"{deps_path}/exsp-2.10.2-external-sha512-abcdefg.tar.gz")
dep4_dest_path = RequestBundleDir(
f"{deps_path}/external-exsp/exsp-2.10.2-external-sha512-abcdefg.tar.gz"
)
mock_move.assert_has_calls([
mock.call(dep1_source_path, dep1_dest_path),
mock.call(dep2_source_path, dep2_dest_path),
mock.call(dep3_source_path, dep3_dest_path),
mock.call(dep4_source_path, dep4_dest_path),
])