def test_download_dependencies_skip_deps(
mock_gwc,
mock_move,
mock_run_cmd,
mock_gawnf,
mock_exists,
mock_td,
tmpdir,
):
bundles_dir = tmpdir.mkdir("bundles")
mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir)
mock_td.return_value.__enter__.return_value = str(
tmpdir.mkdir("cachito-agfdsk"))
mock_exists.return_value = False
mock_run_cmd.return_value = textwrap.dedent("""\
angular-devkit-architect-0.803.26.tgz
rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz
""")
deps = [
{
"bundled": False,
"dev": True,
"name": "@angular-devkit/architect",
"version": "0.803.26",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "@angular/animations",
"version": "8.2.14",
"version_in_nexus": None,
},
{
"bundled":
False,
"dev":
False,
"name":
"rxjs",
"version":
"github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30",
"version_in_nexus":
"6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
},
]
proxy_repo_url = npm.get_npm_proxy_repo_url(1)
general_js.download_dependencies(1, deps, proxy_repo_url,
{"@angular/[email protected]"})
mock_run_cmd.assert_called_once()
# This ensures that the skipped dependency is not downloaded
expected_npm_pack = [
"npm",
"pack",
"@angular-devkit/[email protected]",
"rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
]
assert mock_run_cmd.call_args[0][0] == expected_npm_pack
def test_download_dependencies(
mock_gwc,
mock_move,
mock_run_cmd,
mock_gawnf,
mock_exists,
mock_td,
nexus_ca_cert_exists,
pkg_manager,
tmpdir,
):
bundles_dir = tmpdir.mkdir("bundles")
mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir)
mock_gwc.return_value.cachito_nexus_ca_cert = "/etc/cachito/nexus_ca.pem"
mock_td_path = tmpdir.mkdir("cachito-agfdsk")
mock_td.return_value.__enter__.return_value = str(mock_td_path)
mock_exists.return_value = nexus_ca_cert_exists
mock_run_cmd.return_value = textwrap.dedent("""\
angular-devkit-architect-0.803.26.tgz
angular-animations-8.2.14.tgz
rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz
exsp-2.10.2-external-sha512-abcdefg.tar.gz
""")
deps = [
{
"bundled": False,
"dev": True,
"name": "@angular-devkit/architect",
"version": "0.803.26",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "@angular/animations",
"version": "8.2.14",
"version_in_nexus": None,
},
{
"bundled": True,
"dev": True,
"name": "object-assign",
"version": "4.1.1",
"version_in_nexus": None,
},
{
"bundled":
False,
"dev":
False,
"name":
"rxjs",
"version":
"github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30",
"version_in_nexus":
"6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
},
{
"bundled": False,
"dev": False,
"name": "jsplumb",
"version": "file:../jsplumb-2.10.2.tgz",
"version_in_nexus": None,
},
{
"bundled": False,
"dev": False,
"name": "exsp",
"version": "https://github.com/exsp/exsp/archive/2.10.2.tar.gz",
"version_in_nexus": "2.10.2-external-sha512-abcdefg",
},
]
request_id = 1
request_bundle_dir = bundles_dir.mkdir("temp").mkdir(str(request_id))
deps_path = os.path.join(request_bundle_dir, f"deps/{pkg_manager}")
proxy_repo_url = npm.get_npm_proxy_repo_url(request_id)
general_js.download_dependencies(request_id,
deps,
proxy_repo_url,
pkg_manager=pkg_manager)
mock_npm_rc_path = str(mock_td_path.join(".npmrc"))
if nexus_ca_cert_exists:
mock_gawnf.assert_called_once_with(
mock_npm_rc_path,
"http://*****:*****@angular-devkit/[email protected]",
"@angular/[email protected]",
"rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30",
"[email protected]",
]
assert mock_run_cmd.call_args[0][0] == expected_npm_pack
run_cmd_env_vars = mock_run_cmd.call_args[0][1]["env"]
assert run_cmd_env_vars["NPM_CONFIG_CACHE"] == str(
mock_td_path.join("cache"))
assert run_cmd_env_vars["NPM_CONFIG_USERCONFIG"] == mock_npm_rc_path
assert mock_run_cmd.call_args[0][1]["cwd"] == f"{deps_path}"
dep1_source_path = RequestBundleDir(
f"{deps_path}/angular-devkit-architect-0.803.26.tgz")
dep1_dest_path = RequestBundleDir(
f"{deps_path}/@angular-devkit/architect/angular-devkit-architect-0.803.26.tgz"
)
dep2_source_path = RequestBundleDir(
f"{deps_path}/angular-animations-8.2.14.tgz")
dep2_dest_path = RequestBundleDir(
f"{deps_path}/@angular/animations/angular-animations-8.2.14.tgz")
dep3_source_path = RequestBundleDir(
f"{deps_path}/rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz"
)
dep3_dest_path = RequestBundleDir(
f"{deps_path}/github/ReactiveX/rxjs/rxjs-6.5.5-external-gitcommit-"
"78032157f5c1655436829017bbda787565b48c30.tgz")
dep4_source_path = RequestBundleDir(
f"{deps_path}/exsp-2.10.2-external-sha512-abcdefg.tar.gz")
dep4_dest_path = RequestBundleDir(
f"{deps_path}/external-exsp/exsp-2.10.2-external-sha512-abcdefg.tar.gz"
)
mock_move.assert_has_calls([
mock.call(dep1_source_path, dep1_dest_path),
mock.call(dep2_source_path, dep2_dest_path),
mock.call(dep3_source_path, dep3_dest_path),
mock.call(dep4_source_path, dep4_dest_path),
])
def test_get_npm_proxy_repo_url():
assert npm.get_npm_proxy_repo_url(3).endswith("/repository/cachito-npm-3/")
def fetch_npm_source(request_id, package_configs=None):
"""
Resolve and fetch npm dependencies for a given request.
This function uses the Python ``os.path`` library to manipulate paths, so the path to the
configuration files may differ in format based on the system the Cachito worker is deployed on
(i.e. Linux vs Windows).
:param int request_id: the Cachito request ID this is for
:param list package_configs: the list of optional package configurations submitted by the user
:raise CachitoError: if the task fails
"""
if package_configs is None:
package_configs = []
validate_npm_config()
bundle_dir = RequestBundleDir(request_id)
log.debug("Checking if the application source uses npm")
subpaths = [
os.path.normpath(c["path"]) for c in package_configs if c.get("path")
]
if not subpaths:
# Default to the root of the application source
subpaths = [os.curdir]
_verify_npm_files(bundle_dir, subpaths)
log.info("Configuring Nexus for npm for the request %d", request_id)
set_request_state(request_id, "in_progress", "Configuring Nexus for npm")
repo_name = get_npm_proxy_repo_name(request_id)
prepare_nexus_for_js_request(repo_name)
npm_config_files = []
downloaded_deps = set()
for i, subpath in enumerate(subpaths):
log.info("Fetching the npm dependencies for request %d in subpath %s",
request_id, subpath)
set_request_state(
request_id,
"in_progress",
f'Fetching the npm dependencies at the "{subpath}" directory"',
)
request = get_request(request_id)
package_source_path = str(bundle_dir.app_subpath(subpath).source_dir)
try:
package_and_deps_info = resolve_npm(package_source_path,
request,
skip_deps=downloaded_deps)
except CachitoError:
log.exception("Failed to fetch npm dependencies for request %d",
request_id)
raise
downloaded_deps = downloaded_deps | package_and_deps_info[
"downloaded_deps"]
log.info(
"Generating the npm configuration files for request %d in subpath %s",
request_id,
subpath,
)
remote_package_source_path = os.path.normpath(
os.path.join("app", subpath))
if package_and_deps_info["package.json"]:
package_json_str = json.dumps(
package_and_deps_info["package.json"], indent=2)
package_json_path = os.path.join(remote_package_source_path,
"package.json")
npm_config_files.append(
make_base64_config_file(package_json_str, package_json_path))
if package_and_deps_info["lock_file"]:
package_lock_str = json.dumps(package_and_deps_info["lock_file"],
indent=2)
lock_file_name = package_and_deps_info["lock_file_name"]
lock_file_path = os.path.join(remote_package_source_path,
lock_file_name)
npm_config_files.append(
make_base64_config_file(package_lock_str, lock_file_path))
if i == 0:
env_vars = get_worker_config(
).cachito_default_environment_variables.get("npm", {})
else:
env_vars = None
package = package_and_deps_info["package"]
update_request_with_package(request_id,
package,
env_vars,
package_subpath=subpath)
update_request_with_deps(request_id, package,
package_and_deps_info["deps"])
log.info("Finalizing the Nexus configuration for npm for the request %d",
request_id)
set_request_state(request_id, "in_progress",
"Finalizing the Nexus configuration for npm")
username = get_npm_proxy_username(request_id)
password = finalize_nexus_for_js_request(username, repo_name)
log.info("Generating the .npmrc file(s)")
proxy_repo_url = get_npm_proxy_repo_url(request_id)
npm_config_files.extend(
generate_npmrc_config_files(proxy_repo_url, username, password,
subpaths))
update_request_with_config_files(request_id, npm_config_files)
