def test_download_dependencies_skip_deps( mock_gwc, mock_move, mock_run_cmd, mock_gawnf, mock_exists, mock_td, tmpdir, ): bundles_dir = tmpdir.mkdir("bundles") mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir) mock_td.return_value.__enter__.return_value = str( tmpdir.mkdir("cachito-agfdsk")) mock_exists.return_value = False mock_run_cmd.return_value = textwrap.dedent("""\ angular-devkit-architect-0.803.26.tgz rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz """) deps = [ { "bundled": False, "dev": True, "name": "@angular-devkit/architect", "version": "0.803.26", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "@angular/animations", "version": "8.2.14", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "rxjs", "version": "github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30", "version_in_nexus": "6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", }, ] proxy_repo_url = npm.get_npm_proxy_repo_url(1) general_js.download_dependencies(1, deps, proxy_repo_url, {"@angular/[email protected]"}) mock_run_cmd.assert_called_once() # This ensures that the skipped dependency is not downloaded expected_npm_pack = [ "npm", "pack", "@angular-devkit/[email protected]", "rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", ] assert mock_run_cmd.call_args[0][0] == expected_npm_pack
def resolve_npm(app_source_path, request, skip_deps=None): """ Resolve and fetch npm dependencies for the given app source archive. :param str app_source_path: the full path to the application source code :param dict request: the Cachito request this is for :param set skip_deps: a set of dependency identifiers to not download because they've already been downloaded for this request :return: a dictionary that has the following keys: ``deps`` which is the list of dependencies, ``downloaded_deps`` which is a set of the dependency identifiers of the dependencies that were downloaded as part of this function's execution, ``lock_file`` which is the lock file if it was modified, ``lock_file_name`` is the name of the lock file that was used, ``package`` which is the dictionary describing the main package, and ``package.json`` which is the package.json file if it was modified. :rtype: dict :raises CachitoError: if fetching the dependencies fails or required files are missing """ # npm-shrinkwrap.json and package-lock.json share the same format but serve slightly # different purposes. See the following documentation for more information: # https://docs.npmjs.com/files/package-lock.json. for lock_file in ("npm-shrinkwrap.json", "package-lock.json"): package_lock_path = os.path.join(app_source_path, lock_file) if os.path.exists(package_lock_path): break else: raise CachitoError( "The npm-shrinkwrap.json or package-lock.json file must be present for the npm " "package manager") package_json_path = os.path.join(app_source_path, "package.json") if not os.path.exists(package_json_path): raise CachitoError( "The package.json file must be present for the npm package manager" ) try: package_and_deps_info = get_package_and_deps(package_json_path, package_lock_path) except KeyError: msg = f"The lock file {lock_file} has an unexpected format" log.exception(msg) raise CachitoError(msg) package_and_deps_info["lock_file_name"] = lock_file # By downloading the dependencies, it stores the tarballs in the bundle and also stages the # content in the npm repository for the request proxy_repo_url = get_npm_proxy_repo_url(request["id"]) package_and_deps_info["downloaded_deps"] = download_dependencies( request["id"], package_and_deps_info["deps"], proxy_repo_url, skip_deps) # Remove all the "bundled" keys since that is an implementation detail that should not be # exposed outside of this function for dep in package_and_deps_info["deps"]: dep.pop("bundled") dep.pop("version_in_nexus") return package_and_deps_info
def resolve_yarn(app_source_path, request, skip_deps=None): """ Resolve and fetch npm dependencies for the given app source archive. :param (str | Path) app_source_path: the full path to the application source code :param dict request: the Cachito request this is for :param set skip_deps: a set of dependency identifiers to not download because they've already been downloaded for this request :return: a dictionary that has the following keys: ``deps`` which is the list of dependencies, ``downloaded_deps`` which is a set of the dependency identifiers of the dependencies that were downloaded as part of this function's execution, ``lock_file`` which is the lock file if it was modified, ``package`` which is the dictionary describing the main package, and ``package.json`` which is the package.json file if it was modified. :rtype: dict :raises CachitoError: if fetching the dependencies fails or required files are missing """ app_source_path = Path(app_source_path) package_json_path = app_source_path / "package.json" yarn_lock_path = app_source_path / "yarn.lock" package_and_deps_info = _get_package_and_deps(package_json_path, yarn_lock_path) # By downloading the dependencies, it stores the tarballs in the bundle and also stages the # content in the yarn repository for the request proxy_repo_url = get_yarn_proxy_repo_url(request["id"]) bundle_dir = RequestBundleDir(request["id"]) bundle_dir.yarn_deps_dir.mkdir(exist_ok=True) package_and_deps_info["downloaded_deps"] = download_dependencies( bundle_dir.yarn_deps_dir, package_and_deps_info["deps"], proxy_repo_url, skip_deps=skip_deps, pkg_manager="yarn", ) replacements = package_and_deps_info.pop("nexus_replacements") pkg_json = _replace_deps_in_package_json( package_and_deps_info["package.json"], replacements) yarn_lock = _replace_deps_in_yarn_lock(package_and_deps_info["lock_file"], replacements) package_and_deps_info["package.json"] = pkg_json if _set_proxy_resolved_urls(yarn_lock, get_yarn_proxy_repo_name(request["id"])): package_and_deps_info["lock_file"] = yarn_lock else: package_and_deps_info["lock_file"] = None # Remove all the "bundled" and "version_in_nexus" keys since they are implementation details for dep in package_and_deps_info["deps"]: dep.pop("bundled") dep.pop("version_in_nexus") return package_and_deps_info
def test_download_dependencies( mock_gwc, mock_move, mock_run_cmd, mock_gawnf, mock_exists, mock_td, nexus_ca_cert_exists, pkg_manager, tmpdir, ): bundles_dir = tmpdir.mkdir("bundles") mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir) mock_gwc.return_value.cachito_nexus_ca_cert = "/etc/cachito/nexus_ca.pem" mock_td_path = tmpdir.mkdir("cachito-agfdsk") mock_td.return_value.__enter__.return_value = str(mock_td_path) mock_exists.return_value = nexus_ca_cert_exists mock_run_cmd.return_value = textwrap.dedent("""\ angular-devkit-architect-0.803.26.tgz angular-animations-8.2.14.tgz rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz exsp-2.10.2-external-sha512-abcdefg.tar.gz """) deps = [ { "bundled": False, "dev": True, "name": "@angular-devkit/architect", "version": "0.803.26", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "@angular/animations", "version": "8.2.14", "version_in_nexus": None, }, { "bundled": True, "dev": True, "name": "object-assign", "version": "4.1.1", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "rxjs", "version": "github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30", "version_in_nexus": "6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", }, { "bundled": False, "dev": False, "name": "jsplumb", "version": "file:../jsplumb-2.10.2.tgz", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "exsp", "version": "https://github.com/exsp/exsp/archive/2.10.2.tar.gz", "version_in_nexus": "2.10.2-external-sha512-abcdefg", }, ] request_id = 1 request_bundle_dir = bundles_dir.mkdir("temp").mkdir(str(request_id)) deps_path = os.path.join(request_bundle_dir, f"deps/{pkg_manager}") proxy_repo_url = npm.get_npm_proxy_repo_url(request_id) general_js.download_dependencies(request_id, deps, proxy_repo_url, pkg_manager=pkg_manager) mock_npm_rc_path = str(mock_td_path.join(".npmrc")) if nexus_ca_cert_exists: mock_gawnf.assert_called_once_with( mock_npm_rc_path, "http://*****:*****@angular-devkit/[email protected]", "@angular/[email protected]", "rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", "[email protected]", ] assert mock_run_cmd.call_args[0][0] == expected_npm_pack run_cmd_env_vars = mock_run_cmd.call_args[0][1]["env"] assert run_cmd_env_vars["NPM_CONFIG_CACHE"] == str( mock_td_path.join("cache")) assert run_cmd_env_vars["NPM_CONFIG_USERCONFIG"] == mock_npm_rc_path assert mock_run_cmd.call_args[0][1]["cwd"] == f"{deps_path}" dep1_source_path = RequestBundleDir( f"{deps_path}/angular-devkit-architect-0.803.26.tgz") dep1_dest_path = RequestBundleDir( f"{deps_path}/@angular-devkit/architect/angular-devkit-architect-0.803.26.tgz" ) dep2_source_path = RequestBundleDir( f"{deps_path}/angular-animations-8.2.14.tgz") dep2_dest_path = RequestBundleDir( f"{deps_path}/@angular/animations/angular-animations-8.2.14.tgz") dep3_source_path = RequestBundleDir( f"{deps_path}/rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz" ) dep3_dest_path = RequestBundleDir( f"{deps_path}/github/ReactiveX/rxjs/rxjs-6.5.5-external-gitcommit-" "78032157f5c1655436829017bbda787565b48c30.tgz") dep4_source_path = RequestBundleDir( f"{deps_path}/exsp-2.10.2-external-sha512-abcdefg.tar.gz") dep4_dest_path = RequestBundleDir( f"{deps_path}/external-exsp/exsp-2.10.2-external-sha512-abcdefg.tar.gz" ) mock_move.assert_has_calls([ mock.call(dep1_source_path, dep1_dest_path), mock.call(dep2_source_path, dep2_dest_path), mock.call(dep3_source_path, dep3_dest_path), mock.call(dep4_source_path, dep4_dest_path), ])