def test_download_dependencies_skip_deps( mock_gwc, mock_move, mock_run_cmd, mock_gawnf, mock_exists, mock_td, tmpdir, ): bundles_dir = tmpdir.mkdir("bundles") mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir) mock_td.return_value.__enter__.return_value = str( tmpdir.mkdir("cachito-agfdsk")) mock_exists.return_value = False mock_run_cmd.return_value = textwrap.dedent("""\ angular-devkit-architect-0.803.26.tgz rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz """) deps = [ { "bundled": False, "dev": True, "name": "@angular-devkit/architect", "version": "0.803.26", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "@angular/animations", "version": "8.2.14", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "rxjs", "version": "github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30", "version_in_nexus": "6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", }, ] proxy_repo_url = npm.get_npm_proxy_repo_url(1) general_js.download_dependencies(1, deps, proxy_repo_url, {"@angular/[email protected]"}) mock_run_cmd.assert_called_once() # This ensures that the skipped dependency is not downloaded expected_npm_pack = [ "npm", "pack", "@angular-devkit/[email protected]", "rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", ] assert mock_run_cmd.call_args[0][0] == expected_npm_pack
def test_download_dependencies( mock_gwc, mock_move, mock_run_cmd, mock_gawnf, mock_exists, mock_td, nexus_ca_cert_exists, pkg_manager, tmpdir, ): bundles_dir = tmpdir.mkdir("bundles") mock_gwc.return_value.cachito_bundles_dir = str(bundles_dir) mock_gwc.return_value.cachito_nexus_ca_cert = "/etc/cachito/nexus_ca.pem" mock_td_path = tmpdir.mkdir("cachito-agfdsk") mock_td.return_value.__enter__.return_value = str(mock_td_path) mock_exists.return_value = nexus_ca_cert_exists mock_run_cmd.return_value = textwrap.dedent("""\ angular-devkit-architect-0.803.26.tgz angular-animations-8.2.14.tgz rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz exsp-2.10.2-external-sha512-abcdefg.tar.gz """) deps = [ { "bundled": False, "dev": True, "name": "@angular-devkit/architect", "version": "0.803.26", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "@angular/animations", "version": "8.2.14", "version_in_nexus": None, }, { "bundled": True, "dev": True, "name": "object-assign", "version": "4.1.1", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "rxjs", "version": "github:ReactiveX/rxjs#78032157f5c1655436829017bbda787565b48c30", "version_in_nexus": "6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", }, { "bundled": False, "dev": False, "name": "jsplumb", "version": "file:../jsplumb-2.10.2.tgz", "version_in_nexus": None, }, { "bundled": False, "dev": False, "name": "exsp", "version": "https://github.com/exsp/exsp/archive/2.10.2.tar.gz", "version_in_nexus": "2.10.2-external-sha512-abcdefg", }, ] request_id = 1 request_bundle_dir = bundles_dir.mkdir("temp").mkdir(str(request_id)) deps_path = os.path.join(request_bundle_dir, f"deps/{pkg_manager}") proxy_repo_url = npm.get_npm_proxy_repo_url(request_id) general_js.download_dependencies(request_id, deps, proxy_repo_url, pkg_manager=pkg_manager) mock_npm_rc_path = str(mock_td_path.join(".npmrc")) if nexus_ca_cert_exists: mock_gawnf.assert_called_once_with( mock_npm_rc_path, "http://*****:*****@angular-devkit/[email protected]", "@angular/[email protected]", "rxjs@6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30", "[email protected]", ] assert mock_run_cmd.call_args[0][0] == expected_npm_pack run_cmd_env_vars = mock_run_cmd.call_args[0][1]["env"] assert run_cmd_env_vars["NPM_CONFIG_CACHE"] == str( mock_td_path.join("cache")) assert run_cmd_env_vars["NPM_CONFIG_USERCONFIG"] == mock_npm_rc_path assert mock_run_cmd.call_args[0][1]["cwd"] == f"{deps_path}" dep1_source_path = RequestBundleDir( f"{deps_path}/angular-devkit-architect-0.803.26.tgz") dep1_dest_path = RequestBundleDir( f"{deps_path}/@angular-devkit/architect/angular-devkit-architect-0.803.26.tgz" ) dep2_source_path = RequestBundleDir( f"{deps_path}/angular-animations-8.2.14.tgz") dep2_dest_path = RequestBundleDir( f"{deps_path}/@angular/animations/angular-animations-8.2.14.tgz") dep3_source_path = RequestBundleDir( f"{deps_path}/rxjs-6.5.5-external-gitcommit-78032157f5c1655436829017bbda787565b48c30.tgz" ) dep3_dest_path = RequestBundleDir( f"{deps_path}/github/ReactiveX/rxjs/rxjs-6.5.5-external-gitcommit-" "78032157f5c1655436829017bbda787565b48c30.tgz") dep4_source_path = RequestBundleDir( f"{deps_path}/exsp-2.10.2-external-sha512-abcdefg.tar.gz") dep4_dest_path = RequestBundleDir( f"{deps_path}/external-exsp/exsp-2.10.2-external-sha512-abcdefg.tar.gz" ) mock_move.assert_has_calls([ mock.call(dep1_source_path, dep1_dest_path), mock.call(dep2_source_path, dep2_dest_path), mock.call(dep3_source_path, dep3_dest_path), mock.call(dep4_source_path, dep4_dest_path), ])
def test_get_npm_proxy_repo_url(): assert npm.get_npm_proxy_repo_url(3).endswith("/repository/cachito-npm-3/")
def fetch_npm_source(request_id, package_configs=None): """ Resolve and fetch npm dependencies for a given request. This function uses the Python ``os.path`` library to manipulate paths, so the path to the configuration files may differ in format based on the system the Cachito worker is deployed on (i.e. Linux vs Windows). :param int request_id: the Cachito request ID this is for :param list package_configs: the list of optional package configurations submitted by the user :raise CachitoError: if the task fails """ if package_configs is None: package_configs = [] validate_npm_config() bundle_dir = RequestBundleDir(request_id) log.debug("Checking if the application source uses npm") subpaths = [ os.path.normpath(c["path"]) for c in package_configs if c.get("path") ] if not subpaths: # Default to the root of the application source subpaths = [os.curdir] _verify_npm_files(bundle_dir, subpaths) log.info("Configuring Nexus for npm for the request %d", request_id) set_request_state(request_id, "in_progress", "Configuring Nexus for npm") repo_name = get_npm_proxy_repo_name(request_id) prepare_nexus_for_js_request(repo_name) npm_config_files = [] downloaded_deps = set() for i, subpath in enumerate(subpaths): log.info("Fetching the npm dependencies for request %d in subpath %s", request_id, subpath) set_request_state( request_id, "in_progress", f'Fetching the npm dependencies at the "{subpath}" directory"', ) request = get_request(request_id) package_source_path = str(bundle_dir.app_subpath(subpath).source_dir) try: package_and_deps_info = resolve_npm(package_source_path, request, skip_deps=downloaded_deps) except CachitoError: log.exception("Failed to fetch npm dependencies for request %d", request_id) raise downloaded_deps = downloaded_deps | package_and_deps_info[ "downloaded_deps"] log.info( "Generating the npm configuration files for request %d in subpath %s", request_id, subpath, ) remote_package_source_path = os.path.normpath( os.path.join("app", subpath)) if package_and_deps_info["package.json"]: package_json_str = json.dumps( package_and_deps_info["package.json"], indent=2) package_json_path = os.path.join(remote_package_source_path, "package.json") npm_config_files.append( make_base64_config_file(package_json_str, package_json_path)) if package_and_deps_info["lock_file"]: package_lock_str = json.dumps(package_and_deps_info["lock_file"], indent=2) lock_file_name = package_and_deps_info["lock_file_name"] lock_file_path = os.path.join(remote_package_source_path, lock_file_name) npm_config_files.append( make_base64_config_file(package_lock_str, lock_file_path)) if i == 0: env_vars = get_worker_config( ).cachito_default_environment_variables.get("npm", {}) else: env_vars = None package = package_and_deps_info["package"] update_request_with_package(request_id, package, env_vars, package_subpath=subpath) update_request_with_deps(request_id, package, package_and_deps_info["deps"]) log.info("Finalizing the Nexus configuration for npm for the request %d", request_id) set_request_state(request_id, "in_progress", "Finalizing the Nexus configuration for npm") username = get_npm_proxy_username(request_id) password = finalize_nexus_for_js_request(username, repo_name) log.info("Generating the .npmrc file(s)") proxy_repo_url = get_npm_proxy_repo_url(request_id) npm_config_files.extend( generate_npmrc_config_files(proxy_repo_url, username, password, subpaths)) update_request_with_config_files(request_id, npm_config_files)