def test_update_request_with_package(mock_requests, package_subpath, include_subpath): mock_requests.patch.return_value.ok = True package = { "name": "helloworld", "type": "gomod", "version": "v0.0.0-20200324130456-8aedc0ec8bb5", } env_vars = { "GOCACHE": { "value": "deps/gomod", "kind": "path" }, "GOPATH": { "value": "deps/gomod", "kind": "path" }, } expected_json = { "environment_variables": env_vars, "package": package, } if include_subpath: expected_json["package_subpath"] = package_subpath update_request_with_package(1, package, env_vars, package_subpath=package_subpath) mock_requests.patch.assert_called_once_with( "http://cachito.domain.local/api/v1/requests/1", json=expected_json, timeout=60)
def test_update_request_with_package_failed(mock_requests): mock_requests.patch.return_value.ok = False package = { "name": "helloworld", "type": "gomod", "version": "v0.0.0-20200324130456-8aedc0ec8bb5", } with pytest.raises(CachitoError, match="Setting a package on request 1 failed"): update_request_with_package(1, package)
def test_update_request_with_package_failed_connection(mock_requests): mock_requests.patch.side_effect = requests.ConnectTimeout() package = { "name": "helloworld", "type": "gomod", "version": "v0.0.0-20200324130456-8aedc0ec8bb5", } expected_msg = "The connection failed when adding a package to the request 1" with pytest.raises(CachitoError, match=expected_msg): update_request_with_package(1, package)
def add_git_submodules_as_package(request_id): """ Add git submodules as package to the Cachtio request. :param int request_id: the Cachito request ID this is for :raises CachitoError: if adding submodules as a package fail. """ bundle_dir = RequestBundleDir(request_id) repo = git.Repo(str(bundle_dir.source_root_dir)) for sm in repo.submodules: # Save package to db package = { "type": "git-submodule", "name": sm.name, "version": f"{sm.url}#{sm.hexsha}", } log.debug("Adding submodule '%s' as a package for Cachito request", sm.name) update_request_with_package(request_id, package, package_subpath=sm.path)
def fetch_yarn_source(request_id: int, package_configs: List[dict] = None): """ Resolve and fetch yarn dependencies for a given request. This function uses the Python ``os.path`` library to manipulate paths, so the path to the configuration files may differ in format based on the system the Cachito worker is deployed on (i.e. Linux vs Windows). :param int request_id: the Cachito request ID this is for :param list package_configs: the list of optional package configurations submitted by the user :raise CachitoError: if the task fails """ if package_configs is None: package_configs = [] validate_yarn_config() bundle_dir = RequestBundleDir(request_id) subpaths = [ os.path.normpath(c["path"]) for c in package_configs if c.get("path") ] if not subpaths: # Default to the root of the application source subpaths = [os.curdir] _verify_yarn_files(bundle_dir, subpaths) log.info("Configuring Nexus for yarn for the request %d", request_id) set_request_state(request_id, "in_progress", "Configuring Nexus for yarn") repo_name = get_yarn_proxy_repo_name(request_id) prepare_nexus_for_js_request(repo_name) yarn_config_files = [] downloaded_deps = set() for i, subpath in enumerate(subpaths): log.info("Fetching the yarn dependencies for request %d in subpath %s", request_id, subpath) set_request_state( request_id, "in_progress", f'Fetching the yarn dependencies at the "{subpath}" directory', ) request = get_request(request_id) package_source_path = str(bundle_dir.app_subpath(subpath).source_dir) try: package_and_deps_info = resolve_yarn(package_source_path, request, skip_deps=downloaded_deps) except CachitoError: log.exception("Failed to fetch yarn dependencies for request %d", request_id) raise downloaded_deps = downloaded_deps | package_and_deps_info[ "downloaded_deps"] log.info( "Generating the yarn configuration files for request %d in subpath %s", request_id, subpath, ) remote_package_source_path = os.path.normpath( os.path.join("app", subpath)) if package_and_deps_info["package.json"]: package_json_str = json.dumps( package_and_deps_info["package.json"], indent=2) package_json_path = os.path.join(remote_package_source_path, "package.json") yarn_config_files.append( make_base64_config_file(package_json_str, package_json_path)) if package_and_deps_info["lock_file"]: yarn_lock_str = _yarn_lock_to_str( package_and_deps_info["lock_file"]) yarn_lock_path = os.path.join(remote_package_source_path, "yarn.lock") yarn_config_files.append( make_base64_config_file(yarn_lock_str, yarn_lock_path)) if i == 0: default_env = get_worker_config( ).cachito_default_environment_variables env_vars = { **default_env.get("npm", {}), **default_env.get("yarn", {}) } else: env_vars = None package = package_and_deps_info["package"] update_request_with_package(request_id, package, env_vars, package_subpath=subpath) update_request_with_deps(request_id, package, package_and_deps_info["deps"]) log.info("Finalizing the Nexus configuration for yarn for the request %d", request_id) set_request_state(request_id, "in_progress", "Finalizing the Nexus configuration for yarn") username = get_yarn_proxy_repo_username(request_id) password = finalize_nexus_for_js_request(username, repo_name) log.info("Generating the .npmrc file(s)") proxy_repo_url = get_yarn_proxy_repo_url(request_id) yarn_config_files.extend( generate_npmrc_config_files(proxy_repo_url, username, password, subpaths)) log.info("Adding empty .yarnrc file(s)") for subpath in subpaths: yarnrc_path = os.path.normpath(os.path.join("app", subpath, ".yarnrc")) yarn_config_files.append(make_base64_config_file("", yarnrc_path)) update_request_with_config_files(request_id, yarn_config_files)
def fetch_gomod_source(request_id, dep_replacements=None, package_configs=None): """ Resolve and fetch gomod dependencies for a given request. :param int request_id: the Cachito request ID this is for :param list dep_replacements: dependency replacements with the keys "name" and "version"; only supported with a single path :param list package_configs: the list of optional package configurations submitted by the user :raises CachitoError: if the dependencies could not be retrieved """ config = get_worker_config() if package_configs is None: package_configs = [] bundle_dir = RequestBundleDir(request_id) subpaths = [ os.path.normpath(c["path"]) for c in package_configs if c.get("path") ] if not subpaths: # Default to the root of the application source subpaths = [os.curdir] invalid_gomod_files = _find_missing_gomod_files(bundle_dir, subpaths) if invalid_gomod_files: invalid_files_print = "; ".join(invalid_gomod_files) file_suffix = "s" if len(invalid_gomod_files) > 1 else "" # missing gomod files is supported if there is only one path referenced if config.cachito_gomod_ignore_missing_gomod_file and len( subpaths) == 1: log.warning("go.mod file missing for request at %s", invalid_files_print) return raise CachitoError( "The {} file{} must be present for the gomod package manager". format(invalid_files_print.strip(), file_suffix)) if len(subpaths) > 1 and dep_replacements: raise CachitoError( "Dependency replacements are only supported for a single go module path." ) for i, subpath in enumerate(subpaths): log.info( "Fetching the gomod dependencies for request %d in subpath %s", request_id, subpath) request = set_request_state( request_id, "in_progress", f'Fetching the gomod dependencies at the "{subpath}" directory', ) gomod_source_path = str(bundle_dir.app_subpath(subpath).source_dir) try: gomod = resolve_gomod(gomod_source_path, request, dep_replacements, bundle_dir.source_dir) except CachitoError: log.exception("Failed to fetch gomod dependencies for request %d", request_id) raise if i == 0: env_vars = { "GOCACHE": { "value": "deps/gomod", "kind": "path" }, "GOPATH": { "value": "deps/gomod", "kind": "path" }, } env_vars.update( config.cachito_default_environment_variables.get("gomod", {})) else: env_vars = None update_request_with_package(request_id, gomod["module"], env_vars) update_request_with_deps(request_id, gomod["module"], gomod["module_deps"]) # add package deps for package in gomod["packages"]: if package.get("pkg_deps"): # This also adds the package to the request update_request_with_deps(request_id, package["pkg"], package["pkg_deps"]) else: update_request_with_package(request_id, package["pkg"])
def fetch_pip_source(request_id, package_configs=None): """ Resolve and fetch pip dependencies for a given request. :param int request_id: the Cachito request ID this is for :param list package_configs: the list of optional package configurations submitted by the user """ validate_pip_config() bundle_dir = RequestBundleDir(request_id) log.info("Configuring Nexus for pip for the request %d", request_id) set_request_state(request_id, "in_progress", "Configuring Nexus for pip") pip_repo_name = get_pypi_hosted_repo_name(request_id) raw_repo_name = get_raw_hosted_repo_name(request_id) prepare_nexus_for_pip_request(pip_repo_name, raw_repo_name) log.info("Fetching dependencies for request %d", request_id) package_configs = package_configs or [{}] packages_data = [] requirement_file_paths = [] for pkg_cfg in package_configs: pkg_path = pkg_cfg.get("path", ".") source_dir = bundle_dir.app_subpath(pkg_path).source_dir set_request_state( request_id, "in_progress", f"Fetching dependencies at the {pkg_path!r} directory", ) request = get_request(request_id) pkg_and_deps_info = resolve_pip( source_dir, request, requirement_files=pkg_cfg.get("requirements_files"), build_requirement_files=pkg_cfg.get("requirements_build_files"), ) # defer custom requirement files creation to use the Nexus password in the URLs for requirement_file_path in pkg_and_deps_info.pop("requirements"): requirement_file_paths.append(requirement_file_path) # defer DB operations to use the Nexus password in the env vars packages_data.append(pkg_and_deps_info) log.info("Finalizing the Nexus configuration for pip for the request %d", request_id) set_request_state(request_id, "in_progress", "Finalizing the Nexus configuration for pip") username = get_hosted_repositories_username(request_id) password = finalize_nexus_for_pip_request(pip_repo_name, raw_repo_name, username) # Set environment variables and config files pip_config_files = [] for requirement_file_path in requirement_file_paths: custom_requirement_file = _get_custom_requirement_config_file( requirement_file_path, bundle_dir.source_root_dir, raw_repo_name, username, password) if custom_requirement_file: pip_config_files.append(custom_requirement_file) raw_url = get_pypi_hosted_repo_url(request_id) pip_index_url = get_index_url(raw_url, username, password) env_vars = {"PIP_INDEX_URL": {"value": pip_index_url, "kind": "literal"}} ca_cert = nexus.get_ca_cert() if ca_cert: ca_cert_path = os.path.join("app", "package-index-ca.pem") env_vars["PIP_CERT"] = {"value": ca_cert_path, "kind": "path"} pip_config_files.append(make_base64_config_file(ca_cert, ca_cert_path)) worker_config = get_worker_config() env_vars.update( worker_config.cachito_default_environment_variables.get("pip", {})) # Finally, perform DB operations for pkg_cfg, pkg_data in zip(package_configs, packages_data): pkg_subpath = os.path.normpath(pkg_cfg.get("path", ".")) update_request_with_package(request_id, pkg_data["package"], env_vars, package_subpath=pkg_subpath) update_request_with_deps(request_id, pkg_data["package"], pkg_data["dependencies"]) if pip_config_files: update_request_with_config_files(request_id, pip_config_files)
def fetch_npm_source(request_id, package_configs=None): """ Resolve and fetch npm dependencies for a given request. This function uses the Python ``os.path`` library to manipulate paths, so the path to the configuration files may differ in format based on the system the Cachito worker is deployed on (i.e. Linux vs Windows). :param int request_id: the Cachito request ID this is for :param list package_configs: the list of optional package configurations submitted by the user :raise CachitoError: if the task fails """ if package_configs is None: package_configs = [] validate_npm_config() bundle_dir = RequestBundleDir(request_id) log.debug("Checking if the application source uses npm") subpaths = [ os.path.normpath(c["path"]) for c in package_configs if c.get("path") ] if not subpaths: # Default to the root of the application source subpaths = [os.curdir] _verify_npm_files(bundle_dir, subpaths) log.info("Configuring Nexus for npm for the request %d", request_id) set_request_state(request_id, "in_progress", "Configuring Nexus for npm") repo_name = get_npm_proxy_repo_name(request_id) prepare_nexus_for_js_request(repo_name) npm_config_files = [] downloaded_deps = set() for i, subpath in enumerate(subpaths): log.info("Fetching the npm dependencies for request %d in subpath %s", request_id, subpath) request = set_request_state( request_id, "in_progress", f'Fetching the npm dependencies at the "{subpath}" directory"', ) package_source_path = str(bundle_dir.app_subpath(subpath).source_dir) try: package_and_deps_info = resolve_npm(package_source_path, request, skip_deps=downloaded_deps) except CachitoError: log.exception("Failed to fetch npm dependencies for request %d", request_id) raise downloaded_deps = downloaded_deps | package_and_deps_info[ "downloaded_deps"] log.info( "Generating the npm configuration files for request %d in subpath %s", request_id, subpath, ) remote_package_source_path = os.path.normpath( os.path.join("app", subpath)) if package_and_deps_info["package.json"]: package_json_str = json.dumps( package_and_deps_info["package.json"], indent=2) npm_config_files.append({ "content": base64.b64encode( package_json_str.encode("utf-8")).decode("utf-8"), "path": os.path.join(remote_package_source_path, "package.json"), "type": "base64", }) if package_and_deps_info["lock_file"]: package_lock_str = json.dumps(package_and_deps_info["lock_file"], indent=2) lock_file_name = package_and_deps_info["lock_file_name"] npm_config_files.append({ "content": base64.b64encode( package_lock_str.encode("utf-8")).decode("utf-8"), "path": os.path.join(remote_package_source_path, lock_file_name), "type": "base64", }) if i == 0: env_vars = get_worker_config( ).cachito_default_environment_variables.get("npm", {}) else: env_vars = None package = package_and_deps_info["package"] update_request_with_package(request_id, package, env_vars, package_subpath=subpath) update_request_with_deps(request_id, package, package_and_deps_info["deps"]) log.info("Finalizing the Nexus configuration for npm for the request %d", request_id) set_request_state(request_id, "in_progress", "Finalizing the Nexus configuration for npm") username = get_npm_proxy_username(request_id) password = finalize_nexus_for_js_request(username, repo_name) log.info("Generating the .npmrc file(s)") ca_cert = nexus.get_ca_cert() if ca_cert: # The custom CA will be called registry-ca.pem in the "app" directory npm_config_files.append({ "content": base64.b64encode(ca_cert.encode("utf-8")).decode("utf-8"), "path": os.path.join("app", "registry-ca.pem"), "type": "base64", }) for subpath in subpaths: proxy_repo_url = get_npm_proxy_repo_url(request_id) if ca_cert: # Determine the relative path to the registry-ca.pem file custom_ca_path = os.path.relpath("registry-ca.pem", start=subpath) else: custom_ca_path = None npm_rc = generate_npmrc_content(proxy_repo_url, username, password, custom_ca_path=custom_ca_path) npm_config_files.append({ "content": base64.b64encode(npm_rc.encode("utf-8")).decode("utf-8"), "path": os.path.normpath(os.path.join("app", subpath, ".npmrc")), "type": "base64", }) update_request_with_config_files(request_id, npm_config_files)