def get_project( self, runtime_environment_name: Optional[str] = None, *, missing_dir_ok: bool = False, ) -> Project: """Get the given overlay.""" path = self.get_overlays_directory( runtime_environment_name=runtime_environment_name, missing_dir_ok=missing_dir_ok, ) runtime_environment = RuntimeEnvironment.from_dict( self.get_runtime_environment(runtime_environment_name)) if self.requirements_format == "pipenv": pipfile_lock_path: Optional[str] = os.path.join( path, "Pipfile.lock") if pipfile_lock_path and not os.path.exists(pipfile_lock_path): pipfile_lock_path = None pipfile_path = os.path.join(path, "Pipfile") if not os.path.isfile(pipfile_path): if not os.path.isdir(path): _LOGGER.info("Creating directory structure in %r", path) os.makedirs(path, exist_ok=True) pipfile = Pipfile.from_dict({}) pipfile.to_file(path=pipfile_path) project = Project.from_files( pipfile_path=pipfile_path, pipfile_lock_path=pipfile_lock_path, runtime_environment=runtime_environment, without_pipfile_lock=pipfile_lock_path is None, ) else: requirements_in_file_path = os.path.join(path, "requirements.in") if not os.path.isfile(requirements_in_file_path): requirements_txt_file_path = os.path.join( path, "requirements.txt") if os.path.isfile(requirements_txt_file_path): _LOGGER.warning("Using %r for direct dependencies", requirements_in_file_path) project = Project.from_pip_compile_files( requirements_path=requirements_txt_file_path, requirements_lock_path=None, allow_without_lock=True, runtime_environment=runtime_environment, ) else: raise NotImplementedError( "No requirements.txt/requirements.in files found, it is recommended to " "use Pipenv files for managing dependencies") else: project = Project.from_pip_compile_files( requirements_path=requirements_in_file_path, requirements_lock_path=None, allow_without_lock=True, runtime_environment=runtime_environment, ) return project
def test_from_pip_compile_files_example_dir2(self) -> None: """Test loading project from pip-compile files.""" with cwd(os.path.join(self.data_dir, "requirements", "example_dir2")): assert Project.from_pip_compile_files(allow_without_lock=False) == Project.from_pip_compile_files( allow_without_lock=True ) project = Project.from_pip_compile_files(allow_without_lock=False) assert list(project.iter_dependencies()) == [ PackageVersion(name="flask", version="*", develop=False, index=Source(url="https://pypi.org/simple")) ] assert list(project.iter_dependencies_locked()) == [ PackageVersion( name="click", version="==7.0", develop=False, index=Source(url="https://pypi.org/simple"), hashes=["sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"], ), PackageVersion( name="flask", version="==1.1.1", develop=False, index=Source(url="https://pypi.org/simple"), hashes=["sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6"], ), PackageVersion( name="itsdangerous", version="==1.1.0", develop=False, index=Source(url="https://pypi.org/simple"), hashes=["sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"], ), PackageVersion( name="jinja2", version="==2.10.3", develop=False, index=Source(url="https://pypi.org/simple"), hashes=["sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f"], ), PackageVersion( name="markupsafe", version="==1.1.1", develop=False, index=Source(url="https://pypi.org/simple"), hashes=["sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"], ), PackageVersion( name="werkzeug", version="==0.16.0", develop=False, index=Source(url="https://pypi.org/simple"), hashes=["sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"], ), ]
def test_from_pip_compile_files_example_dir1(self) -> None: """Test loading only if requirements.txt is present.""" with cwd(os.path.join(self.data_dir, "requirements", "example_dir1")): with pytest.raises(FileLoadError): Project.from_pip_compile_files() project = Project.from_pip_compile_files(allow_without_lock=True) assert project.pipfile_lock is None assert list(project.iter_dependencies()) == [ PackageVersion( name="click", version="*", develop=False, hashes=[], index=Source("https://pypi.org/simple") ) ]
def _load_files(requirements_format: str) -> Tuple[str, Optional[str]]: """Load Pipfile/Pipfile.lock or requirements.in/txt from the current directory.""" if requirements_format == "pipenv": _LOGGER.info("Using Pipenv files located in %r directory", os.getcwd()) pipfile_lock_exists = os.path.exists("Pipfile.lock") if pipfile_lock_exists: _LOGGER.info( "Submitting Pipfile.lock as a base for user's stack scoring - see %s", jl("user_stack"), ) project = Project.from_files( without_pipfile_lock=not os.path.exists("Pipfile.lock")) if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] != project.pipfile.hash()["sha256"]): _LOGGER.error( "Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile " "adjusted? This error is not critical.", project.pipfile_lock.meta.hash["sha256"][:6], project.pipfile.hash()["sha256"][:6], ) elif requirements_format in ("pip", "pip-tools", "pip-compile"): _LOGGER.info("Using requirements.txt file located in %r directory", os.getcwd()) project = Project.from_pip_compile_files(allow_without_lock=True) else: raise ValueError( f"Unknown configuration option for requirements format: {requirements_format!r}" ) return ( project.pipfile.to_string(), project.pipfile_lock.to_string() if project.pipfile_lock else None, )
def test_from_pip_compile_files_example_dir3(self) -> None: """Test loading only if only requirements.in is present.""" with cwd(os.path.join(self.data_dir, "requirements", "example_dir3")): project = Project.from_pip_compile_files(allow_without_lock=True) assert project.pipfile_lock is None assert list(project.iter_dependencies()) == [ PackageVersion(name="flask", version="*", develop=False, hashes=[], index=None) ]
def test_no_include_no_pipenv_files( self, builder_context: PipelineBuilderContext) -> None: """Test if Pipenv files are not supplied - no hash is available to check against.""" requirements_in = str(self.data_dir / "projects" / "requirements.in") requirements_txt = str(self.data_dir / "projects" / "requirements.txt") project = Project.from_pip_compile_files( requirements_path=requirements_in, requirements_lock_path=requirements_txt, ) builder_context.project = project assert list(self.UNIT_TESTED.should_include(builder_context)) == []
def qeb_hwt_thamos_advise() -> None: """Qeb-Hwt Thamos Advise Task.""" if not Configuration._REPO_PATH: raise Exception("No path has been provided to REPO_PATH env variable.") if not Path(Configuration._REPO_PATH).exists(): raise FileNotFoundError( f"Cannot find the file on this path: {Configuration._REPO_PATH}") output_messages: list = [] OpenShift.verify_github_app_inputs( github_event_type=Configuration._GITHUB_EVENT_TYPE, github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID, github_installation_id=Configuration._GITHUB_INSTALLATION_ID, github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL, origin=Configuration._ORIGIN, ) os.chdir(Configuration._REPO_PATH) thoth_yaml_config = _Configuration() if not thoth_yaml_config.config_file_exists(): exception_message = _create_message_config_file_error(no_file=True) store_messages(output_messages) trigger_finished_webhook(exception_message=exception_message, has_error=True, error_type="MissingThothYamlFile") return try: # Consider first runtime environment runtime_environment = thoth_yaml_config.content.get( "runtime_environments")[0] # Fetch recommendation type recommendation_type = runtime_environment.get( "recommendation_type") if thoth_yaml_config else "latest" requirements_format = thoth_yaml_config.requirements_format if requirements_format == "pipenv": project = Project.from_files( without_pipfile_lock=not os.path.exists("Pipfile.lock")) elif requirements_format in ("pip", "pip-tools", "pip-compile"): project = Project.from_pip_compile_files(allow_without_lock=True) else: raise ValueError( f"Unknown configuration option for requirements format: {requirements_format!r}" ) pipfile = project.pipfile.to_string() pipfile_lock_str = project.pipfile_lock.to_string( ) if project.pipfile_lock else "" application_stack = PythonStack( requirements=pipfile, requirements_lock=pipfile_lock_str, requirements_format=requirements_format).to_dict() except Exception as exception: _LOGGER.debug(json.loads(exception.body)["error"]) # type: ignore exception_message = json.loads(exception.body)["error"] # type: ignore store_messages(output_messages) trigger_finished_webhook(exception_message=exception_message, has_error=True) return # The input for AdviserTriggerMessage if no exceptions were found message_input = { "component_name": { "type": "str", "value": __COMPONENT_NAME__ }, "service_version": { "type": "str", "value": __service_version__ }, "application_stack": { "type": "Dict", "value": application_stack }, "runtime_environment": { "type": "Dict", "value": runtime_environment }, "recommendation_type": { "type": "str", "value": recommendation_type }, "github_event_type": { "type": "str", "value": Configuration._GITHUB_EVENT_TYPE }, "github_check_run_id": { "type": "int", "value": int(Configuration._GITHUB_CHECK_RUN_ID) }, "github_installation_id": { "type": "int", "value": int(Configuration._GITHUB_INSTALLATION_ID) }, "github_base_repo_url": { "type": "str", "value": Configuration._GITHUB_BASE_REPO_URL }, "origin": { "type": "str", "value": Configuration._ORIGIN }, "source_type": { "type": "str", "value": ThothAdviserIntegrationEnum.GITHUB_APP.name }, } # We store the message to put in the output file here. output_messages = [{ "topic_name": "thoth.adviser-trigger", "message_contents": message_input }] store_messages(output_messages)
def update_keb_installation(): """Load files and pass them to storages update function.""" if _SLUG is None: _LOGGER.info("No slug present, continuing to next step in task.") return service = GithubService( github_app_id=os.getenv("GITHUB_APP_ID"), github_app_private_key_path=os.getenv("GITHUB_PRIVATE_KEY_PATH"), ) # TODO: extend to use other services project = service.get_project(namespace=_SLUG.split("/")[0], repo=_SLUG.split("/")[1]) raw_thoth_config = project.get_file_content(".thoth.yaml") with TemporaryDirectory() as repo_path, cwd(repo_path): thoth_config.load_config_from_string(raw_thoth_config) requirements_format = thoth_config.content["requirements_format"] overlays_dir = thoth_config.content.get("overlays_dir") to_update: List[RuntimeEnvironment] if overlays_dir is not None: to_update = [RuntimeEnvironment.from_dict(r) for r in thoth_config.list_runtime_environments()] else: to_update = [RuntimeEnvironment.from_dict(thoth_config.get_runtime_environment())] for runtime_environment in to_update: if overlays_dir: prefix = f"{overlays_dir}/{runtime_environment.name}/" else: prefix = "" if requirements_format == "pipenv": pipfile_r = project.get_file_content(f"{prefix}Pipfile") with open("Pipfile", "wb") as f: f.write(pipfile_r) try: piplock_r = project.get_file_content(f"{prefix}Pipfile.lock") with open("Pipfile.lock", "wb") as f: f.write(piplock_r) project = Project.from_files(pipfile_path="Pipfile", pipfile_lock_path="Pipfile.lock") except Exception: _LOGGER.debug("No Pipfile.lock found") project = Project.from_files( pipfile_path="Pipfile", without_pipfile_lock=True, runtime_environment=runtime_environment, ) elif requirements_format in ["pip", "pip-tools", "pip-compile"]: try: requirements_r = project.get_file_content(f"{prefix}requirements.txt") with open("requirements.txt", "wb") as f: f.write(requirements_r) project = Project.from_pip_compile_files( requirements_path="requirements.txt", allow_without_lock=True, runtime_environment=runtime_environment, ) except Exception: _LOGGER.debug("No requirements.txt found, trying to download requirements.in") requirements_r = project.get_file_content(f"{prefix}requirements.in") with open("requirements.in", "wb") as f: f.write(requirements_r.content) project = Project.from_pip_compile_files( requirements_path="requirements.in", allow_without_lock=True, runtime_environment=runtime_environment, ) project = Project.from_pip_compile_files(allow_without_lock=True) else: raise NotImplementedError(f"{requirements_format} requirements format not supported.") db.update_kebechet_installation_using_files( slug=_SLUG, runtime_environment_name=runtime_environment.name, installation_id=str(project.github_repo.id), requirements=project.pipfile.to_dict(), requirements_lock=project.pipfile_lock.to_dict(), thoth_config=thoth_config, ) present_installations = db.get_kebechet_github_app_installations_all(slug=_SLUG) cur_env_names = {r.name for r in to_update} all_env_names = {installation["runtime_environment_name"] for installation in present_installations} to_delete = all_env_names - cur_env_names for name in to_delete: db.delete_kebechet_github_app_installations(slug=_SLUG, runtime_environment=name)
def advise_here( recommendation_type: typing.Optional[str] = None, *, runtime_environment: dict = None, runtime_environment_name: typing.Optional[str] = None, dev: bool = False, no_static_analysis: bool = False, no_user_stack: bool = False, nowait: bool = False, force: bool = False, limit: typing.Optional[int] = None, count: int = 1, debug: bool = False, origin: typing.Optional[str] = None, github_event_type: typing.Optional[str] = None, github_check_run_id: typing.Optional[int] = None, github_installation_id: typing.Optional[int] = None, github_base_repo_url: typing.Optional[str] = None, source_type: typing.Optional[ThothAdviserIntegrationEnum] = None, ) -> typing.Optional[tuple]: """Run advise in current directory, requires no arguments.""" requirements_format = thoth_config.requirements_format if requirements_format == "pipenv": _LOGGER.info( "Using Pipenv files located in the project root directory") pipfile_lock_exists = os.path.exists("Pipfile.lock") if pipfile_lock_exists: _LOGGER.info( "Submitting Pipfile.lock as a base for user's stack scoring - see %s", jl("user_stack"), ) project = Project.from_files( without_pipfile_lock=not os.path.exists("Pipfile.lock")) if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] != project.pipfile.hash()["sha256"]): _LOGGER.error( "Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile " "adjusted? This error is not critical.", project.pipfile_lock.meta.hash["sha256"][:6], project.pipfile.hash()["sha256"][:6], ) elif requirements_format in ("pip", "pip-tools", "pip-compile"): _LOGGER.info( "Using requirements.txt file located in the project root directory" ) project = Project.from_pip_compile_files(allow_without_lock=True) else: raise ValueError( f"Unknown configuration option for requirements format: {requirements_format!r}" ) pipfile = project.pipfile.to_string() pipfile_lock_str = project.pipfile_lock.to_string( ) if project.pipfile_lock else "" return advise( pipfile=pipfile, pipfile_lock=pipfile_lock_str, recommendation_type=recommendation_type, runtime_environment=runtime_environment, runtime_environment_name=runtime_environment_name, dev=dev, no_static_analysis=no_static_analysis, no_user_stack=no_user_stack, nowait=nowait, force=force, limit=limit, count=count, debug=debug, origin=origin, source_type=source_type, github_event_type=github_event_type, github_check_run_id=github_check_run_id, github_installation_id=github_installation_id, github_base_repo_url=github_base_repo_url, )