def get_project(
self,
runtime_environment_name: Optional[str] = None,
*,
missing_dir_ok: bool = False,
) -> Project:
"""Get the given overlay."""
path = self.get_overlays_directory(
runtime_environment_name=runtime_environment_name,
missing_dir_ok=missing_dir_ok,
)
runtime_environment = RuntimeEnvironment.from_dict(
self.get_runtime_environment(runtime_environment_name))
if self.requirements_format == "pipenv":
pipfile_lock_path: Optional[str] = os.path.join(
path, "Pipfile.lock")
if pipfile_lock_path and not os.path.exists(pipfile_lock_path):
pipfile_lock_path = None
pipfile_path = os.path.join(path, "Pipfile")
if not os.path.isfile(pipfile_path):
if not os.path.isdir(path):
_LOGGER.info("Creating directory structure in %r", path)
os.makedirs(path, exist_ok=True)
pipfile = Pipfile.from_dict({})
pipfile.to_file(path=pipfile_path)
project = Project.from_files(
pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path,
runtime_environment=runtime_environment,
without_pipfile_lock=pipfile_lock_path is None,
)
else:
requirements_in_file_path = os.path.join(path, "requirements.in")
if not os.path.isfile(requirements_in_file_path):
requirements_txt_file_path = os.path.join(
path, "requirements.txt")
if os.path.isfile(requirements_txt_file_path):
_LOGGER.warning("Using %r for direct dependencies",
requirements_in_file_path)
project = Project.from_pip_compile_files(
requirements_path=requirements_txt_file_path,
requirements_lock_path=None,
allow_without_lock=True,
runtime_environment=runtime_environment,
)
else:
raise NotImplementedError(
"No requirements.txt/requirements.in files found, it is recommended to "
"use Pipenv files for managing dependencies")
else:
project = Project.from_pip_compile_files(
requirements_path=requirements_in_file_path,
requirements_lock_path=None,
allow_without_lock=True,
runtime_environment=runtime_environment,
)
return project
def test_from_pip_compile_files_example_dir2(self) -> None:
"""Test loading project from pip-compile files."""
with cwd(os.path.join(self.data_dir, "requirements", "example_dir2")):
assert Project.from_pip_compile_files(allow_without_lock=False) == Project.from_pip_compile_files(
allow_without_lock=True
)
project = Project.from_pip_compile_files(allow_without_lock=False)
assert list(project.iter_dependencies()) == [
PackageVersion(name="flask", version="*", develop=False, index=Source(url="https://pypi.org/simple"))
]
assert list(project.iter_dependencies_locked()) == [
PackageVersion(
name="click",
version="==7.0",
develop=False,
index=Source(url="https://pypi.org/simple"),
hashes=["sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"],
),
PackageVersion(
name="flask",
version="==1.1.1",
develop=False,
index=Source(url="https://pypi.org/simple"),
hashes=["sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6"],
),
PackageVersion(
name="itsdangerous",
version="==1.1.0",
develop=False,
index=Source(url="https://pypi.org/simple"),
hashes=["sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"],
),
PackageVersion(
name="jinja2",
version="==2.10.3",
develop=False,
index=Source(url="https://pypi.org/simple"),
hashes=["sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f"],
),
PackageVersion(
name="markupsafe",
version="==1.1.1",
develop=False,
index=Source(url="https://pypi.org/simple"),
hashes=["sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"],
),
PackageVersion(
name="werkzeug",
version="==0.16.0",
develop=False,
index=Source(url="https://pypi.org/simple"),
hashes=["sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"],
),
]
def test_from_pip_compile_files_example_dir1(self) -> None:
"""Test loading only if requirements.txt is present."""
with cwd(os.path.join(self.data_dir, "requirements", "example_dir1")):
with pytest.raises(FileLoadError):
Project.from_pip_compile_files()
project = Project.from_pip_compile_files(allow_without_lock=True)
assert project.pipfile_lock is None
assert list(project.iter_dependencies()) == [
PackageVersion(
name="click", version="*", develop=False, hashes=[], index=Source("https://pypi.org/simple")
)
]
def _load_files(requirements_format: str) -> Tuple[str, Optional[str]]:
"""Load Pipfile/Pipfile.lock or requirements.in/txt from the current directory."""
if requirements_format == "pipenv":
_LOGGER.info("Using Pipenv files located in %r directory", os.getcwd())
pipfile_lock_exists = os.path.exists("Pipfile.lock")
if pipfile_lock_exists:
_LOGGER.info(
"Submitting Pipfile.lock as a base for user's stack scoring - see %s",
jl("user_stack"),
)
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] !=
project.pipfile.hash()["sha256"]):
_LOGGER.error(
"Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile "
"adjusted? This error is not critical.",
project.pipfile_lock.meta.hash["sha256"][:6],
project.pipfile.hash()["sha256"][:6],
)
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
_LOGGER.info("Using requirements.txt file located in %r directory",
os.getcwd())
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
return (
project.pipfile.to_string(),
project.pipfile_lock.to_string() if project.pipfile_lock else None,
)
def test_from_pip_compile_files_example_dir3(self) -> None:
"""Test loading only if only requirements.in is present."""
with cwd(os.path.join(self.data_dir, "requirements", "example_dir3")):
project = Project.from_pip_compile_files(allow_without_lock=True)
assert project.pipfile_lock is None
assert list(project.iter_dependencies()) == [
PackageVersion(name="flask", version="*", develop=False, hashes=[], index=None)
]
def test_no_include_no_pipenv_files(
self, builder_context: PipelineBuilderContext) -> None:
"""Test if Pipenv files are not supplied - no hash is available to check against."""
requirements_in = str(self.data_dir / "projects" / "requirements.in")
requirements_txt = str(self.data_dir / "projects" / "requirements.txt")
project = Project.from_pip_compile_files(
requirements_path=requirements_in,
requirements_lock_path=requirements_txt,
)
builder_context.project = project
assert list(self.UNIT_TESTED.should_include(builder_context)) == []
def qeb_hwt_thamos_advise() -> None:
"""Qeb-Hwt Thamos Advise Task."""
if not Configuration._REPO_PATH:
raise Exception("No path has been provided to REPO_PATH env variable.")
if not Path(Configuration._REPO_PATH).exists():
raise FileNotFoundError(
f"Cannot find the file on this path: {Configuration._REPO_PATH}")
output_messages: list = []
OpenShift.verify_github_app_inputs(
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
)
os.chdir(Configuration._REPO_PATH)
thoth_yaml_config = _Configuration()
if not thoth_yaml_config.config_file_exists():
exception_message = _create_message_config_file_error(no_file=True)
store_messages(output_messages)
trigger_finished_webhook(exception_message=exception_message,
has_error=True,
error_type="MissingThothYamlFile")
return
try:
# Consider first runtime environment
runtime_environment = thoth_yaml_config.content.get(
"runtime_environments")[0]
# Fetch recommendation type
recommendation_type = runtime_environment.get(
"recommendation_type") if thoth_yaml_config else "latest"
requirements_format = thoth_yaml_config.requirements_format
if requirements_format == "pipenv":
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
pipfile = project.pipfile.to_string()
pipfile_lock_str = project.pipfile_lock.to_string(
) if project.pipfile_lock else ""
application_stack = PythonStack(
requirements=pipfile,
requirements_lock=pipfile_lock_str,
requirements_format=requirements_format).to_dict()
except Exception as exception:
_LOGGER.debug(json.loads(exception.body)["error"]) # type: ignore
exception_message = json.loads(exception.body)["error"] # type: ignore
store_messages(output_messages)
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
return
# The input for AdviserTriggerMessage if no exceptions were found
message_input = {
"component_name": {
"type": "str",
"value": __COMPONENT_NAME__
},
"service_version": {
"type": "str",
"value": __service_version__
},
"application_stack": {
"type": "Dict",
"value": application_stack
},
"runtime_environment": {
"type": "Dict",
"value": runtime_environment
},
"recommendation_type": {
"type": "str",
"value": recommendation_type
},
"github_event_type": {
"type": "str",
"value": Configuration._GITHUB_EVENT_TYPE
},
"github_check_run_id": {
"type": "int",
"value": int(Configuration._GITHUB_CHECK_RUN_ID)
},
"github_installation_id": {
"type": "int",
"value": int(Configuration._GITHUB_INSTALLATION_ID)
},
"github_base_repo_url": {
"type": "str",
"value": Configuration._GITHUB_BASE_REPO_URL
},
"origin": {
"type": "str",
"value": Configuration._ORIGIN
},
"source_type": {
"type": "str",
"value": ThothAdviserIntegrationEnum.GITHUB_APP.name
},
}
# We store the message to put in the output file here.
output_messages = [{
"topic_name": "thoth.adviser-trigger",
"message_contents": message_input
}]
store_messages(output_messages)
def update_keb_installation():
"""Load files and pass them to storages update function."""
if _SLUG is None:
_LOGGER.info("No slug present, continuing to next step in task.")
return
service = GithubService(
github_app_id=os.getenv("GITHUB_APP_ID"),
github_app_private_key_path=os.getenv("GITHUB_PRIVATE_KEY_PATH"),
) # TODO: extend to use other services
project = service.get_project(namespace=_SLUG.split("/")[0], repo=_SLUG.split("/")[1])
raw_thoth_config = project.get_file_content(".thoth.yaml")
with TemporaryDirectory() as repo_path, cwd(repo_path):
thoth_config.load_config_from_string(raw_thoth_config)
requirements_format = thoth_config.content["requirements_format"]
overlays_dir = thoth_config.content.get("overlays_dir")
to_update: List[RuntimeEnvironment]
if overlays_dir is not None:
to_update = [RuntimeEnvironment.from_dict(r) for r in thoth_config.list_runtime_environments()]
else:
to_update = [RuntimeEnvironment.from_dict(thoth_config.get_runtime_environment())]
for runtime_environment in to_update:
if overlays_dir:
prefix = f"{overlays_dir}/{runtime_environment.name}/"
else:
prefix = ""
if requirements_format == "pipenv":
pipfile_r = project.get_file_content(f"{prefix}Pipfile")
with open("Pipfile", "wb") as f:
f.write(pipfile_r)
try:
piplock_r = project.get_file_content(f"{prefix}Pipfile.lock")
with open("Pipfile.lock", "wb") as f:
f.write(piplock_r)
project = Project.from_files(pipfile_path="Pipfile", pipfile_lock_path="Pipfile.lock")
except Exception:
_LOGGER.debug("No Pipfile.lock found")
project = Project.from_files(
pipfile_path="Pipfile",
without_pipfile_lock=True,
runtime_environment=runtime_environment,
)
elif requirements_format in ["pip", "pip-tools", "pip-compile"]:
try:
requirements_r = project.get_file_content(f"{prefix}requirements.txt")
with open("requirements.txt", "wb") as f:
f.write(requirements_r)
project = Project.from_pip_compile_files(
requirements_path="requirements.txt",
allow_without_lock=True,
runtime_environment=runtime_environment,
)
except Exception:
_LOGGER.debug("No requirements.txt found, trying to download requirements.in")
requirements_r = project.get_file_content(f"{prefix}requirements.in")
with open("requirements.in", "wb") as f:
f.write(requirements_r.content)
project = Project.from_pip_compile_files(
requirements_path="requirements.in",
allow_without_lock=True,
runtime_environment=runtime_environment,
)
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise NotImplementedError(f"{requirements_format} requirements format not supported.")
db.update_kebechet_installation_using_files(
slug=_SLUG,
runtime_environment_name=runtime_environment.name,
installation_id=str(project.github_repo.id),
requirements=project.pipfile.to_dict(),
requirements_lock=project.pipfile_lock.to_dict(),
thoth_config=thoth_config,
)
present_installations = db.get_kebechet_github_app_installations_all(slug=_SLUG)
cur_env_names = {r.name for r in to_update}
all_env_names = {installation["runtime_environment_name"] for installation in present_installations}
to_delete = all_env_names - cur_env_names
for name in to_delete:
db.delete_kebechet_github_app_installations(slug=_SLUG, runtime_environment=name)
def advise_here(
recommendation_type: typing.Optional[str] = None,
*,
runtime_environment: dict = None,
runtime_environment_name: typing.Optional[str] = None,
dev: bool = False,
no_static_analysis: bool = False,
no_user_stack: bool = False,
nowait: bool = False,
force: bool = False,
limit: typing.Optional[int] = None,
count: int = 1,
debug: bool = False,
origin: typing.Optional[str] = None,
github_event_type: typing.Optional[str] = None,
github_check_run_id: typing.Optional[int] = None,
github_installation_id: typing.Optional[int] = None,
github_base_repo_url: typing.Optional[str] = None,
source_type: typing.Optional[ThothAdviserIntegrationEnum] = None,
) -> typing.Optional[tuple]:
"""Run advise in current directory, requires no arguments."""
requirements_format = thoth_config.requirements_format
if requirements_format == "pipenv":
_LOGGER.info(
"Using Pipenv files located in the project root directory")
pipfile_lock_exists = os.path.exists("Pipfile.lock")
if pipfile_lock_exists:
_LOGGER.info(
"Submitting Pipfile.lock as a base for user's stack scoring - see %s",
jl("user_stack"),
)
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] !=
project.pipfile.hash()["sha256"]):
_LOGGER.error(
"Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile "
"adjusted? This error is not critical.",
project.pipfile_lock.meta.hash["sha256"][:6],
project.pipfile.hash()["sha256"][:6],
)
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
_LOGGER.info(
"Using requirements.txt file located in the project root directory"
)
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
pipfile = project.pipfile.to_string()
pipfile_lock_str = project.pipfile_lock.to_string(
) if project.pipfile_lock else ""
return advise(
pipfile=pipfile,
pipfile_lock=pipfile_lock_str,
recommendation_type=recommendation_type,
runtime_environment=runtime_environment,
runtime_environment_name=runtime_environment_name,
dev=dev,
no_static_analysis=no_static_analysis,
no_user_stack=no_user_stack,
nowait=nowait,
force=force,
limit=limit,
count=count,
debug=debug,
origin=origin,
source_type=source_type,
github_event_type=github_event_type,
github_check_run_id=github_check_run_id,
github_installation_id=github_installation_id,
github_base_repo_url=github_base_repo_url,
)
