def test_run_dev(self, context: Context) -> None:
"""Test discarding user's lock file if development dependencies are not present in lock but provided."""
pipfile = str(self.data_dir / "projects" / "dev" / "Pipfile")
pipfile_lock = str(self.data_dir / "projects" / "dev" / "Pipfile.lock")
project = Project.from_files(pipfile_path=pipfile,
pipfile_lock_path=pipfile_lock)
assert project.pipfile.dev_packages.packages
assert project.pipfile_lock.dev_packages.packages
# Remove packages.
project.pipfile_lock.dev_packages.packages.clear()
context.cli_parameters["dev"] = True
context.project = project
unit = self.UNIT_TESTED()
with unit.assigned_context(context):
unit.run()
assert project.pipfile.dev_packages.packages
assert not project.pipfile_lock, "Lock file was not removed from the input"
assert len(context.stack_info) == 1
assert self.verify_justification_schema(context.stack_info)
def get_project(
self,
runtime_environment_name: Optional[str] = None,
*,
missing_dir_ok: bool = False,
) -> Project:
"""Get the given overlay."""
path = self.get_overlays_directory(
runtime_environment_name=runtime_environment_name,
missing_dir_ok=missing_dir_ok,
)
runtime_environment = RuntimeEnvironment.from_dict(
self.get_runtime_environment(runtime_environment_name))
if self.requirements_format == "pipenv":
pipfile_lock_path: Optional[str] = os.path.join(
path, "Pipfile.lock")
if pipfile_lock_path and not os.path.exists(pipfile_lock_path):
pipfile_lock_path = None
pipfile_path = os.path.join(path, "Pipfile")
if not os.path.isfile(pipfile_path):
if not os.path.isdir(path):
_LOGGER.info("Creating directory structure in %r", path)
os.makedirs(path, exist_ok=True)
pipfile = Pipfile.from_dict({})
pipfile.to_file(path=pipfile_path)
project = Project.from_files(
pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path,
runtime_environment=runtime_environment,
without_pipfile_lock=pipfile_lock_path is None,
)
else:
requirements_in_file_path = os.path.join(path, "requirements.in")
if not os.path.isfile(requirements_in_file_path):
requirements_txt_file_path = os.path.join(
path, "requirements.txt")
if os.path.isfile(requirements_txt_file_path):
_LOGGER.warning("Using %r for direct dependencies",
requirements_in_file_path)
project = Project.from_pip_compile_files(
requirements_path=requirements_txt_file_path,
requirements_lock_path=None,
allow_without_lock=True,
runtime_environment=runtime_environment,
)
else:
raise NotImplementedError(
"No requirements.txt/requirements.in files found, it is recommended to "
"use Pipenv files for managing dependencies")
else:
project = Project.from_pip_compile_files(
requirements_path=requirements_in_file_path,
requirements_lock_path=None,
allow_without_lock=True,
runtime_environment=runtime_environment,
)
return project
def _load_files(requirements_format: str) -> Tuple[str, Optional[str]]:
"""Load Pipfile/Pipfile.lock or requirements.in/txt from the current directory."""
if requirements_format == "pipenv":
_LOGGER.info("Using Pipenv files located in %r directory", os.getcwd())
pipfile_lock_exists = os.path.exists("Pipfile.lock")
if pipfile_lock_exists:
_LOGGER.info(
"Submitting Pipfile.lock as a base for user's stack scoring - see %s",
jl("user_stack"),
)
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] !=
project.pipfile.hash()["sha256"]):
_LOGGER.error(
"Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile "
"adjusted? This error is not critical.",
project.pipfile_lock.meta.hash["sha256"][:6],
project.pipfile.hash()["sha256"][:6],
)
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
_LOGGER.info("Using requirements.txt file located in %r directory",
os.getcwd())
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
return (
project.pipfile.to_string(),
project.pipfile_lock.to_string() if project.pipfile_lock else None,
)
def post(self):
"""Get requirements file from disk."""
input_data = self.get_json_body()
kernel_name: str = input_data["kernel_name"]
home = Path.home()
store_path: Path = home.joinpath(".local/share/thoth/kernels")
env_path = Path(store_path).joinpath(kernel_name)
_LOGGER.info("Path used to get dependencies is: %r",
env_path.as_posix())
requirements_format = "pipenv"
pipfile_path = env_path.joinpath("Pipfile")
pipfile_lock_path = env_path.joinpath("Pipfile.lock")
if requirements_format == "pipenv":
_LOGGER.debug("Get Pipfile/Pipfile.lock in %r", env_path)
project = Project.from_files(pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path)
requirements = project.pipfile.to_dict()
requirements_locked = project.pipfile_lock.to_dict()
self.finish(
json.dumps({
"requirements": requirements,
"requirements_lock": requirements_locked
}))
def load_files(base_path: str) -> typing.Tuple[str, typing.Optional[str]]:
"""Load Pipfile/Pipfile.lock from path."""
_LOGGER.info("Looking for Pipenv files located in %r directory", base_path)
pipfile_path = Path(base_path).joinpath("Pipfile")
pipfile_lock_path = Path(base_path).joinpath("Pipfile.lock")
project = Project.from_files(
pipfile_path=pipfile_path,
pipfile_lock_path=pipfile_lock_path,
without_pipfile_lock=not pipfile_lock_path.exists(),
)
if pipfile_lock_path.exists() and project.pipfile_lock.meta.hash[
"sha256"] != project.pipfile.hash()["sha256"]:
_LOGGER.error(
"Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile "
"adjusted? This error is not critical.",
project.pipfile_lock.meta.hash["sha256"][:6],
project.pipfile.hash()["sha256"][:6],
)
return (
project.pipfile.to_string(),
project.pipfile_lock.to_string() if project.pipfile_lock else None,
)
def load_test_project(self, pinned_index: bool = False) -> Project:
"""Instantiate testing project from prepared testing Pipfile and Pipfile.lock files.
@param pinned_index: true for retrieving Pipfile with pinned index
"""
pipfile_path = os.path.join(
self.data_dir, "pipfiles", "Pipfile_provenance1" if not pinned_index else "Pipfile_provenance2",
)
pipfile_lock_path = os.path.join(
self.data_dir, "pipfiles", "Pipfile_provenance1.lock" if not pinned_index else "Pipfile_provenance2.lock",
)
return Project.from_files(pipfile_path, pipfile_lock_path)
def project() -> Project:
"""Create a fixture for a project representation."""
flexmock(Project)
flexmock(RuntimeEnvironment)
pipfile_path = AdviserTestCase.data_dir / "projects" / "Pipfile"
pipfile_lock_path = AdviserTestCase.data_dir / "projects" / "Pipfile.lock"
return Project.from_files(
pipfile_path=str(pipfile_path),
pipfile_lock_path=str(pipfile_lock_path),
runtime_environment=RuntimeEnvironment.from_dict({}),
)
def test_get_outdated_package_versions_indirect(self):
"""Test get outdated package versions indirect."""
# The difference between direct and indirect - Pipenv states "index" in
# the Pipfile.lock file if the given package is a direct dependency.
# The "index" key for indirect dependencies is omitted though. This way
# we check both - logic for indirect/direct is slightly different.
# We cannot use flexmock as Source has slots.
@attr.s
class MySource:
url = attr.ib(type=str)
verify_ssl = attr.ib(type=bool)
name = attr.ib(type=str)
warehouse = attr.ib(type=bool, default=False)
warehouse_api_url = attr.ib(default=None, type=str)
def get_latest_package_version(_, package_name): # noqa: N805
return {
"certifi":
PackageVersion.parse_semantic_version("2018.10.15"),
"chardet": PackageVersion.parse_semantic_version("3.0.4"),
"idna": PackageVersion.parse_semantic_version(
"2.10"), # Bumped from 2.7
"requests":
PackageVersion.parse_semantic_version("2.19.1"),
"termcolor":
PackageVersion.parse_semantic_version("1.1.0"),
"urllib3": PackageVersion.parse_semantic_version("1.23"),
}[package_name]
project = Project.from_files(
os.path.join(self.data_dir, "pipfiles", "Pipfile_test2"),
os.path.join(self.data_dir, "pipfiles", "Pipfile_test2.lock"),
)
new_sources = {}
for source in project.pipfile_lock.meta.sources.values():
new_sources[source.name] = MySource(**source.to_dict())
project.pipfile_lock.meta.sources = new_sources
for package_version in project.iter_dependencies_locked(
with_devel=True):
if package_version.index:
package_version.index = new_sources[package_version.index.name]
result = project.get_outdated_package_versions()
assert len(result) == 1
assert "idna" in result
assert len(result["idna"]) == 2
assert result["idna"][0] is project.pipfile_lock.packages["idna"]
assert isinstance(result["idna"][1], Version)
assert str(result["idna"][1]) == "2.10"
def test_get_outdated_package_versions_direct(self):
"""Test get outdated package versions direct."""
# See previous test comments for more info.
# We cannot use flexmock as Source has slots.
@attr.s
class MySource:
url = attr.ib(type=str)
verify_ssl = attr.ib(type=bool)
name = attr.ib(type=str)
warehouse = attr.ib(type=bool, default=False)
warehouse_api_url = attr.ib(default=None, type=str)
def get_latest_package_version(_, package_name): # noqa: N805
return {
"certifi":
PackageVersion.parse_semantic_version("2018.10.15"),
"chardet": PackageVersion.parse_semantic_version("3.0.4"),
"idna": PackageVersion.parse_semantic_version("2.7"),
"requests": PackageVersion.parse_semantic_version("3.0.0"),
"termcolor":
PackageVersion.parse_semantic_version("1.1.0"),
"urllib3": PackageVersion.parse_semantic_version("1.23"),
}[package_name]
project = Project.from_files(
os.path.join(self.data_dir, "pipfiles", "Pipfile_test2"),
os.path.join(self.data_dir, "pipfiles", "Pipfile_test2.lock"),
)
new_sources = {}
for source in project.pipfile_lock.meta.sources.values():
new_sources[source.name] = MySource(**source.to_dict())
project.pipfile_lock.meta.sources = new_sources
for package_version in project.iter_dependencies_locked(
with_devel=True):
if package_version.index:
package_version.index = new_sources[package_version.index.name]
result = project.get_outdated_package_versions()
assert len(result) == 1
assert "requests" in result
assert len(result["requests"]) == 2
assert result["requests"][0] is project.pipfile_lock.packages[
"requests"]
assert isinstance(result["requests"][1], Version)
assert str(result["requests"][1]) == "3.0.0"
def test_run_dev_noop_no_dev(self, context: Context) -> None:
"""Test not discarding user's lock file if development dependencies are present in lock."""
pipfile = str(self.data_dir / "projects" / "dev" / "Pipfile")
pipfile_lock = str(self.data_dir / "projects" / "dev" / "Pipfile.lock")
project = Project.from_files(pipfile_path=pipfile,
pipfile_lock_path=pipfile_lock)
assert project.pipfile.dev_packages.packages
assert project.pipfile_lock.dev_packages.packages
context.project = project
project.pipfile_lock.dev_packages.packages.clear()
context.cli_parameters["dev"] = False
unit = self.UNIT_TESTED()
with unit.assigned_context(context):
unit.run()
assert project.pipfile.dev_packages.packages
assert project.pipfile_lock
assert len(context.stack_info) == 0
def qeb_hwt_thamos_advise() -> None:
"""Qeb-Hwt Thamos Advise Task."""
if not Configuration._REPO_PATH:
raise Exception("No path has been provided to REPO_PATH env variable.")
if not Path(Configuration._REPO_PATH).exists():
raise FileNotFoundError(
f"Cannot find the file on this path: {Configuration._REPO_PATH}")
output_messages: list = []
OpenShift.verify_github_app_inputs(
github_event_type=Configuration._GITHUB_EVENT_TYPE,
github_check_run_id=Configuration._GITHUB_CHECK_RUN_ID,
github_installation_id=Configuration._GITHUB_INSTALLATION_ID,
github_base_repo_url=Configuration._GITHUB_BASE_REPO_URL,
origin=Configuration._ORIGIN,
)
os.chdir(Configuration._REPO_PATH)
thoth_yaml_config = _Configuration()
if not thoth_yaml_config.config_file_exists():
exception_message = _create_message_config_file_error(no_file=True)
store_messages(output_messages)
trigger_finished_webhook(exception_message=exception_message,
has_error=True,
error_type="MissingThothYamlFile")
return
try:
# Consider first runtime environment
runtime_environment = thoth_yaml_config.content.get(
"runtime_environments")[0]
# Fetch recommendation type
recommendation_type = runtime_environment.get(
"recommendation_type") if thoth_yaml_config else "latest"
requirements_format = thoth_yaml_config.requirements_format
if requirements_format == "pipenv":
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
pipfile = project.pipfile.to_string()
pipfile_lock_str = project.pipfile_lock.to_string(
) if project.pipfile_lock else ""
application_stack = PythonStack(
requirements=pipfile,
requirements_lock=pipfile_lock_str,
requirements_format=requirements_format).to_dict()
except Exception as exception:
_LOGGER.debug(json.loads(exception.body)["error"]) # type: ignore
exception_message = json.loads(exception.body)["error"] # type: ignore
store_messages(output_messages)
trigger_finished_webhook(exception_message=exception_message,
has_error=True)
return
# The input for AdviserTriggerMessage if no exceptions were found
message_input = {
"component_name": {
"type": "str",
"value": __COMPONENT_NAME__
},
"service_version": {
"type": "str",
"value": __service_version__
},
"application_stack": {
"type": "Dict",
"value": application_stack
},
"runtime_environment": {
"type": "Dict",
"value": runtime_environment
},
"recommendation_type": {
"type": "str",
"value": recommendation_type
},
"github_event_type": {
"type": "str",
"value": Configuration._GITHUB_EVENT_TYPE
},
"github_check_run_id": {
"type": "int",
"value": int(Configuration._GITHUB_CHECK_RUN_ID)
},
"github_installation_id": {
"type": "int",
"value": int(Configuration._GITHUB_INSTALLATION_ID)
},
"github_base_repo_url": {
"type": "str",
"value": Configuration._GITHUB_BASE_REPO_URL
},
"origin": {
"type": "str",
"value": Configuration._ORIGIN
},
"source_type": {
"type": "str",
"value": ThothAdviserIntegrationEnum.GITHUB_APP.name
},
}
# We store the message to put in the output file here.
output_messages = [{
"topic_name": "thoth.adviser-trigger",
"message_contents": message_input
}]
store_messages(output_messages)
def update_keb_installation():
"""Load files and pass them to storages update function."""
if _SLUG is None:
_LOGGER.info("No slug present, continuing to next step in task.")
return
service = GithubService(
github_app_id=os.getenv("GITHUB_APP_ID"),
github_app_private_key_path=os.getenv("GITHUB_PRIVATE_KEY_PATH"),
) # TODO: extend to use other services
project = service.get_project(namespace=_SLUG.split("/")[0], repo=_SLUG.split("/")[1])
raw_thoth_config = project.get_file_content(".thoth.yaml")
with TemporaryDirectory() as repo_path, cwd(repo_path):
thoth_config.load_config_from_string(raw_thoth_config)
requirements_format = thoth_config.content["requirements_format"]
overlays_dir = thoth_config.content.get("overlays_dir")
to_update: List[RuntimeEnvironment]
if overlays_dir is not None:
to_update = [RuntimeEnvironment.from_dict(r) for r in thoth_config.list_runtime_environments()]
else:
to_update = [RuntimeEnvironment.from_dict(thoth_config.get_runtime_environment())]
for runtime_environment in to_update:
if overlays_dir:
prefix = f"{overlays_dir}/{runtime_environment.name}/"
else:
prefix = ""
if requirements_format == "pipenv":
pipfile_r = project.get_file_content(f"{prefix}Pipfile")
with open("Pipfile", "wb") as f:
f.write(pipfile_r)
try:
piplock_r = project.get_file_content(f"{prefix}Pipfile.lock")
with open("Pipfile.lock", "wb") as f:
f.write(piplock_r)
project = Project.from_files(pipfile_path="Pipfile", pipfile_lock_path="Pipfile.lock")
except Exception:
_LOGGER.debug("No Pipfile.lock found")
project = Project.from_files(
pipfile_path="Pipfile",
without_pipfile_lock=True,
runtime_environment=runtime_environment,
)
elif requirements_format in ["pip", "pip-tools", "pip-compile"]:
try:
requirements_r = project.get_file_content(f"{prefix}requirements.txt")
with open("requirements.txt", "wb") as f:
f.write(requirements_r)
project = Project.from_pip_compile_files(
requirements_path="requirements.txt",
allow_without_lock=True,
runtime_environment=runtime_environment,
)
except Exception:
_LOGGER.debug("No requirements.txt found, trying to download requirements.in")
requirements_r = project.get_file_content(f"{prefix}requirements.in")
with open("requirements.in", "wb") as f:
f.write(requirements_r.content)
project = Project.from_pip_compile_files(
requirements_path="requirements.in",
allow_without_lock=True,
runtime_environment=runtime_environment,
)
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise NotImplementedError(f"{requirements_format} requirements format not supported.")
db.update_kebechet_installation_using_files(
slug=_SLUG,
runtime_environment_name=runtime_environment.name,
installation_id=str(project.github_repo.id),
requirements=project.pipfile.to_dict(),
requirements_lock=project.pipfile_lock.to_dict(),
thoth_config=thoth_config,
)
present_installations = db.get_kebechet_github_app_installations_all(slug=_SLUG)
cur_env_names = {r.name for r in to_update}
all_env_names = {installation["runtime_environment_name"] for installation in present_installations}
to_delete = all_env_names - cur_env_names
for name in to_delete:
db.delete_kebechet_github_app_installations(slug=_SLUG, runtime_environment=name)
def advise_here(
recommendation_type: typing.Optional[str] = None,
*,
runtime_environment: dict = None,
runtime_environment_name: typing.Optional[str] = None,
dev: bool = False,
no_static_analysis: bool = False,
no_user_stack: bool = False,
nowait: bool = False,
force: bool = False,
limit: typing.Optional[int] = None,
count: int = 1,
debug: bool = False,
origin: typing.Optional[str] = None,
github_event_type: typing.Optional[str] = None,
github_check_run_id: typing.Optional[int] = None,
github_installation_id: typing.Optional[int] = None,
github_base_repo_url: typing.Optional[str] = None,
source_type: typing.Optional[ThothAdviserIntegrationEnum] = None,
) -> typing.Optional[tuple]:
"""Run advise in current directory, requires no arguments."""
requirements_format = thoth_config.requirements_format
if requirements_format == "pipenv":
_LOGGER.info(
"Using Pipenv files located in the project root directory")
pipfile_lock_exists = os.path.exists("Pipfile.lock")
if pipfile_lock_exists:
_LOGGER.info(
"Submitting Pipfile.lock as a base for user's stack scoring - see %s",
jl("user_stack"),
)
project = Project.from_files(
without_pipfile_lock=not os.path.exists("Pipfile.lock"))
if (pipfile_lock_exists and project.pipfile_lock.meta.hash["sha256"] !=
project.pipfile.hash()["sha256"]):
_LOGGER.error(
"Pipfile hash stated in Pipfile.lock %r does not correspond to Pipfile hash %r - was Pipfile "
"adjusted? This error is not critical.",
project.pipfile_lock.meta.hash["sha256"][:6],
project.pipfile.hash()["sha256"][:6],
)
elif requirements_format in ("pip", "pip-tools", "pip-compile"):
_LOGGER.info(
"Using requirements.txt file located in the project root directory"
)
project = Project.from_pip_compile_files(allow_without_lock=True)
else:
raise ValueError(
f"Unknown configuration option for requirements format: {requirements_format!r}"
)
pipfile = project.pipfile.to_string()
pipfile_lock_str = project.pipfile_lock.to_string(
) if project.pipfile_lock else ""
return advise(
pipfile=pipfile,
pipfile_lock=pipfile_lock_str,
recommendation_type=recommendation_type,
runtime_environment=runtime_environment,
runtime_environment_name=runtime_environment_name,
dev=dev,
no_static_analysis=no_static_analysis,
no_user_stack=no_user_stack,
nowait=nowait,
force=force,
limit=limit,
count=count,
debug=debug,
origin=origin,
source_type=source_type,
github_event_type=github_event_type,
github_check_run_id=github_check_run_id,
github_installation_id=github_installation_id,
github_base_repo_url=github_base_repo_url,
)
