def _get_requirements_txt_dependencies(req_dev: bool) -> dict:
"""Gather dependencies from fully pinned down stack.
Gather dependencies from either requirements.txt or requirements-dev.txt file,
our requirements.txt and requirements-dev.txt holds fully pinned down stack.
"""
result = {}
input_file = 'requirements-dev.txt' if req_dev else 'requirements.txt'
with open(input_file, 'r') as requirements_file:
content = requirements_file.read()
for line in content.splitlines():
if line.strip().startswith(('#', '-')):
continue
package_and_version = line.split('==', maxsplit=1)
if len(package_and_version) != 2:
raise DependencyManagementError(
f"File {input_file} does not state fully locked "
f"dependencies: {line!r} is not fully qualified dependency"
)
package_name, package_version = package_and_version
result[package_name] = {
# FIXME: tabs?
'version': package_version.split(r' ', maxsplit=1)[0],
'dev': False
}
return result
def _get_dependency_version(dependency: str, is_dev: bool) -> str:
"""Get version of the given dependency from Pipfile.lock."""
try:
with open("Pipfile.lock") as pipfile_lock:
pipfile_lock_content = json.load(pipfile_lock)
except Exception as exc:
# TODO: open a PR to fix this
raise DependencyManagementError(
f"Failed to load Pipfile.lock file: {str(exc)}"
) from exc
# We look for normalized dependency in Pipfile.lock.
normalized_dependency = re.sub(r"[-_.]+", "-", dependency).lower()
version = (
pipfile_lock_content["develop" if is_dev else "default"]
.get(normalized_dependency, {})
.get("version")
)
# if not kept as normaized depedency in Pipfile.lock.
version = (
pipfile_lock_content["develop" if is_dev else "default"]
.get(dependency, {})
.get("version")
)
if not version:
raise InternalError(
f"Failed to retrieve version information for dependency {dependency}, (dev: {is_dev})"
)
return version[len("==") :]
def _get_requirements_txt_dependencies(req_dev: bool) -> dict:
"""Gather dependencies from fully pinned down stack.
Gather dependencies from either requirements.txt or requirements-dev.txt file,
our requirements.txt and requirements-dev.txt holds fully pinned down stack.
"""
result = {}
input_file = "requirements-dev.txt" if req_dev else "requirements.txt"
with open(input_file, "r") as requirements_file:
content = requirements_file.read()
for line in content.splitlines():
line = line.strip()
if line.startswith(("#", "-")) or not line:
continue
package_and_version = line.split("==", maxsplit=1)
if len(package_and_version) != 2:
raise DependencyManagementError(
f"File {input_file} does not state fully locked "
f"dependencies: {line!r} is not fully qualified dependency"
)
package_name, package_version = package_and_version
result[package_name] = {
# FIXME: tabs?
"version": package_version.split(r" ", maxsplit=1)[0],
"dev": False,
}
return result
def _get_all_packages_versions() -> dict:
"""Parse Pipfile.lock file and retrieve all packages in the corresponding locked versions."""
try:
with open('Pipfile.lock') as pipfile_lock:
pipfile_lock_content = json.load(pipfile_lock)
except Exception as exc:
# TODO: open a PR to fix this
raise DependencyManagementError(
f"Failed to load Pipfile.lock file: {str(exc)}") from exc
result = {}
for package_name, package_info in pipfile_lock_content[
'default'].items():
result[package_name.lower()] = {
'dev': False,
'version': package_info['version'][len('=='):]
}
for package_name, package_info in pipfile_lock_content[
'develop'].items():
result[package_name.lower()] = {
'dev': False,
'version': package_info['version'][len('=='):]
}
return result
def _relock_all(self, exc: PipenvError, labels: list) -> None:
"""Re-lock all dependencies given the Pipfile."""
pip_url = construct_raw_file_url(
self.service_url, self.slug, "Pipfile", self.service_type
)
piplock_url = construct_raw_file_url(
self.service_url, self.slug, "Pipfile.lock", self.service_type
)
issue = self.sm.open_issue_if_not_exist(
_ISSUE_REPLICATE_ENV_NAME,
lambda: ISSUE_REPLICATE_ENV.format(
**exc.__dict__,
sha=self.sha,
pip_url=pip_url,
piplock_url=piplock_url,
environment_details=self.get_environment_details(),
),
refresh_comment=partial(self._add_refresh_comment, exc),
labels=labels,
)
self._pipenv_update_all()
commit_msg = "Automatic dependency re-locking"
branch_name = "kebechet-dependency-relock"
existing_prs = self._get_prs(branch_name)
if len(existing_prs) == 1:
pr = list(existing_prs)[0]
commits = pr.get_all_commits()
if len(commits) != 1:
pr.comment(
"There have been done changes in the original pull request (multiple commits found), "
"aborting doing changes to the modified opened pull request"
)
return None
if self.sha != commits[0]:
self._git_push(
":pushpin: " + commit_msg,
branch_name,
["Pipfile.lock"],
force_push=True,
)
pr.comment(
f"Pull request has been rebased on top of the current master with SHA {self.sha}"
)
elif len(existing_prs) == 0:
# Default case
self._git_push(":pushpin: " + commit_msg, branch_name, ["Pipfile.lock"])
pr_id = self.sm.open_merge_request(
commit_msg, branch_name, f"Fixes: #{issue.id}", labels
)
_LOGGER.info(
f"Issued automatic dependency re-locking in PR #{pr_id} to fix issue #{issue.id}"
)
else:
raise DependencyManagementError(
f"Found two or more pull requests for automatic relock for branch {branch_name}"
)
def _get_all_packages_versions(self) -> dict:
"""Parse Pipfile.lock file and retrieve all packages in the corresponding locked versions."""
try:
with open("Pipfile.lock") as pipfile_lock:
pipfile_lock_content = json.load(pipfile_lock)
except Exception as exc:
# TODO: open a PR to fix this
raise DependencyManagementError(
f"Failed to load Pipfile.lock file: {str(exc)}"
) from exc
result = {}
for package_name, package_info in pipfile_lock_content["default"].items():
if "git" in package_info:
self._create_unsupported_package_issue(package_name)
raise DependencyManagementError(
"Failed to find version in package that uses git source."
)
result[package_name.lower()] = {
"dev": False,
"version": package_info["version"][len("==") :],
}
for package_name, package_info in pipfile_lock_content["develop"].items():
if "git" in package_info:
self._create_unsupported_package_issue(package_name)
raise DependencyManagementError(
"Failed to find version in package that uses git source."
)
result[package_name.lower()] = {
"dev": False,
"version": package_info["version"][len("==") :],
}
# Close git as a source issues.
issue = self.get_issue_by_title(_ISSUE_UNSUPPORTED_PACKAGE)
if issue:
issue.comment(
f"Issue closed as no packages use git as a source anymore. Related SHA - {self.sha}"
)
issue.close()
return result
def _create_pipenv_environment(cls, input_file: str) -> None:
"""Create a pipenv environment - Pipfile and Pipfile.lock from requirements.in or requirements-dev.in file."""
if os.path.isfile(input_file) and input_file == "requirements.in":
_LOGGER.info(f"Installing dependencies from {input_file}")
cls.run_pipenv(f"pipenv install -r {input_file}")
elif os.path.isfile(input_file) and input_file == "requirements-dev.in":
_LOGGER.info(f"Installing dependencies from {input_file}")
cls.run_pipenv(f"pipenv install -r {input_file} --dev")
else:
raise DependencyManagementError(
"No dependency management found in the repo - no Pipfile nor requirements.in nor requirements-dev.in"
)
def _create_initial_lock(self, labels: list, pipenv_used: bool, req_dev: bool) -> bool:
"""Perform initial requirements lock into requirements.txt file."""
# We use lock_func to optimize run - it will be called only if actual locking needs to be performed.
if not pipenv_used and not os.path.isfile('requirements.txt') and not req_dev:
_LOGGER.info("Initial lock based on requirements.in will be done")
lock_func = partial(self._pipenv_lock_requirements, 'requirements.txt')
elif not pipenv_used and not os.path.isfile('requirements-dev.txt') and req_dev:
_LOGGER.info("Initial lock based on requirements-dev.in will be done")
lock_func = partial(self._pipenv_lock_requirements, 'requirements-dev.txt')
elif pipenv_used and not os.path.isfile('Pipfile.lock'):
_LOGGER.info("Initial lock based on Pipfile will be done")
lock_func = partial(self.run_pipenv, 'pipenv lock')
else:
return False
branch_name = "kebechet-initial-lock"
request = {mr for mr in self.sm.repository.merge_requests
if mr.head_branch_name == branch_name and mr.state in ('opened', 'open')}
if req_dev and not pipenv_used:
files = ['requirements-dev.txt']
elif not req_dev and not pipenv_used:
files = ['requirements.txt']
else:
files = ['Pipfile.lock']
commit_msg = "Initial dependency lock"
if len(request) == 0:
lock_func()
self._git_push(commit_msg, branch_name, files)
request = self.sm.open_merge_request(commit_msg, branch_name, body='', labels=labels)
_LOGGER.info(f"Initial dependency lock present in PR #{request.number}")
elif len(request) == 1:
request = list(request)[0]
commits = request.commits
if len(request.commits) != 1:
_LOGGER.info("There have been done changes in the original pull request (multiple commits found), "
"aborting doing changes to the adjusted opened pull request")
return False
if self.sha != commits[0].parent.sha:
lock_func()
self._git_push(commit_msg, branch_name, files, force_push=True)
request.add_comment(f"Pull request has been rebased on top of the current master with SHA {self.sha}")
else:
_LOGGER.info(f"Pull request #{request.number} is up to date for the current master branch")
else:
raise DependencyManagementError(
f"Found two or more pull requests for initial requirements lock for branch {branch_name}"
)
return True
def _get_direct_dependencies() -> tuple:
"""Get all direct dependencies stated in the Pipfile file."""
try:
pipfile_content = toml.load('Pipfile')
except Exception as exc:
# TODO: open a PR to fix this
raise DependencyManagementError(f"Failed to load Pipfile: {str(exc)}") from exc
default = list(package_name.lower() for package_name in pipfile_content['packages'].keys())
develop = list(package_name.lower() for package_name in pipfile_content['dev-packages'].keys())
return default, develop
def _get_dependency_version(dependency: str, is_dev: bool) -> str:
"""Get version of the given dependency from Pipfile.lock."""
try:
with open('Pipfile.lock') as pipfile_lock:
pipfile_lock_content = json.load(pipfile_lock)
except Exception as exc:
# TODO: open a PR to fix this
raise DependencyManagementError(f"Failed to load Pipfile.lock file: {str(exc)}") from exc
version = pipfile_lock_content['develop' if is_dev else 'default'].get(
dependency, {}).get('version')
if not version:
raise InternalError(
f"Failed to retrieve version information for dependency {dependency}, (dev: {is_dev})")
return version[len('=='):]
def _create_initial_lock(
self, labels: list, pipenv_used: bool, req_dev: bool
) -> bool:
"""Perform initial requirements lock into requirements.txt file."""
# We use lock_func to optimize run - it will be called only if actual locking needs to be performed.
if not pipenv_used and not os.path.isfile("requirements.txt") and not req_dev:
_LOGGER.info("Initial lock based on requirements.in will be done")
lock_func = partial(self._pipenv_lock_requirements, "requirements.txt")
elif not pipenv_used and not os.path.isfile("requirements-dev.txt") and req_dev:
_LOGGER.info("Initial lock based on requirements-dev.in will be done")
lock_func = partial(self._pipenv_lock_requirements, "requirements-dev.txt")
elif pipenv_used and not os.path.isfile("Pipfile.lock"):
_LOGGER.info("Initial lock based on Pipfile will be done")
lock_func = partial(self.run_pipenv, "pipenv lock")
else:
return False
branch_name = "kebechet-initial-lock"
request = self._get_prs(source_branch_name=branch_name)
if req_dev and not pipenv_used:
files = ["requirements-dev.txt"]
elif not req_dev and not pipenv_used:
files = ["requirements.txt"]
else:
files = ["Pipfile.lock"]
commit_msg = "Initial dependency lock"
if len(request) == 0:
lock_func()
self._git_push(commit_msg, branch_name, files)
request = self.sm.open_merge_request(
commit_msg, branch_name, body="", labels=labels
)
_LOGGER.info(
f"Initial dependency lock present in PR #{request.id}" # type: ignore
)
elif len(request) == 1:
request = list(request)[0]
commits = request.get_all_commits() # type: ignore
if len(commits) != 1:
_LOGGER.info(
"There have been done changes in the original pull request (multiple commits found), "
"aborting doing changes to the adjusted opened pull request"
)
return False
if self.sha != commits[0]:
lock_func()
self._git_push(commit_msg, branch_name, files, force_push=True)
request.comment( # type: ignore
f"Pull request has been rebased on top of the current master with SHA {self.sha}"
)
else:
_LOGGER.info(
f"Pull request #{request.id} is up to date for the current master branch" # type: ignore
)
else:
raise DependencyManagementError(
f"Found two or more pull requests for initial requirements lock for branch {branch_name}"
)
return True
