def _get_requirements_txt_dependencies(req_dev: bool) -> dict: """Gather dependencies from fully pinned down stack. Gather dependencies from either requirements.txt or requirements-dev.txt file, our requirements.txt and requirements-dev.txt holds fully pinned down stack. """ result = {} input_file = 'requirements-dev.txt' if req_dev else 'requirements.txt' with open(input_file, 'r') as requirements_file: content = requirements_file.read() for line in content.splitlines(): if line.strip().startswith(('#', '-')): continue package_and_version = line.split('==', maxsplit=1) if len(package_and_version) != 2: raise DependencyManagementError( f"File {input_file} does not state fully locked " f"dependencies: {line!r} is not fully qualified dependency" ) package_name, package_version = package_and_version result[package_name] = { # FIXME: tabs? 'version': package_version.split(r' ', maxsplit=1)[0], 'dev': False } return result
def _get_dependency_version(dependency: str, is_dev: bool) -> str: """Get version of the given dependency from Pipfile.lock.""" try: with open("Pipfile.lock") as pipfile_lock: pipfile_lock_content = json.load(pipfile_lock) except Exception as exc: # TODO: open a PR to fix this raise DependencyManagementError( f"Failed to load Pipfile.lock file: {str(exc)}" ) from exc # We look for normalized dependency in Pipfile.lock. normalized_dependency = re.sub(r"[-_.]+", "-", dependency).lower() version = ( pipfile_lock_content["develop" if is_dev else "default"] .get(normalized_dependency, {}) .get("version") ) # if not kept as normaized depedency in Pipfile.lock. version = ( pipfile_lock_content["develop" if is_dev else "default"] .get(dependency, {}) .get("version") ) if not version: raise InternalError( f"Failed to retrieve version information for dependency {dependency}, (dev: {is_dev})" ) return version[len("==") :]
def _get_requirements_txt_dependencies(req_dev: bool) -> dict: """Gather dependencies from fully pinned down stack. Gather dependencies from either requirements.txt or requirements-dev.txt file, our requirements.txt and requirements-dev.txt holds fully pinned down stack. """ result = {} input_file = "requirements-dev.txt" if req_dev else "requirements.txt" with open(input_file, "r") as requirements_file: content = requirements_file.read() for line in content.splitlines(): line = line.strip() if line.startswith(("#", "-")) or not line: continue package_and_version = line.split("==", maxsplit=1) if len(package_and_version) != 2: raise DependencyManagementError( f"File {input_file} does not state fully locked " f"dependencies: {line!r} is not fully qualified dependency" ) package_name, package_version = package_and_version result[package_name] = { # FIXME: tabs? "version": package_version.split(r" ", maxsplit=1)[0], "dev": False, } return result
def _get_all_packages_versions() -> dict: """Parse Pipfile.lock file and retrieve all packages in the corresponding locked versions.""" try: with open('Pipfile.lock') as pipfile_lock: pipfile_lock_content = json.load(pipfile_lock) except Exception as exc: # TODO: open a PR to fix this raise DependencyManagementError( f"Failed to load Pipfile.lock file: {str(exc)}") from exc result = {} for package_name, package_info in pipfile_lock_content[ 'default'].items(): result[package_name.lower()] = { 'dev': False, 'version': package_info['version'][len('=='):] } for package_name, package_info in pipfile_lock_content[ 'develop'].items(): result[package_name.lower()] = { 'dev': False, 'version': package_info['version'][len('=='):] } return result
def _relock_all(self, exc: PipenvError, labels: list) -> None: """Re-lock all dependencies given the Pipfile.""" pip_url = construct_raw_file_url( self.service_url, self.slug, "Pipfile", self.service_type ) piplock_url = construct_raw_file_url( self.service_url, self.slug, "Pipfile.lock", self.service_type ) issue = self.sm.open_issue_if_not_exist( _ISSUE_REPLICATE_ENV_NAME, lambda: ISSUE_REPLICATE_ENV.format( **exc.__dict__, sha=self.sha, pip_url=pip_url, piplock_url=piplock_url, environment_details=self.get_environment_details(), ), refresh_comment=partial(self._add_refresh_comment, exc), labels=labels, ) self._pipenv_update_all() commit_msg = "Automatic dependency re-locking" branch_name = "kebechet-dependency-relock" existing_prs = self._get_prs(branch_name) if len(existing_prs) == 1: pr = list(existing_prs)[0] commits = pr.get_all_commits() if len(commits) != 1: pr.comment( "There have been done changes in the original pull request (multiple commits found), " "aborting doing changes to the modified opened pull request" ) return None if self.sha != commits[0]: self._git_push( ":pushpin: " + commit_msg, branch_name, ["Pipfile.lock"], force_push=True, ) pr.comment( f"Pull request has been rebased on top of the current master with SHA {self.sha}" ) elif len(existing_prs) == 0: # Default case self._git_push(":pushpin: " + commit_msg, branch_name, ["Pipfile.lock"]) pr_id = self.sm.open_merge_request( commit_msg, branch_name, f"Fixes: #{issue.id}", labels ) _LOGGER.info( f"Issued automatic dependency re-locking in PR #{pr_id} to fix issue #{issue.id}" ) else: raise DependencyManagementError( f"Found two or more pull requests for automatic relock for branch {branch_name}" )
def _get_all_packages_versions(self) -> dict: """Parse Pipfile.lock file and retrieve all packages in the corresponding locked versions.""" try: with open("Pipfile.lock") as pipfile_lock: pipfile_lock_content = json.load(pipfile_lock) except Exception as exc: # TODO: open a PR to fix this raise DependencyManagementError( f"Failed to load Pipfile.lock file: {str(exc)}" ) from exc result = {} for package_name, package_info in pipfile_lock_content["default"].items(): if "git" in package_info: self._create_unsupported_package_issue(package_name) raise DependencyManagementError( "Failed to find version in package that uses git source." ) result[package_name.lower()] = { "dev": False, "version": package_info["version"][len("==") :], } for package_name, package_info in pipfile_lock_content["develop"].items(): if "git" in package_info: self._create_unsupported_package_issue(package_name) raise DependencyManagementError( "Failed to find version in package that uses git source." ) result[package_name.lower()] = { "dev": False, "version": package_info["version"][len("==") :], } # Close git as a source issues. issue = self.get_issue_by_title(_ISSUE_UNSUPPORTED_PACKAGE) if issue: issue.comment( f"Issue closed as no packages use git as a source anymore. Related SHA - {self.sha}" ) issue.close() return result
def _create_pipenv_environment(cls, input_file: str) -> None: """Create a pipenv environment - Pipfile and Pipfile.lock from requirements.in or requirements-dev.in file.""" if os.path.isfile(input_file) and input_file == "requirements.in": _LOGGER.info(f"Installing dependencies from {input_file}") cls.run_pipenv(f"pipenv install -r {input_file}") elif os.path.isfile(input_file) and input_file == "requirements-dev.in": _LOGGER.info(f"Installing dependencies from {input_file}") cls.run_pipenv(f"pipenv install -r {input_file} --dev") else: raise DependencyManagementError( "No dependency management found in the repo - no Pipfile nor requirements.in nor requirements-dev.in" )
def _create_initial_lock(self, labels: list, pipenv_used: bool, req_dev: bool) -> bool: """Perform initial requirements lock into requirements.txt file.""" # We use lock_func to optimize run - it will be called only if actual locking needs to be performed. if not pipenv_used and not os.path.isfile('requirements.txt') and not req_dev: _LOGGER.info("Initial lock based on requirements.in will be done") lock_func = partial(self._pipenv_lock_requirements, 'requirements.txt') elif not pipenv_used and not os.path.isfile('requirements-dev.txt') and req_dev: _LOGGER.info("Initial lock based on requirements-dev.in will be done") lock_func = partial(self._pipenv_lock_requirements, 'requirements-dev.txt') elif pipenv_used and not os.path.isfile('Pipfile.lock'): _LOGGER.info("Initial lock based on Pipfile will be done") lock_func = partial(self.run_pipenv, 'pipenv lock') else: return False branch_name = "kebechet-initial-lock" request = {mr for mr in self.sm.repository.merge_requests if mr.head_branch_name == branch_name and mr.state in ('opened', 'open')} if req_dev and not pipenv_used: files = ['requirements-dev.txt'] elif not req_dev and not pipenv_used: files = ['requirements.txt'] else: files = ['Pipfile.lock'] commit_msg = "Initial dependency lock" if len(request) == 0: lock_func() self._git_push(commit_msg, branch_name, files) request = self.sm.open_merge_request(commit_msg, branch_name, body='', labels=labels) _LOGGER.info(f"Initial dependency lock present in PR #{request.number}") elif len(request) == 1: request = list(request)[0] commits = request.commits if len(request.commits) != 1: _LOGGER.info("There have been done changes in the original pull request (multiple commits found), " "aborting doing changes to the adjusted opened pull request") return False if self.sha != commits[0].parent.sha: lock_func() self._git_push(commit_msg, branch_name, files, force_push=True) request.add_comment(f"Pull request has been rebased on top of the current master with SHA {self.sha}") else: _LOGGER.info(f"Pull request #{request.number} is up to date for the current master branch") else: raise DependencyManagementError( f"Found two or more pull requests for initial requirements lock for branch {branch_name}" ) return True
def _get_direct_dependencies() -> tuple: """Get all direct dependencies stated in the Pipfile file.""" try: pipfile_content = toml.load('Pipfile') except Exception as exc: # TODO: open a PR to fix this raise DependencyManagementError(f"Failed to load Pipfile: {str(exc)}") from exc default = list(package_name.lower() for package_name in pipfile_content['packages'].keys()) develop = list(package_name.lower() for package_name in pipfile_content['dev-packages'].keys()) return default, develop
def _get_dependency_version(dependency: str, is_dev: bool) -> str: """Get version of the given dependency from Pipfile.lock.""" try: with open('Pipfile.lock') as pipfile_lock: pipfile_lock_content = json.load(pipfile_lock) except Exception as exc: # TODO: open a PR to fix this raise DependencyManagementError(f"Failed to load Pipfile.lock file: {str(exc)}") from exc version = pipfile_lock_content['develop' if is_dev else 'default'].get( dependency, {}).get('version') if not version: raise InternalError( f"Failed to retrieve version information for dependency {dependency}, (dev: {is_dev})") return version[len('=='):]
def _create_initial_lock( self, labels: list, pipenv_used: bool, req_dev: bool ) -> bool: """Perform initial requirements lock into requirements.txt file.""" # We use lock_func to optimize run - it will be called only if actual locking needs to be performed. if not pipenv_used and not os.path.isfile("requirements.txt") and not req_dev: _LOGGER.info("Initial lock based on requirements.in will be done") lock_func = partial(self._pipenv_lock_requirements, "requirements.txt") elif not pipenv_used and not os.path.isfile("requirements-dev.txt") and req_dev: _LOGGER.info("Initial lock based on requirements-dev.in will be done") lock_func = partial(self._pipenv_lock_requirements, "requirements-dev.txt") elif pipenv_used and not os.path.isfile("Pipfile.lock"): _LOGGER.info("Initial lock based on Pipfile will be done") lock_func = partial(self.run_pipenv, "pipenv lock") else: return False branch_name = "kebechet-initial-lock" request = self._get_prs(source_branch_name=branch_name) if req_dev and not pipenv_used: files = ["requirements-dev.txt"] elif not req_dev and not pipenv_used: files = ["requirements.txt"] else: files = ["Pipfile.lock"] commit_msg = "Initial dependency lock" if len(request) == 0: lock_func() self._git_push(commit_msg, branch_name, files) request = self.sm.open_merge_request( commit_msg, branch_name, body="", labels=labels ) _LOGGER.info( f"Initial dependency lock present in PR #{request.id}" # type: ignore ) elif len(request) == 1: request = list(request)[0] commits = request.get_all_commits() # type: ignore if len(commits) != 1: _LOGGER.info( "There have been done changes in the original pull request (multiple commits found), " "aborting doing changes to the adjusted opened pull request" ) return False if self.sha != commits[0]: lock_func() self._git_push(commit_msg, branch_name, files, force_push=True) request.comment( # type: ignore f"Pull request has been rebased on top of the current master with SHA {self.sha}" ) else: _LOGGER.info( f"Pull request #{request.id} is up to date for the current master branch" # type: ignore ) else: raise DependencyManagementError( f"Found two or more pull requests for initial requirements lock for branch {branch_name}" ) return True