def fedpkg_push(directory, branch, fail=True): if not os.path.isdir(directory): raise ReleaseException("Cannot access fedpkg repository:") return shell_command(directory, "fedpkg push", f"Pushing branch {branch!r} to Fedora failed:", fail)
def make_new_github_release(self): def release_handler(success): result = "released" if success else "failed to release" msg = f"I just {result} version {self.new_release['version']} on Github" level = logging.INFO if success else logging.ERROR self.logger.log(level, msg) self.github.comment.append(msg) try: latest_release = self.github.latest_release() except ReleaseException as exc: raise ReleaseException( f"Failed getting latest Github release (zip).\n{exc}") if Version.coerce(latest_release) >= Version.coerce( self.new_release['version']): self.logger.info( f"{self.new_release['version']} has already been released on Github" ) else: try: released, self.new_release = self.github.make_new_release( self.new_release) if released: release_handler(success=True) except ReleaseException: release_handler(success=False) raise self.github.update_changelog(self.new_release['version']) return self.new_release
def fedpkg_merge(directory, branch, ff_only=True, fail=True): if not os.path.isdir(directory): raise ReleaseException("Cannot access fedpkg repository:") return shell_command( directory, f"git merge master {'--ff-only' if ff_only else ''}", f"Merging master to branch {branch!r} failed:", fail)
def make_new_release(self, new_release): """ Makes new release to Github. This has to be done using github api v3 because v4 (GraphQL) doesn't support this yet :param new_release: version number of the new release :return: tuple (released, new_release) - released is bool, new_release contains info about the new release """ payload = { "tag_name": new_release['version'], "target_commitish": new_release['commitish'], "name": new_release['version'], "prerelease": False, "draft": False } url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/" f"{self.conf.repository_name}/releases") self.logger.debug( f"About to release {new_release['version']} on Github") response = requests.post(url=url, headers=self.headers, json=payload) if response.status_code != 201: msg = f"Failed to create new release on github:\n{response.text}" raise ReleaseException(msg) released = True new_release = self.download_extract_zip(new_release) self.update_changelog(self.latest_release(), new_release['version'], new_release['fs_path'], response.json()['id']) return released, new_release
def make_new_github_release(self): def release_handler(success): result = "released" if success else "failed to release" msg = f"I just {result} version {self.new_release.version} on {self.git_service.name}" level = logging.INFO if success else logging.ERROR self.logger.log(level, msg) self.github.comment.append(msg) try: latest_release = self.github.latest_release() except ReleaseException as exc: raise ReleaseException( f"Failed getting latest {self.git_service.name} release (zip).\n{exc}" ) if Version.coerce(latest_release) >= Version.coerce( self.new_release.version): self.logger.info( f"{self.new_release.version} has already been released on {self.git_service.name}" ) else: try: if self.conf.dry_run: return None released, self.new_release = self.github.make_new_release( self.new_release) if released: release_handler(success=True) except ReleaseException: release_handler(success=False) raise return self.new_release
def make_pr(self, branch, version, log, changed_version_files, base='master', labels=None): """ Makes a pull request with info on the new release :param branch: name of the branch to make PR from :param version: version that is being released :param log: changelog :param changed_version_files: list of files that have been changed in order to update version :param base: base of the PR. 'master' by default :param labels: list of str, labels to be put on PR :return: url of the PR """ message = ( f'Hi,\n you have requested a release PR from me. Here it is!\n' f'This is the changelog I created:\n' f'### Changes\n{log}\n\nYou can change it by editing `CHANGELOG.md` ' f'in the root of this repository and pushing to `{branch}` branch' f' before merging this PR.\n') if len(changed_version_files) == 1: message += 'I have also updated the `__version__ ` in file:\n' elif len(changed_version_files) > 1: message += ( 'There were multiple files where `__version__ ` was set, ' 'so I left updating them up to you. These are the files:\n') elif not changed_version_files: message += "I didn't find any files where `__version__` is set." for file in changed_version_files: message += f'* {file}\n' payload = { 'title': f'{version} release', 'head': branch, 'base': base, 'body': message, 'maintainer_can_modify': True } url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/" f"{self.conf.repository_name}/pulls") self.logger.debug(f'Attempting a PR for {branch} branch') response = requests.post(url=url, headers=self.headers, json=payload) if response.status_code == 201: parsed = response.json() self.logger.info(f"Created PR: {parsed['html_url']}") # put labels on PR if labels is not None: self.put_labels_on_issue(parsed['number'], labels) return parsed['html_url'] else: msg = (f"Something went wrong with creating " f"PR on github:\n{response.text}") raise ReleaseException(msg)
def shell_command(work_directory, cmd, error_message, fail=True): """ Execute a shell command :param work_directory: A directory to execute the command in :param cmd: The shell command :param error_message: An error message to return in case of failure :param fail: If failure should cause termination of the bot :return: Boolean indicating success/failure """ cmd = shlex.split(cmd) shell = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, cwd=work_directory, universal_newlines=True) configuration.logger.debug(f"{shell.args}\n{shell.stdout}") if shell.returncode != 0: configuration.logger.error(f"{error_message}\n{shell.stderr}") if fail: raise ReleaseException( f"{shell.args!r} failed with {error_message!r}") return False return True
def fedpkg_new_sources(directory, branch, sources="", fail=True): if not os.path.isdir(directory): raise ReleaseException("Cannot access fedpkg repository:") return shell_command(directory, f"fedpkg new-sources {sources}", f"Adding new sources on branch {branch} failed:", fail)
def update_spec(spec_path, new_release): """ Update spec with new version and changelog for that version, change release to 1 :param spec_path: Path to package .spec file :param new_release: an array containing info about new release, see main() for definition """ if not os.path.isfile(spec_path): raise ReleaseException("No spec file found in dist-git repository!") # make changelog and get version locale.setlocale(locale.LC_TIME, "en_US.UTF-8") changelog = (f"* {datetime.datetime.now():%a %b %d %Y} {new_release['author_name']!s} " f"<{new_release['author_email']!s}> {new_release['version']}-1\n") # add entries if new_release.get('changelog'): for item in new_release['changelog']: changelog += f"- {item}\n" else: changelog += f"- {new_release['version']} release\n" # change the version and add changelog in spec file with open(spec_path, 'r+') as spec_file: spec = spec_file.read() # replace version spec = re.sub(r'(Version:\s*)([0-9]|[.])*', r'\g<1>' + new_release['version'], spec) # make release 1 spec = re.sub(r'(Release:\s*)([0-9]*)(.*)', r'\g<1>1\g<3>', spec) # insert changelog spec = re.sub(r'(%changelog\n)', r'\g<1>' + changelog + '\n', spec) # write and close spec_file.seek(0) spec_file.write(spec) spec_file.truncate() spec_file.close()
def make_new_release(self, new_release): """ Makes new release to Github. This has to be done using github api v3 because v4 (GraphQL) doesn't support this yet :param new_release: version number of the new release :return: tuple (released, new_release) - released is bool, new_release contains info about the new release """ payload = { "tag_name": new_release.version, "target_commitish": new_release.commitish, "name": new_release.version, "prerelease": False, "draft": False } url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/" f"{self.conf.repository_name}/releases") self.logger.debug(f"About to release {new_release.version} on Github") response = self.do_request(method="POST", url=url, json_payload=payload, use_github_auth=True) if response.status_code != 201: msg = f"Failed to create new release on github:\n{response.text}" raise ReleaseException(msg) return True, new_release
def make_new_github_release(self): def release_handler(success): result = "released" if success else "failed to release" msg = f"I just {result} version {self.new_release['version']} on Github" level = logging.INFO if success else logging.ERROR self.logger.log(level, msg) self.github.comment.append(msg) try: latest_github = self.github.latest_release() if Version.coerce(latest_github) >= Version.coerce( self.new_release['version']): self.logger.info( f"{self.new_release['version']} has already been released on Github" ) # to fill in new_release['fs_path'] so that we can continue with PyPi upload self.new_release = self.github.download_extract_zip( self.new_release) return self.new_release except ReleaseException as exc: raise ReleaseException( f"Failed getting latest Github release (zip).\n{exc}") try: released, self.new_release = self.github.make_new_release( self.new_release) if released: release_handler(success=True) except ReleaseException: release_handler(success=False) raise return self.new_release
def fedpkg_sources(directory, branch, fail=True): if not os.path.isdir(directory): raise ReleaseException("Cannot access fedpkg repository:") return shell_command( directory, "fedpkg sources", f"Retrieving sources for branch {branch} failed:", fail)
def make_release_pr(self, new_pr, gitchangelog): """ Makes the steps to prepare new branch for the release PR, like generating changelog and updating version :param new_pr: object of class new_pr with info about the new release :param gitchangelog: bool, use gitchangelog :return: True on success, False on fail """ repo = new_pr.repo version = new_pr.version branch = f"{version}-release" if self.branch_exists(branch): self.logger.warning( f"Branch {branch} already exists, aborting creating PR." ) return False if self.conf.dry_run: msg = ( f"I would make a new PR for release of version " f"{version} based on the issue." ) self.logger.info(msg) return False try: name, email = self.get_user_contact() repo.set_credentials(name, email) repo.set_credential_store() # The bot first checks out the default branch and from it # it creates the new branch, checks out to it and then perform the release # This makes sure that the new release_pr branch has all the commits # from the default branch for the latest release. repo.checkout(self.project.default_branch) changelog = repo.get_log_since_last_release( new_pr.previous_version, gitchangelog ) repo.checkout_new_branch(branch) changed = look_for_version_files(repo.repo_path, new_pr.version) if insert_in_changelog( f"{repo.repo_path}/CHANGELOG.md", new_pr.version, changelog ): repo.add(["CHANGELOG.md"]) if changed: repo.add(changed) repo.commit(f"{version} release", allow_empty=True) repo.push(branch) if not self.pr_exists(f"{version} release"): new_pr.pr_url = self.make_pr( branch=branch, version=f"{version}", log=changelog, changed_version_files=changed, labels=new_pr.labels, ) return True except GitException as exc: raise ReleaseException(exc) finally: repo.checkout(self.project.default_branch) return False
def fedpkg_spectool(directory, branch, fail=True): if not os.path.isdir(directory): raise ReleaseException("Cannot access fedpkg repository:") spec_files = glob(os.path.join(directory, "*spec")) spec_files = " ".join(spec_files) return shell_command(directory, f"spectool -g {spec_files}", f"Retrieving new sources for branch {branch} failed:", fail)
def fake_clone_func(self, directory, name): directory = Path(directory) if not directory.is_dir(): raise ReleaseException( f"Cannot clone into non-existent directory {directory}:") shell_command(directory, f"fedpkg clone {name!r} --anonymous", "Cloning fedora repository failed:") return str(directory / name)
def latest_version(self): """Get latest version of the package from PyPi""" response = requests.get( url=f"{self.PYPI_URL}{self.conf.repository_name}/json") if response.status_code == 200: return response.json()['info']['version'] else: msg = f"Pypi package {self.conf.repository_name!r} doesn't exist:\n{response.text}" raise ReleaseException(msg)
def fedpkg_clone_repository(directory, name): if not os.path.isdir(directory): raise ReleaseException("Cannot clone fedpkg repository into non-existent directory:") if shell_command(directory, f"fedpkg clone {name!r}", "Cloning fedora repository failed:"): return os.path.join(directory, name) else: return ''
def build_wheel(project_root, python_version): """ Builds wheel for specified version of python :param project_root: location of setup.py :param python_version: python version to build wheel for """ interpreter = "python2" if python_version == 3: interpreter = "python3" elif python_version != 2: # no other versions of python other than 2 and three are supported raise ReleaseException(f"Unsupported python version: {python_version}") if not os.path.isfile(os.path.join(project_root, 'setup.py')): raise ReleaseException("Cannot find setup.py:") run_command(project_root, f"{interpreter} setup.py bdist_wheel", f"Cannot build wheel for python {python_version}")
def build_sdist(project_root): """ Builds source distribution out of setup.py :param project_root: location of setup.py """ if os.path.isfile(os.path.join(project_root, 'setup.py')): run_command(project_root, "python setup.py sdist", "Cannot build sdist:") else: raise ReleaseException("Cannot find setup.py:")
def make_pr( self, branch, version, log, changed_version_files, base: str = None, labels=None ): """ Makes a pull request with info on the new release :param branch: name of the branch to make PR from :param version: version that is being released :param log: changelog :param changed_version_files: list of files that have been changed in order to update version :param base: base of the PR. defaults to project's default branch :param labels: list of str, labels to be put on PR :return: url of the PR """ message = ( f"Hi,\n you have requested a release PR from me. Here it is!\n" f"This is the changelog I created:\n" f"### Changes\n{log}\n\nYou can change it by editing `CHANGELOG.md` " f"in the root of this repository and pushing to `{branch}` branch" f" before merging this PR.\n" ) if len(changed_version_files) == 1: message += "I have also updated the `__version__ ` in file:\n" elif len(changed_version_files) > 1: message += ( "There were multiple files where `__version__ ` was set, " "so I left updating them up to you. These are the files:\n" ) elif not changed_version_files: message += "I didn't find any files where `__version__` is set." for file in changed_version_files: message += f"* {file}\n" try: if base is None: base = self.project.default_branch new_pr = self.project.create_pr( title=f"{version} release", body=message, target_branch=base, source_branch=branch, ) self.logger.info(f"Created PR: {new_pr}") if labels and which_service(self.project) == GitService.Github: # ogr-lib implements labeling only for Github labels self.project.add_pr_labels(new_pr.id, labels=labels) return new_pr.url except Exception: msg = ( f"Something went wrong with creating " f"PR on {which_service(self.project).name}" ) raise ReleaseException(msg)
def build_wheel(project_root): """ Builds wheel for specified version of python :param project_root: location of setup.py """ if not os.path.isfile(os.path.join(project_root, 'setup.py')): raise ReleaseException("Cannot find setup.py:") run_command(project_root, "python3 setup.py bdist_wheel", "Cannot build wheel:")
def latest_version(self): """Get latest version of the package from PyPi or 0.0.0""" response = requests.get( url=f"{self.PYPI_URL}{self.conf.pypi_project}/json") if response.status_code == 200: return response.json()['info']['version'] elif response.status_code == 404: return '0.0.0' else: msg = f"Error getting latest version from PyPi:\n{response.text}" raise ReleaseException(msg)
def fedpkg_build(self, directory, branch, scratch=False, fail=True): if not os.path.isdir(directory): raise ReleaseException("Cannot access fedpkg repository:") self.logger.debug(f"Building branch {branch!r} in Fedora. It can take a long time.") success = shell_command(directory, f"fedpkg build {'--scratch' if scratch else ''}", f"Building branch {branch!r} in Fedora failed:", fail) if success: self.builds.append(f"{branch}") return success
def release(self): """ Release project on PyPi """ project_root = self.git.repo_path if os.path.isdir(project_root): self.logger.debug("About to release on PyPi") self.build_sdist(project_root) self.build_wheel(project_root) self.upload(project_root) else: raise ReleaseException( "Cannot find project root for PyPi release:")
def make_release_pull_request(self): """ Makes release pull request and handles outcome :return: whether making PR was successful """ def pr_handler(success): """ Handler for the outcome of making a PR :param success: whether making PR was successful :return: """ result = "made" if success else "failed to make" msg = f"I just {result} a PR request for a release version {self.new_pr.version}" level = logging.INFO if success else logging.ERROR self.logger.log(level, msg) if success: msg += f"\n Here's a [link to the PR]({self.new_pr.pr_url})" comment_backup = self.github.comment.copy() self.github.comment = [msg] self.project.get_issue(self.new_pr.issue_number).comment(msg) self.github.comment = comment_backup if success: self.project.get_issue(self.new_pr.issue_number).close() self.logger.debug(f"Closed issue #{self.new_pr.issue_number}") latest_gh_str = self.github.latest_release() self.new_pr.previous_version = latest_gh_str if Version.coerce(latest_gh_str) >= Version.coerce(self.new_pr.version): msg = f"Version ({latest_gh_str}) is already released and this issue is ignored." self.logger.warning(msg) return False msg = ( f"Making a new PR for release of version " f"{self.new_pr.version} based on the issue." ) if not self.conf.dry_run: self.logger.info(msg) try: self.new_pr.repo = self.git if not self.new_pr.repo: raise ReleaseException("Couldn't clone repository!") if self.github.make_release_pr(self.new_pr, self.conf.gitchangelog): pr_handler(success=True) return True except ReleaseException: pr_handler(success=False) raise return False
def make_release_pull_request(self): """ Makes release pull request and handles outcome :return: whether making PR was successful """ def pr_handler(success): """ Handler for the outcome of making a PR :param success: whether making PR was successful :return: """ result = 'made' if success else 'failed to make' msg = f"I just {result} a PR request for a release version {self.new_pr['version']}" level = logging.INFO if success else logging.ERROR self.logger.log(level, msg) if success: msg += f"\n Here's a [link to the PR]({self.new_pr['pr_url']})" comment_backup = self.github.comment.copy() self.github.comment = [msg] self.github.add_comment(self.new_pr['issue_id']) self.github.comment = comment_backup if success: self.github.close_issue(self.new_pr['issue_number']) self.new_pr['repo'].cleanup() prev_version = self.github.latest_release() # if there are no previous releases, set version to 0.0.0 prev_version = prev_version if prev_version else '0.0.0' self.new_pr['previous_version'] = prev_version if Version.coerce(prev_version) >= Version.coerce( self.new_pr['version']): msg = f"Version ({prev_version}) is already released and this issue is ignored." self.logger.warning(msg) return False msg = f"Making a new PR for release of version {self.new_pr['version']} based on an issue." self.logger.info(msg) try: self.new_pr['repo'] = self.github.clone_repository() if not self.new_pr['repo']: raise ReleaseException("Couldn't clone repository!") if self.github.make_release_pr(self.new_pr): pr_handler(success=True) return True except ReleaseException: pr_handler(success=False) raise return False
def release(self, conf_array): """ Release project on PyPi :param conf_array: structure with information about the new release """ project_root = conf_array['fs_path'] if os.path.isdir(project_root): self.logger.debug("About to release on PyPi") self.build_sdist(project_root) for version in conf_array['python_versions']: self.build_wheel(project_root, version) self.upload(project_root) else: raise ReleaseException("Cannot find project root for PyPi release:")
def get_configuration(args): """ Get the bot's configuration using the args """ if args.configuration: args.configuration = Path(args.configuration).resolve() if not args.configuration.is_file(): raise ReleaseException( f"Supplied configuration file is not found: {args.configuration}") if args.debug: configuration.logger.setLevel(logging.DEBUG) for key, value in vars(args).items(): setattr(configuration, key, value) configuration.dry_run = args.dry_run
def upload(self, project_root): """ Uploads the package distribution to PyPi :param project_root: directory with dist/ folder """ if os.path.isdir(os.path.join(project_root, 'dist')): spec_files = glob(os.path.join(project_root, "dist/*")) files = "" for file in spec_files: files += f"{file} " self.logger.debug(f"Uploading {files} to PyPi") shell_command(project_root, f"twine upload {files}", "Cannot upload python distribution:") else: raise ReleaseException("dist/ folder cannot be found:")
def branch_exists(self, branch): """ Makes a call to github api to check if branch already exists :param branch: name of the branch :return: True if exists, False if not """ url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/" f"{self.conf.repository_name}/branches/{branch}") response = self.do_request(method="GET", url=url) if response.status_code == 200: return True elif response.status_code == 404: self.logger.debug(response.text) return False else: msg = f"Unexpected response code from Github:\n{response.text}" raise ReleaseException(msg)