def fedpkg_push(directory, branch, fail=True):
if not os.path.isdir(directory):
raise ReleaseException("Cannot access fedpkg repository:")
return shell_command(directory, "fedpkg push",
f"Pushing branch {branch!r} to Fedora failed:",
fail)
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest Github release (zip).\n{exc}")
if Version.coerce(latest_release) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on Github"
)
else:
try:
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
self.github.update_changelog(self.new_release['version'])
return self.new_release
def fedpkg_merge(directory, branch, ff_only=True, fail=True):
if not os.path.isdir(directory):
raise ReleaseException("Cannot access fedpkg repository:")
return shell_command(
directory, f"git merge master {'--ff-only' if ff_only else ''}",
f"Merging master to branch {branch!r} failed:", fail)
def make_new_release(self, new_release):
"""
Makes new release to Github.
This has to be done using github api v3 because v4 (GraphQL) doesn't support this yet
:param new_release: version number of the new release
:return: tuple (released, new_release) - released is bool, new_release contains info about
the new release
"""
payload = {
"tag_name": new_release['version'],
"target_commitish": new_release['commitish'],
"name": new_release['version'],
"prerelease": False,
"draft": False
}
url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/"
f"{self.conf.repository_name}/releases")
self.logger.debug(
f"About to release {new_release['version']} on Github")
response = requests.post(url=url, headers=self.headers, json=payload)
if response.status_code != 201:
msg = f"Failed to create new release on github:\n{response.text}"
raise ReleaseException(msg)
released = True
new_release = self.download_extract_zip(new_release)
self.update_changelog(self.latest_release(), new_release['version'],
new_release['fs_path'],
response.json()['id'])
return released, new_release
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release.version} on {self.git_service.name}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest {self.git_service.name} release (zip).\n{exc}"
)
if Version.coerce(latest_release) >= Version.coerce(
self.new_release.version):
self.logger.info(
f"{self.new_release.version} has already been released on {self.git_service.name}"
)
else:
try:
if self.conf.dry_run:
return None
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_pr(self,
branch,
version,
log,
changed_version_files,
base='master',
labels=None):
"""
Makes a pull request with info on the new release
:param branch: name of the branch to make PR from
:param version: version that is being released
:param log: changelog
:param changed_version_files: list of files that have been changed
in order to update version
:param base: base of the PR. 'master' by default
:param labels: list of str, labels to be put on PR
:return: url of the PR
"""
message = (
f'Hi,\n you have requested a release PR from me. Here it is!\n'
f'This is the changelog I created:\n'
f'### Changes\n{log}\n\nYou can change it by editing `CHANGELOG.md` '
f'in the root of this repository and pushing to `{branch}` branch'
f' before merging this PR.\n')
if len(changed_version_files) == 1:
message += 'I have also updated the `__version__ ` in file:\n'
elif len(changed_version_files) > 1:
message += (
'There were multiple files where `__version__ ` was set, '
'so I left updating them up to you. These are the files:\n')
elif not changed_version_files:
message += "I didn't find any files where `__version__` is set."
for file in changed_version_files:
message += f'* {file}\n'
payload = {
'title': f'{version} release',
'head': branch,
'base': base,
'body': message,
'maintainer_can_modify': True
}
url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/"
f"{self.conf.repository_name}/pulls")
self.logger.debug(f'Attempting a PR for {branch} branch')
response = requests.post(url=url, headers=self.headers, json=payload)
if response.status_code == 201:
parsed = response.json()
self.logger.info(f"Created PR: {parsed['html_url']}")
# put labels on PR
if labels is not None:
self.put_labels_on_issue(parsed['number'], labels)
return parsed['html_url']
else:
msg = (f"Something went wrong with creating "
f"PR on github:\n{response.text}")
raise ReleaseException(msg)
def shell_command(work_directory, cmd, error_message, fail=True):
"""
Execute a shell command
:param work_directory: A directory to execute the command in
:param cmd: The shell command
:param error_message: An error message to return in case of failure
:param fail: If failure should cause termination of the bot
:return: Boolean indicating success/failure
"""
cmd = shlex.split(cmd)
shell = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
cwd=work_directory,
universal_newlines=True)
configuration.logger.debug(f"{shell.args}\n{shell.stdout}")
if shell.returncode != 0:
configuration.logger.error(f"{error_message}\n{shell.stderr}")
if fail:
raise ReleaseException(
f"{shell.args!r} failed with {error_message!r}")
return False
return True
def fedpkg_new_sources(directory, branch, sources="", fail=True):
if not os.path.isdir(directory):
raise ReleaseException("Cannot access fedpkg repository:")
return shell_command(directory, f"fedpkg new-sources {sources}",
f"Adding new sources on branch {branch} failed:",
fail)
def update_spec(spec_path, new_release):
"""
Update spec with new version and changelog for that version, change release to 1
:param spec_path: Path to package .spec file
:param new_release: an array containing info about new release, see main() for definition
"""
if not os.path.isfile(spec_path):
raise ReleaseException("No spec file found in dist-git repository!")
# make changelog and get version
locale.setlocale(locale.LC_TIME, "en_US.UTF-8")
changelog = (f"* {datetime.datetime.now():%a %b %d %Y} {new_release['author_name']!s} "
f"<{new_release['author_email']!s}> {new_release['version']}-1\n")
# add entries
if new_release.get('changelog'):
for item in new_release['changelog']:
changelog += f"- {item}\n"
else:
changelog += f"- {new_release['version']} release\n"
# change the version and add changelog in spec file
with open(spec_path, 'r+') as spec_file:
spec = spec_file.read()
# replace version
spec = re.sub(r'(Version:\s*)([0-9]|[.])*', r'\g<1>' + new_release['version'], spec)
# make release 1
spec = re.sub(r'(Release:\s*)([0-9]*)(.*)', r'\g<1>1\g<3>', spec)
# insert changelog
spec = re.sub(r'(%changelog\n)', r'\g<1>' + changelog + '\n', spec)
# write and close
spec_file.seek(0)
spec_file.write(spec)
spec_file.truncate()
spec_file.close()
def make_new_release(self, new_release):
"""
Makes new release to Github.
This has to be done using github api v3 because v4 (GraphQL) doesn't support this yet
:param new_release: version number of the new release
:return: tuple (released, new_release) - released is bool, new_release contains info about
the new release
"""
payload = {
"tag_name": new_release.version,
"target_commitish": new_release.commitish,
"name": new_release.version,
"prerelease": False,
"draft": False
}
url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/"
f"{self.conf.repository_name}/releases")
self.logger.debug(f"About to release {new_release.version} on Github")
response = self.do_request(method="POST",
url=url,
json_payload=payload,
use_github_auth=True)
if response.status_code != 201:
msg = f"Failed to create new release on github:\n{response.text}"
raise ReleaseException(msg)
return True, new_release
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_github = self.github.latest_release()
if Version.coerce(latest_github) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on Github"
)
# to fill in new_release['fs_path'] so that we can continue with PyPi upload
self.new_release = self.github.download_extract_zip(
self.new_release)
return self.new_release
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest Github release (zip).\n{exc}")
try:
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def fedpkg_sources(directory, branch, fail=True):
if not os.path.isdir(directory):
raise ReleaseException("Cannot access fedpkg repository:")
return shell_command(
directory, "fedpkg sources",
f"Retrieving sources for branch {branch} failed:", fail)
def make_release_pr(self, new_pr, gitchangelog):
"""
Makes the steps to prepare new branch for the release PR,
like generating changelog and updating version
:param new_pr: object of class new_pr with info about the new release
:param gitchangelog: bool, use gitchangelog
:return: True on success, False on fail
"""
repo = new_pr.repo
version = new_pr.version
branch = f"{version}-release"
if self.branch_exists(branch):
self.logger.warning(
f"Branch {branch} already exists, aborting creating PR."
)
return False
if self.conf.dry_run:
msg = (
f"I would make a new PR for release of version "
f"{version} based on the issue."
)
self.logger.info(msg)
return False
try:
name, email = self.get_user_contact()
repo.set_credentials(name, email)
repo.set_credential_store()
# The bot first checks out the default branch and from it
# it creates the new branch, checks out to it and then perform the release
# This makes sure that the new release_pr branch has all the commits
# from the default branch for the latest release.
repo.checkout(self.project.default_branch)
changelog = repo.get_log_since_last_release(
new_pr.previous_version, gitchangelog
)
repo.checkout_new_branch(branch)
changed = look_for_version_files(repo.repo_path, new_pr.version)
if insert_in_changelog(
f"{repo.repo_path}/CHANGELOG.md", new_pr.version, changelog
):
repo.add(["CHANGELOG.md"])
if changed:
repo.add(changed)
repo.commit(f"{version} release", allow_empty=True)
repo.push(branch)
if not self.pr_exists(f"{version} release"):
new_pr.pr_url = self.make_pr(
branch=branch,
version=f"{version}",
log=changelog,
changed_version_files=changed,
labels=new_pr.labels,
)
return True
except GitException as exc:
raise ReleaseException(exc)
finally:
repo.checkout(self.project.default_branch)
return False
def fedpkg_spectool(directory, branch, fail=True):
if not os.path.isdir(directory):
raise ReleaseException("Cannot access fedpkg repository:")
spec_files = glob(os.path.join(directory, "*spec"))
spec_files = " ".join(spec_files)
return shell_command(directory, f"spectool -g {spec_files}",
f"Retrieving new sources for branch {branch} failed:", fail)
def fake_clone_func(self, directory, name):
directory = Path(directory)
if not directory.is_dir():
raise ReleaseException(
f"Cannot clone into non-existent directory {directory}:")
shell_command(directory, f"fedpkg clone {name!r} --anonymous",
"Cloning fedora repository failed:")
return str(directory / name)
def latest_version(self):
"""Get latest version of the package from PyPi"""
response = requests.get(
url=f"{self.PYPI_URL}{self.conf.repository_name}/json")
if response.status_code == 200:
return response.json()['info']['version']
else:
msg = f"Pypi package {self.conf.repository_name!r} doesn't exist:\n{response.text}"
raise ReleaseException(msg)
def fedpkg_clone_repository(directory, name):
if not os.path.isdir(directory):
raise ReleaseException("Cannot clone fedpkg repository into non-existent directory:")
if shell_command(directory, f"fedpkg clone {name!r}",
"Cloning fedora repository failed:"):
return os.path.join(directory, name)
else:
return ''
def build_wheel(project_root, python_version):
"""
Builds wheel for specified version of python
:param project_root: location of setup.py
:param python_version: python version to build wheel for
"""
interpreter = "python2"
if python_version == 3:
interpreter = "python3"
elif python_version != 2:
# no other versions of python other than 2 and three are supported
raise ReleaseException(f"Unsupported python version: {python_version}")
if not os.path.isfile(os.path.join(project_root, 'setup.py')):
raise ReleaseException("Cannot find setup.py:")
run_command(project_root, f"{interpreter} setup.py bdist_wheel",
f"Cannot build wheel for python {python_version}")
def build_sdist(project_root):
"""
Builds source distribution out of setup.py
:param project_root: location of setup.py
"""
if os.path.isfile(os.path.join(project_root, 'setup.py')):
run_command(project_root, "python setup.py sdist", "Cannot build sdist:")
else:
raise ReleaseException("Cannot find setup.py:")
def make_pr(
self, branch, version, log, changed_version_files, base: str = None, labels=None
):
"""
Makes a pull request with info on the new release
:param branch: name of the branch to make PR from
:param version: version that is being released
:param log: changelog
:param changed_version_files: list of files that have been changed
in order to update version
:param base: base of the PR. defaults to project's default branch
:param labels: list of str, labels to be put on PR
:return: url of the PR
"""
message = (
f"Hi,\n you have requested a release PR from me. Here it is!\n"
f"This is the changelog I created:\n"
f"### Changes\n{log}\n\nYou can change it by editing `CHANGELOG.md` "
f"in the root of this repository and pushing to `{branch}` branch"
f" before merging this PR.\n"
)
if len(changed_version_files) == 1:
message += "I have also updated the `__version__ ` in file:\n"
elif len(changed_version_files) > 1:
message += (
"There were multiple files where `__version__ ` was set, "
"so I left updating them up to you. These are the files:\n"
)
elif not changed_version_files:
message += "I didn't find any files where `__version__` is set."
for file in changed_version_files:
message += f"* {file}\n"
try:
if base is None:
base = self.project.default_branch
new_pr = self.project.create_pr(
title=f"{version} release",
body=message,
target_branch=base,
source_branch=branch,
)
self.logger.info(f"Created PR: {new_pr}")
if labels and which_service(self.project) == GitService.Github:
# ogr-lib implements labeling only for Github labels
self.project.add_pr_labels(new_pr.id, labels=labels)
return new_pr.url
except Exception:
msg = (
f"Something went wrong with creating "
f"PR on {which_service(self.project).name}"
)
raise ReleaseException(msg)
def build_wheel(project_root):
"""
Builds wheel for specified version of python
:param project_root: location of setup.py
"""
if not os.path.isfile(os.path.join(project_root, 'setup.py')):
raise ReleaseException("Cannot find setup.py:")
run_command(project_root, "python3 setup.py bdist_wheel",
"Cannot build wheel:")
def latest_version(self):
"""Get latest version of the package from PyPi or 0.0.0"""
response = requests.get(
url=f"{self.PYPI_URL}{self.conf.pypi_project}/json")
if response.status_code == 200:
return response.json()['info']['version']
elif response.status_code == 404:
return '0.0.0'
else:
msg = f"Error getting latest version from PyPi:\n{response.text}"
raise ReleaseException(msg)
def fedpkg_build(self, directory, branch, scratch=False, fail=True):
if not os.path.isdir(directory):
raise ReleaseException("Cannot access fedpkg repository:")
self.logger.debug(f"Building branch {branch!r} in Fedora. It can take a long time.")
success = shell_command(directory,
f"fedpkg build {'--scratch' if scratch else ''}",
f"Building branch {branch!r} in Fedora failed:", fail)
if success:
self.builds.append(f"{branch}")
return success
def release(self):
"""
Release project on PyPi
"""
project_root = self.git.repo_path
if os.path.isdir(project_root):
self.logger.debug("About to release on PyPi")
self.build_sdist(project_root)
self.build_wheel(project_root)
self.upload(project_root)
else:
raise ReleaseException(
"Cannot find project root for PyPi release:")
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = "made" if success else "failed to make"
msg = f"I just {result} a PR request for a release version {self.new_pr.version}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr.pr_url})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.project.get_issue(self.new_pr.issue_number).comment(msg)
self.github.comment = comment_backup
if success:
self.project.get_issue(self.new_pr.issue_number).close()
self.logger.debug(f"Closed issue #{self.new_pr.issue_number}")
latest_gh_str = self.github.latest_release()
self.new_pr.previous_version = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(self.new_pr.version):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = (
f"Making a new PR for release of version "
f"{self.new_pr.version} based on the issue."
)
if not self.conf.dry_run:
self.logger.info(msg)
try:
self.new_pr.repo = self.git
if not self.new_pr.repo:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr, self.conf.gitchangelog):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr['version']}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr['pr_url']})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr['issue_id'])
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr['issue_number'])
self.new_pr['repo'].cleanup()
prev_version = self.github.latest_release()
# if there are no previous releases, set version to 0.0.0
prev_version = prev_version if prev_version else '0.0.0'
self.new_pr['previous_version'] = prev_version
if Version.coerce(prev_version) >= Version.coerce(
self.new_pr['version']):
msg = f"Version ({prev_version}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = f"Making a new PR for release of version {self.new_pr['version']} based on an issue."
self.logger.info(msg)
try:
self.new_pr['repo'] = self.github.clone_repository()
if not self.new_pr['repo']:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def release(self, conf_array):
"""
Release project on PyPi
:param conf_array: structure with information about the new release
"""
project_root = conf_array['fs_path']
if os.path.isdir(project_root):
self.logger.debug("About to release on PyPi")
self.build_sdist(project_root)
for version in conf_array['python_versions']:
self.build_wheel(project_root, version)
self.upload(project_root)
else:
raise ReleaseException("Cannot find project root for PyPi release:")
def get_configuration(args):
"""
Get the bot's configuration using the args
"""
if args.configuration:
args.configuration = Path(args.configuration).resolve()
if not args.configuration.is_file():
raise ReleaseException(
f"Supplied configuration file is not found: {args.configuration}")
if args.debug:
configuration.logger.setLevel(logging.DEBUG)
for key, value in vars(args).items():
setattr(configuration, key, value)
configuration.dry_run = args.dry_run
def upload(self, project_root):
"""
Uploads the package distribution to PyPi
:param project_root: directory with dist/ folder
"""
if os.path.isdir(os.path.join(project_root, 'dist')):
spec_files = glob(os.path.join(project_root, "dist/*"))
files = ""
for file in spec_files:
files += f"{file} "
self.logger.debug(f"Uploading {files} to PyPi")
shell_command(project_root, f"twine upload {files}",
"Cannot upload python distribution:")
else:
raise ReleaseException("dist/ folder cannot be found:")
def branch_exists(self, branch):
"""
Makes a call to github api to check if branch already exists
:param branch: name of the branch
:return: True if exists, False if not
"""
url = (f"{self.API3_ENDPOINT}repos/{self.conf.repository_owner}/"
f"{self.conf.repository_name}/branches/{branch}")
response = self.do_request(method="GET", url=url)
if response.status_code == 200:
return True
elif response.status_code == 404:
self.logger.debug(response.text)
return False
else:
msg = f"Unexpected response code from Github:\n{response.text}"
raise ReleaseException(msg)
