def __init__(self, configuration):
self.conf = configuration
self.github = Github(configuration)
self.pypi = PyPi(configuration)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
self.new_release = {}
def setup_method(self, method):
""" setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration.set_logging(level=10)
configuration.debug = True
self.pypi = PyPi(configuration)
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
self.project = configuration.project
self.git_service = which_service(self.project) # Github/Pagure
def __init__(self, configuration):
self.conf = configuration
url = f'https://github.com/{self.conf.repository_owner}/{self.conf.repository_name}.git'
self.git = Git(url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
# FIXME: it's cumbersome to work with these dicts - it's unclear how the content changes;
# get rid of them and replace them with individual variables
self.new_release = {}
self.new_pr = {}
def pypi(self, tmpdir):
conf = Configuration()
path = str(tmpdir)
src = Path(__file__).parent / "src/rlsbot-test"
shutil.copy2(str(src / "setup.py"), path)
shutil.copy2(str(src / "rlsbot_test.py"), path)
self.run_cmd("git init .", work_directory=str(tmpdir))
set_git_credentials(str(tmpdir), "Release Bot", "*****@*****.**")
self.run_cmd("git add .", work_directory=str(tmpdir))
self.run_cmd("git commit -m 'initial commit'",
work_directory=str(tmpdir))
git_repo = Git(str(tmpdir), conf)
pypi = PyPi(configuration, git_repo)
(flexmock(pypi).should_receive("upload").replace_with(lambda x: None))
return pypi
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
url = f'https://github.com/{self.conf.repository_owner}/{self.conf.repository_name}.git'
self.git = Git(url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
# FIXME: it's cumbersome to work with these dicts - it's unclear how the content changes;
# get rid of them and replace them with individual variables
self.new_release = {}
self.new_pr = {}
def cleanup(self):
if 'tempdir' in self.new_release:
self.new_release['tempdir'].cleanup()
self.new_release = {}
self.new_pr = {}
self.github.comment = []
self.fedora.progress_log = []
self.git.cleanup()
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_configuration()
release_conf = self.conf.load_release_conf(conf)
self.new_release.update(release_conf)
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
cursor = ''
release_issues = {}
while True:
edges = self.github.walk_through_open_issues(start=cursor,
direction='before')
if not edges:
self.logger.debug(f'No more open issues found')
break
else:
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match:
version = match[1].strip()
if validate(version):
if edge['node']['authorAssociation'] in [
'MEMBER', 'OWNER', 'COLLABORATOR'
]:
release_issues[version] = edge['node']
self.logger.info(
f'Found new release issue with version: {version}'
)
else:
self.logger.warning(
f"Author association {edge['node']['authorAssociation']!r} "
f"not in ['MEMBER', 'OWNER', 'COLLABORATOR']"
)
else:
self.logger.warning(
f"{version!r} is not a valid version")
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
for version, node in release_issues.items():
self.new_pr = {
'version': version,
'issue_id': node['id'],
'issue_number': node['number'],
'labels': self.new_release.get('labels')
}
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
while True:
edges = self.github.walk_through_prs(start=cursor,
direction='before',
closed=True)
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match and validate(match[1]):
merge_commit = edge['node']['mergeCommit']
self.logger.info(
f"Found merged release PR with version {match[1]}, "
f"commit id: {merge_commit['oid']}")
new_release = {
'version': match[1],
'commitish': merge_commit['oid'],
'pr_id': edge['node']['id'],
'author_name': merge_commit['author']['name'],
'author_email': merge_commit['author']['email']
}
self.new_release.update(new_release)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr['version']}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr['pr_url']})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr['issue_id'])
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr['issue_number'])
latest_gh_str = self.github.latest_release()
self.new_pr['previous_version'] = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(
self.new_pr['version']):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = f"Making a new PR for release of version {self.new_pr['version']} based on an issue."
self.logger.info(msg)
try:
self.new_pr['repo'] = self.git
if not self.new_pr['repo']:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest Github release (zip).\n{exc}")
if Version.coerce(latest_release) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on Github"
)
else:
try:
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
self.github.update_changelog(self.new_release['version'])
return self.new_release
def make_new_pypi_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on PyPi"
)
return False
self.git.fetch_tags()
self.git.checkout(self.new_release['version'])
try:
self.pypi.release()
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return True
def make_new_fedora_release(self):
if not self.new_release.get('fedora'):
self.logger.debug('Skipping Fedora release')
return
self.logger.info("Triggering Fedora release")
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} on Fedora"
builds = ', '.join(self.fedora.builds)
bodhi_update_url = "https://bodhi.fedoraproject.org/updates/new"
if builds:
msg += f", successfully built for branches: {builds}."
msg += f" Follow this link to create bodhi update(s): {bodhi_update_url}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
name, email = self.github.get_user_contact()
self.new_release['commit_name'] = name
self.new_release['commit_email'] = email
success_ = self.fedora.release(self.new_release)
release_handler(success_)
except ReleaseException:
release_handler(success=False)
raise
def run(self):
self.logger.info(
f"release-bot v{configuration.version} reporting for duty!")
try:
while True:
self.git.pull()
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
if self.make_new_pypi_release():
# There's no way how to tell whether there's already such a fedora 'release'
# so try to do it only when we just did PyPi release
self.make_new_fedora_release()
except ReleaseException as exc:
self.logger.error(exc)
# Moved out of the previous try-except block, because if it
# encounters ReleaseException while checking for PyPi sources
# it doesn't check for GitHub issues.
try:
if self.new_release.get(
'trigger_on_issue'
) and self.find_open_release_issues():
if self.new_release.get('labels') is not None:
self.github.put_labels_on_issue(
self.new_pr['issue_number'],
self.new_release.get('labels'))
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.get('pr_id'))
self.logger.debug(
f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
finally:
self.cleanup()
class TestPypi2:
def setup_method(self, method):
""" setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration.set_logging(level=10)
configuration.debug = True
self.pypi = PyPi(configuration)
def teardown_method(self, method):
""" teardown any state that was previously setup with a setup_method
call.
"""
if re.match(r'test_install_.', method.__name__) or re.match(
r'test_release_.', method.__name__):
self.run_cmd('pip2 uninstall rlsbot-test -y', "/")
self.run_cmd('pip3 uninstall rlsbot-test -y', "/")
def run_cmd(self, cmd, work_directory):
shell = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=work_directory,
universal_newlines=True)
return shell
@pytest.fixture
def minimal_conf_array(self, tmpdir):
return {'python_versions': [2, 3], 'fs_path': tmpdir}
@pytest.fixture
def package_setup(self, tmpdir):
path = Path(str(tmpdir)) / "rlsbot-test"
src = Path(__file__).parent / "src/rlsbot-test"
shutil.copytree(str(src), str(path))
return path
@pytest.fixture()
def no_upload(self):
(flexmock(
self.pypi).should_receive("upload").replace_with(lambda x: None))
@pytest.fixture
def non_existent_path(self, tmpdir):
path = Path(str(tmpdir)) / 'fooo'
return str(path)
def test_missing_setup_sdist(self, non_existent_path):
with pytest.raises(ReleaseException):
self.pypi.build_sdist(non_existent_path)
def test_missing_setup_wheel(self, non_existent_path):
with pytest.raises(ReleaseException):
self.pypi.build_wheel(non_existent_path, 2)
def test_missing_project_wrapper(self, minimal_conf_array,
non_existent_path):
minimal_conf_array['fs_path'] = non_existent_path
with pytest.raises(ReleaseException):
self.pypi.release(minimal_conf_array)
def test_sdist(self, package_setup):
self.pypi.build_sdist(package_setup)
assert (package_setup / 'dist/rlsbot-test-1.0.0.tar.gz').is_file()
def test_wheel_2(self, package_setup):
self.pypi.build_wheel(package_setup, 2)
assert list(package_setup.glob('dist/rlsbot_test-1.0.0-py2*.whl'))
def test_install_2(self, package_setup):
self.pypi.build_sdist(package_setup)
self.pypi.build_wheel(package_setup, 2)
wheel2 = list(package_setup.glob('dist/rlsbot_test-1.0.0-py2*.whl'))
assert self.run_cmd(f'pip2 install --user {wheel2[0]}',
package_setup).returncode == 0
assert self.run_cmd(f'pip2 show rlsbot-test',
package_setup).returncode == 0
assert self.run_cmd(f'$HOME/.local/bin/rlsbot-test',
package_setup).returncode == 0
def test_release_2(self, minimal_conf_array, package_setup, no_upload):
minimal_conf_array['fs_path'] = package_setup
self.pypi.release(minimal_conf_array)
assert (package_setup / 'dist/rlsbot-test-1.0.0.tar.gz').is_file()
wheel2 = list(package_setup.glob('dist/rlsbot_test-1.0.0-py2*.whl'))
assert wheel2
assert self.run_cmd(f'pip2 install --user {wheel2[0]}',
package_setup).returncode == 0
assert self.run_cmd(f'pip2 show rlsbot-test',
package_setup).returncode == 0
assert self.run_cmd(f'$HOME/.local/bin/rlsbot-test',
package_setup).returncode == 0
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
self.project = configuration.project
self.git_service = which_service(self.project) # Github/Pagure
def cleanup(self):
self.new_release = NewRelease()
self.new_pr = NewPR()
self.github.comment = []
self.git.cleanup()
@staticmethod
def create_flask_instance(configuration):
"""Create flask instance for receiving Github webhooks"""
app = Flask(__name__)
app.add_url_rule(
'/webhook-handler/', # route for github callbacks
view_func=GithubWebhooksHandler.as_view('github_webhooks_handler',
conf=configuration),
methods=[
'POST',
])
app.run(host='0.0.0.0', port=8080)
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_file("release-conf.yaml")
release_conf = self.conf.load_release_conf(conf)
setup_cfg = self.github.get_file("setup.cfg")
self.conf.set_pypi_project(release_conf, setup_cfg)
self.new_release.update(
changelog=release_conf.get('changelog'),
author_name=release_conf.get('author_name'),
author_email=release_conf.get('author_email'),
pypi=release_conf.get('pypi'),
trigger_on_issue=release_conf.get('trigger_on_issue'),
labels=release_conf.get('labels'))
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
release_issues = {}
latest_version = Version(self.github.latest_release())
opened_issues = self.project.get_issue_list(IssueStatus.open)
if not opened_issues:
self.logger.debug(f'No more open issues found')
else:
for issue in opened_issues:
match, version = process_version_from_title(
issue.title, latest_version)
if match:
if self.project.can_close_issue(which_username(self.conf),
issue):
release_issues[version] = issue
self.logger.info(
f'Found new release issue with version: {version}')
else:
self.logger.warning(
f"User {which_username(self.conf)} "
f"has no permission to modify issue")
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
if which_service(self.project) == GitService.Github:
labels = self.new_release.labels
elif which_service(self.project) == GitService.Pagure:
# Putting labels on Pagure issues is not implemented yet inside ogr-lib
labels = None
for version, issue in release_issues.items():
self.new_pr.update_new_pr_details(version=version,
issue_id=None,
issue_number=issue.id,
labels=labels)
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
latest_version = Version(self.github.latest_release())
merged_prs = self.github.walk_through_prs(PRStatus.merged)
if not merged_prs:
self.logger.debug(f'No merged release PR found')
return False
for merged_pr in merged_prs:
match, version = process_version_from_title(
merged_pr.title, latest_version)
if match:
self.logger.info(
f"Found merged release PR with version {version}")
self.new_release.update_pr_details(
version=version,
commitish='master',
pr_id=None,
pr_number=merged_pr.id,
author_email=None,
author_name=merged_pr.author)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr.version}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr.pr_url})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.project.issue_comment(self.new_pr.issue_number, msg)
self.github.comment = comment_backup
if success:
self.project.issue_close(self.new_pr.issue_number)
self.logger.debug(f'Closed issue #{self.new_pr.issue_number}')
latest_gh_str = self.github.latest_release()
self.new_pr.previous_version = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(
self.new_pr.version):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = (f"Making a new PR for release of version "
f"{self.new_pr.version} based on the issue.")
if not self.conf.dry_run:
self.logger.info(msg)
try:
self.new_pr.repo = self.git
if not self.new_pr.repo:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr,
self.conf.gitchangelog):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release.version} on {self.git_service.name}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest {self.git_service.name} release (zip).\n{exc}"
)
if Version.coerce(latest_release) >= Version.coerce(
self.new_release.version):
self.logger.info(
f"{self.new_release.version} has already been released on {self.git_service.name}"
)
else:
try:
if self.conf.dry_run:
return None
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_new_pypi_release(self):
if not self.new_release.pypi:
self.logger.debug('Skipping PyPi release')
return False
def release_handler(success):
result = "released" if success else "failed to release"
if self.conf.dry_run:
msg = f"I would have {result} version {self.new_release.version} on PyPI now."
else:
msg = f"I just {result} version {self.new_release.version} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(
self.new_release.version):
msg = f"{self.conf.pypi_project}-{self.new_release.version} " \
f"or higher version has already been released on PyPi"
self.logger.info(msg)
return False
self.git.fetch_tags()
self.git.checkout(self.new_release.version)
try:
if self.pypi.release() is False:
return False
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
finally:
self.git.checkout('master')
return True
def run(self):
self.logger.info(
f"release-bot v{configuration.version} reporting for duty!")
if self.conf.dry_run:
self.logger.info("Running in dry-run mode.")
try:
while True:
self.git.pull()
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
self.make_new_pypi_release()
except ReleaseException as exc:
self.logger.error(exc)
# Moved out of the previous try-except block, because if it
# encounters ReleaseException while checking for PyPi sources
# it doesn't check for GitHub issues.
try:
if self.new_release.trigger_on_issue and self.find_open_release_issues(
):
if self.new_release.labels is not None:
self.project.add_issue_labels(
self.new_pr.issue_number,
self.new_release.labels)
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
if self.github.comment:
msg = '\n'.join(self.github.comment)
self.project.pr_comment(self.new_release.pr_number, msg)
self.github.comment = [] # clean up
if not self.conf.refresh_interval:
self.logger.debug(
"Refresh interval has not been provided. Reconciliation finished."
)
break
self.logger.debug(
f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
finally:
self.cleanup()
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.github = Github(configuration)
self.pypi = PyPi(configuration)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
self.new_release = {}
self.new_pr = {}
def cleanup(self):
if 'tempdir' in self.new_release:
self.new_release['tempdir'].cleanup()
self.new_release = {}
self.new_pr = {}
self.github.comment = []
self.fedora.progress_log = []
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_configuration()
release_conf = self.conf.load_release_conf(conf)
self.new_release.update(release_conf)
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
cursor = ''
release_issues = {}
while True:
edges = self.github.walk_through_open_issues(start=cursor,
direction='before')
if not edges:
self.logger.debug(f'No more open issues found')
break
else:
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match and validate(match[1]) and \
edge['node']['authorAssociation'] in ['MEMBER', 'OWNER', 'COLLABORATOR']:
release_issues[match[1]] = edge['node']
self.logger.info(
f'Found new release issue with version: {match[1]}'
)
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
for version, node in release_issues.items():
self.new_pr = {
'version': version,
'issue_id': node['id'],
'issue_number': node['number'],
'labels': self.new_release.get('labels')
}
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
while True:
edges = self.github.walk_through_prs(start=cursor,
direction='before',
closed=True)
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match and validate(match[1]):
merge_commit = edge['node']['mergeCommit']
self.logger.info(
f"Found merged release PR with version {match[1]}, "
f"commit id: {merge_commit['oid']}")
new_release = {
'version': match[1],
'commitish': merge_commit['oid'],
'pr_id': edge['node']['id'],
'author_name': merge_commit['author']['name'],
'author_email': merge_commit['author']['email']
}
self.new_release.update(new_release)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr['version']}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr['pr_url']})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr['issue_id'])
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr['issue_number'])
self.new_pr['repo'].cleanup()
prev_version = self.github.latest_release()
# if there are no previous releases, set version to 0.0.0
prev_version = prev_version if prev_version else '0.0.0'
self.new_pr['previous_version'] = prev_version
if Version.coerce(prev_version) >= Version.coerce(
self.new_pr['version']):
msg = f"Version ({prev_version}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = f"Making a new PR for release of version {self.new_pr['version']} based on an issue."
self.logger.info(msg)
try:
self.new_pr['repo'] = self.github.clone_repository()
if not self.new_pr['repo']:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_github = self.github.latest_release()
if Version.coerce(latest_github) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on Github"
)
# to fill in new_release['fs_path'] so that we can continue with PyPi upload
self.new_release = self.github.download_extract_zip(
self.new_release)
return self.new_release
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest Github release (zip).\n{exc}")
try:
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_new_pypi_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
# if there are no previous releases, set version to 0.0.0
latest_pypi = latest_pypi if latest_pypi else '0.0.0'
if Version.coerce(latest_pypi) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on PyPi"
)
return False
try:
self.pypi.release(self.new_release)
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return True
def make_new_fedora_release(self):
if not self.new_release.get('fedora'):
self.logger.debug('Skipping Fedora release')
return
self.logger.info("Triggering Fedora release")
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} on Fedora"
builds = ', '.join(self.fedora.builds)
if builds:
msg += f", successfully built for branches: {builds}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
name, email = self.github.get_user_contact()
self.new_release['commit_name'] = name
self.new_release['commit_email'] = email
success_ = self.fedora.release(self.new_release)
release_handler(success_)
except ReleaseException:
release_handler(success=False)
raise
def run(self):
self.logger.info(
f"release-bot v{configuration.version} reporting for duty!")
while True:
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
if self.make_new_pypi_release():
# There's no way how to tell whether there's already such a fedora 'release'
# so try to do it only when we just did PyPi release
self.make_new_fedora_release()
if self.new_release.get('trigger_on_issue'
) and self.find_open_release_issues():
if self.new_release.get('labels') is not None:
self.github.put_labels_on_issue(
self.new_pr['issue_number'],
self.new_release.get('labels'))
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.get('pr_id'))
self.cleanup()
self.logger.debug(
f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.github = Github(configuration)
self.pypi = PyPi(configuration)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
self.new_release = {}
def cleanup(self):
if 'tempdir' in self.new_release:
self.new_release['tempdir'].cleanup()
self.new_release = {}
self.github.comment = []
self.fedora.progress_log = []
def load_release_conf(self):
# load release configuration from release-conf.yaml in repository
release_conf = self.conf.load_release_conf(self.new_release['fs_path'])
self.new_release.update(release_conf)
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
while True:
edges = self.github.walk_through_closed_prs(start=cursor, direction='before')
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release', edge['node']['title'].lower())
if match and validate(match[1]):
merge_commit = edge['node']['mergeCommit']
self.logger.info(f"Found merged release PR with version {match[1]}, "
f"commit id: {merge_commit['oid']}")
self.new_release = {'version': match[1],
'commitish': merge_commit['oid'],
'pr_id': edge['node']['id'],
'author_name': merge_commit['author']['name'],
'author_email': merge_commit['author']['email']}
return True
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
released, self.new_release = self.github.make_new_release(self.new_release,
self.pypi.latest_version())
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_new_pypi_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(self.new_release['version']):
self.logger.info(f"{self.new_release['version']} has already been released on PyPi")
return False
try:
self.pypi.release(self.new_release)
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return True
def make_new_fedora_release(self):
if not self.new_release.get('fedora'):
self.logger.debug('Skipping Fedora release')
return
self.logger.info("Triggering Fedora release")
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} on Fedora"
builds = ', '.join(self.fedora.builds)
if builds:
msg += f", successfully built for branches: {builds}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
success_ = self.fedora.release(self.new_release)
release_handler(success_)
except ReleaseException:
release_handler(success=False)
raise
def run(self):
self.logger.info(f"release-bot v{configuration.version} reporting for duty!")
while True:
try:
if self.find_newest_release_pull_request():
self.make_new_github_release()
self.load_release_conf()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
if self.make_new_pypi_release():
# There's no way how to tell whether there's already such a fedora 'release'
# so try to do it only when we just did PyPi release
self.make_new_fedora_release()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.get('pr_id'))
self.cleanup()
self.logger.debug(f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
def cleanup(self):
self.new_release = NewRelease()
self.new_pr = NewPR()
self.github.comment = []
self.git.cleanup()
def create_flask_instance(self):
"""Create flask instance for receiving Github webhooks"""
app = Flask(__name__)
app.add_url_rule('/webhook-handler/', # route for github callbacks
view_func=GithubWebhooksHandler.as_view('github_webhooks_handler',
release_bot=self,
conf=configuration),
methods=['POST', ])
app.run(host='0.0.0.0', port=8080)
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_file("release-conf.yaml")
release_conf = self.conf.load_release_conf(conf)
setup_cfg = self.github.get_file("setup.cfg")
self.conf.set_pypi_project(release_conf, setup_cfg)
self.new_release.update(
changelog=release_conf.get('changelog'),
author_name=release_conf.get('author_name'),
author_email=release_conf.get('author_email'),
pypi=release_conf.get('pypi'),
trigger_on_issue=release_conf.get('trigger_on_issue'),
labels=release_conf.get('labels')
)
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
cursor = ''
release_issues = {}
latest_version = Version(self.github.latest_release())
while True:
edges = self.github.walk_through_open_issues(start=cursor, direction='before')
if not edges:
self.logger.debug(f'No more open issues found')
break
else:
for edge in reversed(edges):
cursor = edge['cursor']
title = edge['node']['title'].lower().strip()
match, version = process_version_from_title(title, latest_version)
if match:
if edge['node']['authorAssociation'] in ['MEMBER', 'OWNER',
'COLLABORATOR']:
release_issues[version] = edge['node']
self.logger.info(f'Found new release issue with version: {version}')
else:
self.logger.warning(
f"Author association {edge['node']['authorAssociation']!r} "
f"not in ['MEMBER', 'OWNER', 'COLLABORATOR']")
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
for version, node in release_issues.items():
self.new_pr.update_new_pr_details(
version=version,
issue_id=node['id'],
issue_number=node['number'],
labels=self.new_release.labels
)
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
latest_version = Version(self.github.latest_release())
while True:
edges = self.github.walk_through_prs(start=cursor, direction='before', closed=True)
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
title = edge['node']['title'].lower().strip()
match, version = process_version_from_title(title, latest_version)
if match:
merge_commit = edge['node']['mergeCommit']
self.logger.info(f"Found merged release PR with version {version}, "
f"commit id: {merge_commit['oid']}")
self.new_release.update_pr_details(
version=version,
commitish=merge_commit['oid'],
pr_id=edge['node']['id'],
author_email=merge_commit['author']['email'],
author_name=merge_commit['author']['name']
)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr.version}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr.pr_url})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr.issue_id)
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr.issue_number)
latest_gh_str = self.github.latest_release()
self.new_pr.previous_version = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(self.new_pr.version):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = (f"Making a new PR for release of version "
f"{self.new_pr.version} based on the issue.")
if not self.conf.dry_run:
self.logger.info(msg)
try:
self.new_pr.repo = self.git
if not self.new_pr.repo:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr, self.conf.gitchangelog):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release.version} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(f"Failed getting latest Github release (zip).\n{exc}")
if Version.coerce(latest_release) >= Version.coerce(self.new_release.version):
self.logger.info(
f"{self.new_release.version} has already been released on Github")
else:
try:
if self.conf.dry_run:
return None
released, self.new_release = self.github.make_new_release(self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
self.github.update_changelog(self.new_release.version)
return self.new_release
def make_new_pypi_release(self):
if not self.new_release.pypi:
self.logger.debug('Skipping PyPi release')
return False
def release_handler(success):
result = "released" if success else "failed to release"
if self.conf.dry_run:
msg = f"I would have {result} version {self.new_release.version} on PyPI now."
else:
msg = f"I just {result} version {self.new_release.version} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(self.new_release.version):
msg = f"{self.conf.pypi_project}-{self.new_release.version} " \
f"or higher version has already been released on PyPi"
self.logger.info(msg)
return False
self.git.fetch_tags()
self.git.checkout(self.new_release.version)
try:
if self.pypi.release() == False:
return False
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
finally:
self.git.checkout('master')
return True
def run(self):
self.logger.info(f"release-bot v{configuration.version} reporting for duty!")
if self.conf.dry_run:
self.logger.info("Running in dry-run mode.")
try:
while True:
self.git.pull()
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
self.make_new_pypi_release()
except ReleaseException as exc:
self.logger.error(exc)
# Moved out of the previous try-except block, because if it
# encounters ReleaseException while checking for PyPi sources
# it doesn't check for GitHub issues.
try:
if self.new_release.trigger_on_issue and self.find_open_release_issues():
if self.new_release.labels is not None:
self.github.put_labels_on_issue(self.new_pr.issue_number,
self.new_release.labels)
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.pr_id)
self.logger.debug(f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
finally:
self.cleanup()