def __init__(self, configuration):
self.conf = configuration
self.github = Github(configuration)
self.pypi = PyPi(configuration)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
self.new_release = {}
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
class TestGithub:
"""Tests bot communication with Github"""
def setup_method(self):
"""setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration = prepare_conf()
self.g_utils = GithubUtils()
self.g_utils.create_repo()
self.g_utils.setup_repo()
# set conf
configuration.repository_name = self.g_utils.repo
configuration.github_username = self.g_utils.github_user
configuration.clone_url = (
f"https://github.com/{self.g_utils.github_user}/{self.g_utils.repo}.git"
)
configuration.project = configuration.get_project()
repo_url = f"https://github.com/{self.g_utils.github_user}/{self.g_utils.repo}"
git = Git(repo_url, configuration)
self.github = Github(configuration, git)
def teardown_method(self):
"""teardown any state that was previously setup with a setup_method
call.
"""
if self.g_utils.repo:
try:
self.g_utils.delete_repo()
except Exception as ex:
# no need to fail the test, just warn
warnings.warn(
f"Could not delete repository {self.g_utils.repo}: {ex!r}")
self.g_utils.repo = None
@pytest.fixture()
def open_issue(self):
"""Opens issue in a repository"""
return self.g_utils.open_issue()
def test_get_file(self):
"""Tests fetching release-conf from Github"""
assert self.github.get_file("release-conf.yaml") == RELEASE_CONF
def test_latest_rls_not_existing(self):
"""Tests version number when there is no latest release"""
assert self.github.latest_release() == "0.0.0"
def test_branch_exists_true(self):
"""Tests if branch exists"""
assert self.github.branch_exists("main")
def test_branch_exists_false(self):
"""Tests if branch doesn't exist"""
assert not self.github.branch_exists("not-main")
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
self.project = configuration.project
self.git_service = which_service(self.project) # Github/Pagure
def __init__(self, configuration):
self.conf = configuration
url = f'https://github.com/{self.conf.repository_owner}/{self.conf.repository_name}.git'
self.git = Git(url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
# FIXME: it's cumbersome to work with these dicts - it's unclear how the content changes;
# get rid of them and replace them with individual variables
self.new_release = {}
self.new_pr = {}
def setup_method(self):
""" setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration = prepare_conf()
self.g_utils = GithubUtils()
self.g_utils.create_repo()
self.g_utils.setup_repo()
# set conf
configuration.repository_name = self.g_utils.repo
configuration.github_username = self.g_utils.github_user
configuration.refresh_interval = 1
repo_url = f"https://github.com/{self.g_utils.github_user}/{self.g_utils.repo}"
git = Git(repo_url, configuration)
self.github = Github(configuration, git)
def setup_method(self):
""" setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration.set_logging(level=10)
configuration.debug = True
self.g_utils = GithubUtils(self.github_token, self.github_user)
self.g_utils.create_repo()
self.g_utils.setup_repo()
# set conf
configuration.repository_name = self.g_utils.repo
configuration.repository_owner = self.github_user
configuration.github_token = self.github_token
configuration.github_username = self.github_user
configuration.refresh_interval = 1
self.github = Github(configuration)
def test_latest_release():
r1 = GithubRelease(tag_name='',
url='',
created_at='',
tarball_url='',
git_tag=GitTag(name='0.0.1', commit_sha='123'),
project=flexmock(GitProject),
raw_release=flexmock(title='0.0.1'))
r2 = GithubRelease(tag_name='',
url='',
created_at='',
tarball_url='',
git_tag=GitTag(name='0.0.2', commit_sha='123'),
project=flexmock(GitProject),
raw_release=flexmock(title='0.0.2'))
mocked_releases = [r1, r2]
git = flexmock(Git)
c = flexmock(configuration)
c.project = flexmock(get_releases=lambda: mocked_releases)
github = Github(c, git)
obtained_release = github.latest_release()
assert obtained_release == "0.0.2"
mocked_releases = []
c.project = flexmock(get_releases=lambda: mocked_releases)
github = Github(c, git)
obtained_release = github.latest_release()
assert obtained_release == "0.0.0"
def test_latest_release():
mocked_releases = [
Release(title='0.0.1',
body='',
tag_name='',
url='',
created_at='',
tarball_url='',
git_tag=GitTag(name='0.0.1', commit_sha='123')),
Release(title='0.0.2',
body='',
tag_name='',
url='',
created_at='',
tarball_url='',
git_tag=GitTag(name='0.0.2', commit_sha='123'))
]
git = flexmock(Git)
c = flexmock(configuration)
c.project = flexmock(get_releases=lambda: mocked_releases)
github = Github(c, git)
obtained_release = github.latest_release()
assert obtained_release == '0.0.2'
mocked_releases = []
c.project = flexmock(get_releases=lambda: mocked_releases)
github = Github(c, git)
obtained_release = github.latest_release()
assert obtained_release == '0.0.0'
class TestGithub:
"""Tests bot communication with Github"""
def setup_method(self):
""" setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration = prepare_conf()
self.g_utils = GithubUtils()
self.g_utils.create_repo()
self.g_utils.setup_repo()
# set conf
configuration.repository_name = self.g_utils.repo
configuration.github_username = self.g_utils.github_user
configuration.clone_url = f"https://github.com/{self.g_utils.github_user}/{self.g_utils.repo}.git"
configuration.refresh_interval = 1
configuration.project = configuration.get_project()
repo_url = f"https://github.com/{self.g_utils.github_user}/{self.g_utils.repo}"
git = Git(repo_url, configuration)
self.github = Github(configuration, git)
def teardown_method(self):
""" teardown any state that was previously setup with a setup_method
call.
"""
if self.g_utils.repo:
try:
self.g_utils.delete_repo()
except Exception as ex:
# no need to fail the test, just warn
warnings.warn(f"Could not delete repository {self.g_utils.repo}: {ex!r}")
self.g_utils.repo = None
@pytest.fixture()
def open_issue(self):
"""Opens issue in a repository"""
return self.g_utils.open_issue()
@pytest.fixture()
def open_issue_graphql(self):
"""Opens issue and returns it's GraphQL id"""
number = self.g_utils.open_issue()
query = f"issue(number: {number}) {{id}}"
response = self.github.query_repository(query).json()
self.github.detect_api_errors(response)
return number, response['data']['repository']['issue']['id']
def test_get_file(self):
"""Tests fetching release-conf from Github"""
assert self.github.get_file("release-conf.yaml") == RELEASE_CONF
def test_latest_rls_not_existing(self):
"""Tests version number when there is no latest release"""
assert self.github.latest_release() == '0.0.0'
def test_branch_exists_true(self):
"""Tests if branch exists"""
assert self.github.branch_exists('master')
def test_branch_exists_false(self):
"""Tests if branch doesn't exist"""
assert not self.github.branch_exists('not-master')
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
self.project = configuration.project
self.git_service = which_service(self.project) # Github/Pagure
def cleanup(self):
self.new_release = NewRelease()
self.new_pr = NewPR()
self.github.comment = []
self.git.cleanup()
@staticmethod
def create_flask_instance(configuration):
"""Create flask instance for receiving Github webhooks"""
app = Flask(__name__)
app.add_url_rule(
'/webhook-handler/', # route for github callbacks
view_func=GithubWebhooksHandler.as_view('github_webhooks_handler',
conf=configuration),
methods=[
'POST',
])
app.run(host='0.0.0.0', port=8080)
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_file("release-conf.yaml")
release_conf = self.conf.load_release_conf(conf)
setup_cfg = self.github.get_file("setup.cfg")
self.conf.set_pypi_project(release_conf, setup_cfg)
self.new_release.update(
changelog=release_conf.get('changelog'),
author_name=release_conf.get('author_name'),
author_email=release_conf.get('author_email'),
pypi=release_conf.get('pypi'),
trigger_on_issue=release_conf.get('trigger_on_issue'),
labels=release_conf.get('labels'))
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
release_issues = {}
latest_version = Version(self.github.latest_release())
opened_issues = self.project.get_issue_list(IssueStatus.open)
if not opened_issues:
self.logger.debug(f'No more open issues found')
else:
for issue in opened_issues:
match, version = process_version_from_title(
issue.title, latest_version)
if match:
if self.project.can_close_issue(which_username(self.conf),
issue):
release_issues[version] = issue
self.logger.info(
f'Found new release issue with version: {version}')
else:
self.logger.warning(
f"User {which_username(self.conf)} "
f"has no permission to modify issue")
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
if which_service(self.project) == GitService.Github:
labels = self.new_release.labels
elif which_service(self.project) == GitService.Pagure:
# Putting labels on Pagure issues is not implemented yet inside ogr-lib
labels = None
for version, issue in release_issues.items():
self.new_pr.update_new_pr_details(version=version,
issue_id=None,
issue_number=issue.id,
labels=labels)
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
latest_version = Version(self.github.latest_release())
merged_prs = self.github.walk_through_prs(PRStatus.merged)
if not merged_prs:
self.logger.debug(f'No merged release PR found')
return False
for merged_pr in merged_prs:
match, version = process_version_from_title(
merged_pr.title, latest_version)
if match:
self.logger.info(
f"Found merged release PR with version {version}")
self.new_release.update_pr_details(
version=version,
commitish='master',
pr_id=None,
pr_number=merged_pr.id,
author_email=None,
author_name=merged_pr.author)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr.version}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr.pr_url})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.project.issue_comment(self.new_pr.issue_number, msg)
self.github.comment = comment_backup
if success:
self.project.issue_close(self.new_pr.issue_number)
self.logger.debug(f'Closed issue #{self.new_pr.issue_number}')
latest_gh_str = self.github.latest_release()
self.new_pr.previous_version = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(
self.new_pr.version):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = (f"Making a new PR for release of version "
f"{self.new_pr.version} based on the issue.")
if not self.conf.dry_run:
self.logger.info(msg)
try:
self.new_pr.repo = self.git
if not self.new_pr.repo:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr,
self.conf.gitchangelog):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release.version} on {self.git_service.name}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest {self.git_service.name} release (zip).\n{exc}"
)
if Version.coerce(latest_release) >= Version.coerce(
self.new_release.version):
self.logger.info(
f"{self.new_release.version} has already been released on {self.git_service.name}"
)
else:
try:
if self.conf.dry_run:
return None
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_new_pypi_release(self):
if not self.new_release.pypi:
self.logger.debug('Skipping PyPi release')
return False
def release_handler(success):
result = "released" if success else "failed to release"
if self.conf.dry_run:
msg = f"I would have {result} version {self.new_release.version} on PyPI now."
else:
msg = f"I just {result} version {self.new_release.version} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(
self.new_release.version):
msg = f"{self.conf.pypi_project}-{self.new_release.version} " \
f"or higher version has already been released on PyPi"
self.logger.info(msg)
return False
self.git.fetch_tags()
self.git.checkout(self.new_release.version)
try:
if self.pypi.release() is False:
return False
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
finally:
self.git.checkout('master')
return True
def run(self):
self.logger.info(
f"release-bot v{configuration.version} reporting for duty!")
if self.conf.dry_run:
self.logger.info("Running in dry-run mode.")
try:
while True:
self.git.pull()
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
self.make_new_pypi_release()
except ReleaseException as exc:
self.logger.error(exc)
# Moved out of the previous try-except block, because if it
# encounters ReleaseException while checking for PyPi sources
# it doesn't check for GitHub issues.
try:
if self.new_release.trigger_on_issue and self.find_open_release_issues(
):
if self.new_release.labels is not None:
self.project.add_issue_labels(
self.new_pr.issue_number,
self.new_release.labels)
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
if self.github.comment:
msg = '\n'.join(self.github.comment)
self.project.pr_comment(self.new_release.pr_number, msg)
self.github.comment = [] # clean up
if not self.conf.refresh_interval:
self.logger.debug(
"Refresh interval has not been provided. Reconciliation finished."
)
break
self.logger.debug(
f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
finally:
self.cleanup()
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.github = Github(configuration)
self.pypi = PyPi(configuration)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
self.new_release = {}
self.new_pr = {}
def cleanup(self):
if 'tempdir' in self.new_release:
self.new_release['tempdir'].cleanup()
self.new_release = {}
self.new_pr = {}
self.github.comment = []
self.fedora.progress_log = []
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_configuration()
release_conf = self.conf.load_release_conf(conf)
self.new_release.update(release_conf)
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
cursor = ''
release_issues = {}
while True:
edges = self.github.walk_through_open_issues(start=cursor,
direction='before')
if not edges:
self.logger.debug(f'No more open issues found')
break
else:
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match and validate(match[1]) and \
edge['node']['authorAssociation'] in ['MEMBER', 'OWNER', 'COLLABORATOR']:
release_issues[match[1]] = edge['node']
self.logger.info(
f'Found new release issue with version: {match[1]}'
)
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
for version, node in release_issues.items():
self.new_pr = {
'version': version,
'issue_id': node['id'],
'issue_number': node['number'],
'labels': self.new_release.get('labels')
}
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
while True:
edges = self.github.walk_through_prs(start=cursor,
direction='before',
closed=True)
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match and validate(match[1]):
merge_commit = edge['node']['mergeCommit']
self.logger.info(
f"Found merged release PR with version {match[1]}, "
f"commit id: {merge_commit['oid']}")
new_release = {
'version': match[1],
'commitish': merge_commit['oid'],
'pr_id': edge['node']['id'],
'author_name': merge_commit['author']['name'],
'author_email': merge_commit['author']['email']
}
self.new_release.update(new_release)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr['version']}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr['pr_url']})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr['issue_id'])
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr['issue_number'])
self.new_pr['repo'].cleanup()
prev_version = self.github.latest_release()
# if there are no previous releases, set version to 0.0.0
prev_version = prev_version if prev_version else '0.0.0'
self.new_pr['previous_version'] = prev_version
if Version.coerce(prev_version) >= Version.coerce(
self.new_pr['version']):
msg = f"Version ({prev_version}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = f"Making a new PR for release of version {self.new_pr['version']} based on an issue."
self.logger.info(msg)
try:
self.new_pr['repo'] = self.github.clone_repository()
if not self.new_pr['repo']:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_github = self.github.latest_release()
if Version.coerce(latest_github) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on Github"
)
# to fill in new_release['fs_path'] so that we can continue with PyPi upload
self.new_release = self.github.download_extract_zip(
self.new_release)
return self.new_release
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest Github release (zip).\n{exc}")
try:
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_new_pypi_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
# if there are no previous releases, set version to 0.0.0
latest_pypi = latest_pypi if latest_pypi else '0.0.0'
if Version.coerce(latest_pypi) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on PyPi"
)
return False
try:
self.pypi.release(self.new_release)
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return True
def make_new_fedora_release(self):
if not self.new_release.get('fedora'):
self.logger.debug('Skipping Fedora release')
return
self.logger.info("Triggering Fedora release")
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} on Fedora"
builds = ', '.join(self.fedora.builds)
if builds:
msg += f", successfully built for branches: {builds}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
name, email = self.github.get_user_contact()
self.new_release['commit_name'] = name
self.new_release['commit_email'] = email
success_ = self.fedora.release(self.new_release)
release_handler(success_)
except ReleaseException:
release_handler(success=False)
raise
def run(self):
self.logger.info(
f"release-bot v{configuration.version} reporting for duty!")
while True:
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
if self.make_new_pypi_release():
# There's no way how to tell whether there's already such a fedora 'release'
# so try to do it only when we just did PyPi release
self.make_new_fedora_release()
if self.new_release.get('trigger_on_issue'
) and self.find_open_release_issues():
if self.new_release.get('labels') is not None:
self.github.put_labels_on_issue(
self.new_pr['issue_number'],
self.new_release.get('labels'))
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.get('pr_id'))
self.cleanup()
self.logger.debug(
f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
def test_latest_release(expected_f):
expected = expected_f()
co = prepare_conf()
g = Github(co, None)
assert g.latest_release() == expected
class TestGithub:
"""Tests bot communication with Github"""
def setup_method(self):
""" setup any state tied to the execution of the given method in a
class. setup_method is invoked for every test method of a class.
"""
configuration = prepare_conf()
self.g_utils = GithubUtils()
self.g_utils.create_repo()
self.g_utils.setup_repo()
# set conf
configuration.repository_name = self.g_utils.repo
configuration.github_username = self.g_utils.github_user
configuration.refresh_interval = 1
repo_url = f"https://github.com/{self.g_utils.github_user}/{self.g_utils.repo}"
git = Git(repo_url, configuration)
self.github = Github(configuration, git)
def teardown_method(self):
""" teardown any state that was previously setup with a setup_method
call.
"""
if self.g_utils.repo:
self.g_utils.delete_repo()
self.g_utils.repo = None
@pytest.fixture()
def open_issue(self):
"""Opens issue in a repository"""
return self.g_utils.open_issue()
@pytest.fixture()
def open_issue_graphql(self):
"""Opens issue and returns it's GraphQL id"""
number = self.g_utils.open_issue()
query = f"issue(number: {number}) {{id}}"
response = self.github.query_repository(query).json()
self.github.detect_api_errors(response)
return number, response['data']['repository']['issue']['id']
def test_get_configuration(self):
"""Tests fetching release-conf from Github"""
assert self.github.get_configuration() == RELEASE_CONF
def test_close_issue(self, open_issue):
"""Tests closing issue"""
assert self.github.close_issue(open_issue)
def test_latest_rls_not_existing(self):
"""Tests version number when there is no latest release"""
assert self.github.latest_release() == '0.0.0'
def test_branch_exists_true(self):
"""Tests if branch exists"""
assert self.github.branch_exists('master')
def test_branch_exists_false(self):
"""Tests if branch doesn't exist"""
assert not self.github.branch_exists('not-master')
def test_add_comment(self, open_issue_graphql):
"""Tests adding comment on issue"""
number, graphql_id = open_issue_graphql
comments_count = self.g_utils.count_comments(number)
self.github.comment = "Test comment"
self.github.add_comment(graphql_id)
assert self.g_utils.count_comments(number) == comments_count + 1
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.github = Github(configuration)
self.pypi = PyPi(configuration)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
self.new_release = {}
def cleanup(self):
if 'tempdir' in self.new_release:
self.new_release['tempdir'].cleanup()
self.new_release = {}
self.github.comment = []
self.fedora.progress_log = []
def load_release_conf(self):
# load release configuration from release-conf.yaml in repository
release_conf = self.conf.load_release_conf(self.new_release['fs_path'])
self.new_release.update(release_conf)
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
while True:
edges = self.github.walk_through_closed_prs(start=cursor, direction='before')
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release', edge['node']['title'].lower())
if match and validate(match[1]):
merge_commit = edge['node']['mergeCommit']
self.logger.info(f"Found merged release PR with version {match[1]}, "
f"commit id: {merge_commit['oid']}")
self.new_release = {'version': match[1],
'commitish': merge_commit['oid'],
'pr_id': edge['node']['id'],
'author_name': merge_commit['author']['name'],
'author_email': merge_commit['author']['email']}
return True
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
released, self.new_release = self.github.make_new_release(self.new_release,
self.pypi.latest_version())
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return self.new_release
def make_new_pypi_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(self.new_release['version']):
self.logger.info(f"{self.new_release['version']} has already been released on PyPi")
return False
try:
self.pypi.release(self.new_release)
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return True
def make_new_fedora_release(self):
if not self.new_release.get('fedora'):
self.logger.debug('Skipping Fedora release')
return
self.logger.info("Triggering Fedora release")
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} on Fedora"
builds = ', '.join(self.fedora.builds)
if builds:
msg += f", successfully built for branches: {builds}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
success_ = self.fedora.release(self.new_release)
release_handler(success_)
except ReleaseException:
release_handler(success=False)
raise
def run(self):
self.logger.info(f"release-bot v{configuration.version} reporting for duty!")
while True:
try:
if self.find_newest_release_pull_request():
self.make_new_github_release()
self.load_release_conf()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
if self.make_new_pypi_release():
# There's no way how to tell whether there's already such a fedora 'release'
# so try to do it only when we just did PyPi release
self.make_new_fedora_release()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.get('pr_id'))
self.cleanup()
self.logger.debug(f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
url = f'https://github.com/{self.conf.repository_owner}/{self.conf.repository_name}.git'
self.git = Git(url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.fedora = Fedora(configuration)
self.logger = configuration.logger
# FIXME: it's cumbersome to work with these dicts - it's unclear how the content changes;
# get rid of them and replace them with individual variables
self.new_release = {}
self.new_pr = {}
def cleanup(self):
if 'tempdir' in self.new_release:
self.new_release['tempdir'].cleanup()
self.new_release = {}
self.new_pr = {}
self.github.comment = []
self.fedora.progress_log = []
self.git.cleanup()
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_configuration()
release_conf = self.conf.load_release_conf(conf)
self.new_release.update(release_conf)
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
cursor = ''
release_issues = {}
while True:
edges = self.github.walk_through_open_issues(start=cursor,
direction='before')
if not edges:
self.logger.debug(f'No more open issues found')
break
else:
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match:
version = match[1].strip()
if validate(version):
if edge['node']['authorAssociation'] in [
'MEMBER', 'OWNER', 'COLLABORATOR'
]:
release_issues[version] = edge['node']
self.logger.info(
f'Found new release issue with version: {version}'
)
else:
self.logger.warning(
f"Author association {edge['node']['authorAssociation']!r} "
f"not in ['MEMBER', 'OWNER', 'COLLABORATOR']"
)
else:
self.logger.warning(
f"{version!r} is not a valid version")
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
for version, node in release_issues.items():
self.new_pr = {
'version': version,
'issue_id': node['id'],
'issue_number': node['number'],
'labels': self.new_release.get('labels')
}
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
while True:
edges = self.github.walk_through_prs(start=cursor,
direction='before',
closed=True)
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
match = re.match(r'(.+) release',
edge['node']['title'].lower())
if match and validate(match[1]):
merge_commit = edge['node']['mergeCommit']
self.logger.info(
f"Found merged release PR with version {match[1]}, "
f"commit id: {merge_commit['oid']}")
new_release = {
'version': match[1],
'commitish': merge_commit['oid'],
'pr_id': edge['node']['id'],
'author_name': merge_commit['author']['name'],
'author_email': merge_commit['author']['email']
}
self.new_release.update(new_release)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr['version']}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr['pr_url']})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr['issue_id'])
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr['issue_number'])
latest_gh_str = self.github.latest_release()
self.new_pr['previous_version'] = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(
self.new_pr['version']):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = f"Making a new PR for release of version {self.new_pr['version']} based on an issue."
self.logger.info(msg)
try:
self.new_pr['repo'] = self.git
if not self.new_pr['repo']:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(
f"Failed getting latest Github release (zip).\n{exc}")
if Version.coerce(latest_release) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on Github"
)
else:
try:
released, self.new_release = self.github.make_new_release(
self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
self.github.update_changelog(self.new_release['version'])
return self.new_release
def make_new_pypi_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release['version']} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(
self.new_release['version']):
self.logger.info(
f"{self.new_release['version']} has already been released on PyPi"
)
return False
self.git.fetch_tags()
self.git.checkout(self.new_release['version'])
try:
self.pypi.release()
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
return True
def make_new_fedora_release(self):
if not self.new_release.get('fedora'):
self.logger.debug('Skipping Fedora release')
return
self.logger.info("Triggering Fedora release")
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} on Fedora"
builds = ', '.join(self.fedora.builds)
bodhi_update_url = "https://bodhi.fedoraproject.org/updates/new"
if builds:
msg += f", successfully built for branches: {builds}."
msg += f" Follow this link to create bodhi update(s): {bodhi_update_url}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
name, email = self.github.get_user_contact()
self.new_release['commit_name'] = name
self.new_release['commit_email'] = email
success_ = self.fedora.release(self.new_release)
release_handler(success_)
except ReleaseException:
release_handler(success=False)
raise
def run(self):
self.logger.info(
f"release-bot v{configuration.version} reporting for duty!")
try:
while True:
self.git.pull()
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
if self.make_new_pypi_release():
# There's no way how to tell whether there's already such a fedora 'release'
# so try to do it only when we just did PyPi release
self.make_new_fedora_release()
except ReleaseException as exc:
self.logger.error(exc)
# Moved out of the previous try-except block, because if it
# encounters ReleaseException while checking for PyPi sources
# it doesn't check for GitHub issues.
try:
if self.new_release.get(
'trigger_on_issue'
) and self.find_open_release_issues():
if self.new_release.get('labels') is not None:
self.github.put_labels_on_issue(
self.new_pr['issue_number'],
self.new_release.get('labels'))
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.get('pr_id'))
self.logger.debug(
f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
finally:
self.cleanup()
class ReleaseBot:
def __init__(self, configuration):
self.conf = configuration
self.git = Git(self.conf.clone_url, self.conf)
self.github = Github(configuration, self.git)
self.pypi = PyPi(configuration, self.git)
self.logger = configuration.logger
self.new_release = NewRelease()
self.new_pr = NewPR()
def cleanup(self):
self.new_release = NewRelease()
self.new_pr = NewPR()
self.github.comment = []
self.git.cleanup()
def create_flask_instance(self):
"""Create flask instance for receiving Github webhooks"""
app = Flask(__name__)
app.add_url_rule('/webhook-handler/', # route for github callbacks
view_func=GithubWebhooksHandler.as_view('github_webhooks_handler',
release_bot=self,
conf=configuration),
methods=['POST', ])
app.run(host='0.0.0.0', port=8080)
def load_release_conf(self):
"""
Updates new_release with latest release-conf.yaml from repository
:return:
"""
# load release configuration from release-conf.yaml in repository
conf = self.github.get_file("release-conf.yaml")
release_conf = self.conf.load_release_conf(conf)
setup_cfg = self.github.get_file("setup.cfg")
self.conf.set_pypi_project(release_conf, setup_cfg)
self.new_release.update(
changelog=release_conf.get('changelog'),
author_name=release_conf.get('author_name'),
author_email=release_conf.get('author_email'),
pypi=release_conf.get('pypi'),
trigger_on_issue=release_conf.get('trigger_on_issue'),
labels=release_conf.get('labels')
)
def find_open_release_issues(self):
"""
Looks for opened release issues on github
:return: True on found, False if not found
"""
cursor = ''
release_issues = {}
latest_version = Version(self.github.latest_release())
while True:
edges = self.github.walk_through_open_issues(start=cursor, direction='before')
if not edges:
self.logger.debug(f'No more open issues found')
break
else:
for edge in reversed(edges):
cursor = edge['cursor']
title = edge['node']['title'].lower().strip()
match, version = process_version_from_title(title, latest_version)
if match:
if edge['node']['authorAssociation'] in ['MEMBER', 'OWNER',
'COLLABORATOR']:
release_issues[version] = edge['node']
self.logger.info(f'Found new release issue with version: {version}')
else:
self.logger.warning(
f"Author association {edge['node']['authorAssociation']!r} "
f"not in ['MEMBER', 'OWNER', 'COLLABORATOR']")
if len(release_issues) > 1:
msg = f'Multiple release issues are open {release_issues}, please reduce them to one'
self.logger.error(msg)
return False
if len(release_issues) == 1:
for version, node in release_issues.items():
self.new_pr.update_new_pr_details(
version=version,
issue_id=node['id'],
issue_number=node['number'],
labels=self.new_release.labels
)
return True
else:
return False
def find_newest_release_pull_request(self):
"""
Find newest merged release PR
:return: bool, whether PR was found
"""
cursor = ''
latest_version = Version(self.github.latest_release())
while True:
edges = self.github.walk_through_prs(start=cursor, direction='before', closed=True)
if not edges:
self.logger.debug(f'No merged release PR found')
return False
for edge in reversed(edges):
cursor = edge['cursor']
title = edge['node']['title'].lower().strip()
match, version = process_version_from_title(title, latest_version)
if match:
merge_commit = edge['node']['mergeCommit']
self.logger.info(f"Found merged release PR with version {version}, "
f"commit id: {merge_commit['oid']}")
self.new_release.update_pr_details(
version=version,
commitish=merge_commit['oid'],
pr_id=edge['node']['id'],
author_email=merge_commit['author']['email'],
author_name=merge_commit['author']['name']
)
return True
def make_release_pull_request(self):
"""
Makes release pull request and handles outcome
:return: whether making PR was successful
"""
def pr_handler(success):
"""
Handler for the outcome of making a PR
:param success: whether making PR was successful
:return:
"""
result = 'made' if success else 'failed to make'
msg = f"I just {result} a PR request for a release version {self.new_pr.version}"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
if success:
msg += f"\n Here's a [link to the PR]({self.new_pr.pr_url})"
comment_backup = self.github.comment.copy()
self.github.comment = [msg]
self.github.add_comment(self.new_pr.issue_id)
self.github.comment = comment_backup
if success:
self.github.close_issue(self.new_pr.issue_number)
latest_gh_str = self.github.latest_release()
self.new_pr.previous_version = latest_gh_str
if Version.coerce(latest_gh_str) >= Version.coerce(self.new_pr.version):
msg = f"Version ({latest_gh_str}) is already released and this issue is ignored."
self.logger.warning(msg)
return False
msg = (f"Making a new PR for release of version "
f"{self.new_pr.version} based on the issue.")
if not self.conf.dry_run:
self.logger.info(msg)
try:
self.new_pr.repo = self.git
if not self.new_pr.repo:
raise ReleaseException("Couldn't clone repository!")
if self.github.make_release_pr(self.new_pr, self.conf.gitchangelog):
pr_handler(success=True)
return True
except ReleaseException:
pr_handler(success=False)
raise
return False
def make_new_github_release(self):
def release_handler(success):
result = "released" if success else "failed to release"
msg = f"I just {result} version {self.new_release.version} on Github"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
try:
latest_release = self.github.latest_release()
except ReleaseException as exc:
raise ReleaseException(f"Failed getting latest Github release (zip).\n{exc}")
if Version.coerce(latest_release) >= Version.coerce(self.new_release.version):
self.logger.info(
f"{self.new_release.version} has already been released on Github")
else:
try:
if self.conf.dry_run:
return None
released, self.new_release = self.github.make_new_release(self.new_release)
if released:
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
self.github.update_changelog(self.new_release.version)
return self.new_release
def make_new_pypi_release(self):
if not self.new_release.pypi:
self.logger.debug('Skipping PyPi release')
return False
def release_handler(success):
result = "released" if success else "failed to release"
if self.conf.dry_run:
msg = f"I would have {result} version {self.new_release.version} on PyPI now."
else:
msg = f"I just {result} version {self.new_release.version} on PyPI"
level = logging.INFO if success else logging.ERROR
self.logger.log(level, msg)
self.github.comment.append(msg)
latest_pypi = self.pypi.latest_version()
if Version.coerce(latest_pypi) >= Version.coerce(self.new_release.version):
msg = f"{self.conf.pypi_project}-{self.new_release.version} " \
f"or higher version has already been released on PyPi"
self.logger.info(msg)
return False
self.git.fetch_tags()
self.git.checkout(self.new_release.version)
try:
if self.pypi.release() == False:
return False
release_handler(success=True)
except ReleaseException:
release_handler(success=False)
raise
finally:
self.git.checkout('master')
return True
def run(self):
self.logger.info(f"release-bot v{configuration.version} reporting for duty!")
if self.conf.dry_run:
self.logger.info("Running in dry-run mode.")
try:
while True:
self.git.pull()
try:
self.load_release_conf()
if self.find_newest_release_pull_request():
self.make_new_github_release()
# Try to do PyPi release regardless whether we just did github release
# for case that in previous iteration (of the 'while True' loop)
# we succeeded with github release, but failed with PyPi release
self.make_new_pypi_release()
except ReleaseException as exc:
self.logger.error(exc)
# Moved out of the previous try-except block, because if it
# encounters ReleaseException while checking for PyPi sources
# it doesn't check for GitHub issues.
try:
if self.new_release.trigger_on_issue and self.find_open_release_issues():
if self.new_release.labels is not None:
self.github.put_labels_on_issue(self.new_pr.issue_number,
self.new_release.labels)
self.make_release_pull_request()
except ReleaseException as exc:
self.logger.error(exc)
self.github.add_comment(self.new_release.pr_id)
self.logger.debug(f"Done. Going to sleep for {self.conf.refresh_interval}s")
time.sleep(self.conf.refresh_interval)
finally:
self.cleanup()