class RestClientTest(TestCase):
""" small test for The RestClient
This should not be to much, since there is an hourly limit of requests for the github api
"""
def setUp(self):
"""setup"""
super(RestClientTest, self).setUp()
self.client = RestClient('https://api.github.com', username=GITHUB_LOGIN, token=GITHUB_TOKEN)
def test_client(self):
"""Do a test api call"""
status, body = self.client.repos[GITHUB_USER][GITHUB_REPO].contents.a_directory['a_file.txt'].get()
self.assertEqual(status, 200)
# dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo= == 'this is a line of text' in base64 encoding
self.assertEqual(body['content'].strip(), u"dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo=")
status, body = self.client.repos['hpcugent']['easybuild-framework'].pulls[1].get()
self.assertEqual(status, 200)
self.assertEqual(body['merge_commit_sha'], u'fba3e13815f3d2a9dfbd2f89f1cf678dd58bb1f1')
def test_request_methods(self):
"""Test all request methods"""
status, body = self.client.head()
self.assertEqual(status, 200)
try:
status, body = self.client.user.emails.post(body='*****@*****.**')
self.assertTrue(False, 'posting to unauthorized endpoint did not trhow a http error')
except HTTPError:
pass
try:
status, body = self.client.user.emails.delete(body='*****@*****.**')
self.assertTrue(False, 'deleting to unauthorized endpoint did not trhow a http error')
except HTTPError:
pass
def __init__(self,
githubuser,
reponame,
branchname="master",
username=None,
password=None,
token=None):
"""Construct a new githubfs object
@param githubuser: the github user's repo we want to use.
@param reponame: The name of the repository we want to use.
@param branchname: Then name of the branch to use (defaults to master)
@param username: (optional) your github username.
@param password: (optional) your github password.
@param token: (optional) a github api token.
"""
if token is None:
token = fetch_github_token(username)
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.gh = RestClient(GITHUB_API_URL,
username=username,
password=password,
token=token)
self.githubuser = githubuser
self.reponame = reponame
self.branchname = branchname
def github_api_get_request(request_f, github_user=None, **kwargs):
"""
Helper method, for performing get requests to GitHub API.
@param request_f: function that should be called to compose request, providing a RestClient instance
@param github_user: GitHub user name (to try and obtain matching GitHub token)
@return: tuple with return status and data
"""
if github_user is None:
github_user = build_option('github_user')
token = fetch_github_token(github_user)
url = request_f(
RestClient(GITHUB_API_URL, username=github_user, token=token))
try:
status, data = url.get(**kwargs)
except socket.gaierror, err:
_log.warning("Error occured while performing get request: %s" % err)
status, data = 0, None
def create_gist(txt, fn, descr=None, github_user=None):
"""Create a gist with the provided text."""
if descr is None:
descr = "(none)"
github_token = fetch_github_token(github_user)
body = {
"description": descr,
"public": True,
"files": {
fn: {
"content": txt,
}
}
}
g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
status, data = g.gists.post(body=body)
if not status == HTTP_STATUS_CREATED:
_log.error("Failed to create gist; status %s, data: %s" % (status, data))
return data['html_url']
def post_comment_in_issue(issue,
txt,
repo=GITHUB_EASYCONFIGS_REPO,
github_user=None):
"""Post a comment in the specified PR."""
if not isinstance(issue, int):
try:
issue = int(issue)
except ValueError, err:
raise EasyBuildError(
"Failed to parse specified pull request number '%s' as an int: %s; ",
issue, err)
github_token = fetch_github_token(github_user)
g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
pr_url = g.repos[GITHUB_EB_MAIN][repo].issues[issue]
status, data = pr_url.comments.post(body={'body': txt})
if not status == HTTP_STATUS_CREATED:
raise EasyBuildError(
"Failed to create comment in PR %s#%d; status %s, data: %s", repo,
issue, status, data)
class GithubToken(object):
"""Representation of a GitHub token."""
# singleton metaclass: only one instance is created
__metaclass__ = Singleton
def main():
"""the main function"""
fancylogger.logToScreen(enable=True, stdout=True)
fancylogger.setLogLevelInfo()
options = {
'github-user': ('Your github username to use', None, 'store', None, 'g'),
'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'),
'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'),
'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'),
}
go = simple_option(options)
log = go.log
if not (go.options.all or go.options.closed_pr or go.options.orphans):
log.error("Please tell me what to do?")
if go.options.github_user is None:
eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[])
username = eb_go.options.github_user
log.debug("Fetch github username from easybuild, found: %s", username)
else:
username = go.options.github_user
if username is None:
log.error("Could not find a github username")
else:
log.info("Using username = %s", username)
token = fetch_github_token(username)
gh = RestClient(GITHUB_API_URL, username=username, token=token)
# ToDo: add support for pagination
status, gists = gh.gists.get(per_page=100)
if status != HTTP_STATUS_OK:
log.error("Failed to get a lists of gists for user %s: error code %s, message = %s",
username, status, gists)
else:
log.info("Found %s gists", len(gists))
regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P<PR>[0-9]+))?\)?$")
pr_cache = {}
num_deleted = 0
for gist in gists:
if not gist["description"]:
continue
re_pr_num = regex.search(gist["description"])
delete_gist = False
if re_pr_num:
log.debug("Found a Easybuild gist (id=%s)", gist["id"])
pr_num = re_pr_num.group("PR")
if go.options.all:
delete_gist = True
elif pr_num and go.options.closed_pr:
log.debug("Found Easybuild test report for PR #%s", pr_num)
if pr_num not in pr_cache:
status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get()
if status != HTTP_STATUS_OK:
log.error("Failed to get pull-request #%s: error code %s, message = %s",
pr_num, status, pr)
pr_cache[pr_num] = pr["state"]
if pr_cache[pr_num] == "closed":
log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"])
delete_gist = True
elif not pr_num and go.options.orphans:
log.debug("Found Easybuild test report without PR (id=%s)", gist["id"])
delete_gist = True
if delete_gist:
status, del_gist = gh.gists[gist["id"]].delete()
if status != HTTP_DELETE_OK:
log.error("Unable to remove gist (id=%s): error code %s, message = %s",
gist["id"], status, del_gist)
else:
log.info("Delete gist with id=%s", gist["id"])
num_deleted += 1
log.info("Deleted %s gists", num_deleted)
def new_pr(paths, title=None, descr=None, commit_msg=None):
"""Open new pull request using specified files."""
_log.experimental("Opening new pull request for: %s", ', '.join(paths))
pr_branch_name = build_option('pr_branch_name')
pr_target_account = build_option('pr_target_account')
pr_target_repo = build_option('pr_target_repo')
# collect GitHub info we'll need
# * GitHub username to push branch to repo
# * GitHub token to open PR
github_user = build_option('github_user')
if github_user is None:
raise EasyBuildError("GitHub user must be specified to use --new-pr")
github_token = fetch_github_token(github_user)
if github_token is None:
raise EasyBuildError(
"GitHub token for user '%s' must be available to use --new-pr",
github_user)
# create branch, commit files to it & push to GitHub
file_info, git_repo, branch, diff_stat = _easyconfigs_pr_common(
paths,
pr_branch=pr_branch_name,
target_account=pr_target_account,
commit_msg=commit_msg)
# only use most common toolchain(s) in toolchain label of PR title
toolchains = [
'%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']
]
toolchains_counted = sorted([(toolchains.count(tc), tc)
for tc in nub(toolchains)])
toolchain_label = ','.join([
tc for (cnt, tc) in toolchains_counted
if cnt == toolchains_counted[-1][0]
])
# only use most common module class(es) in moduleclass label of PR title
classes = [ec['moduleclass'] for ec in file_info['ecs']]
classes_counted = sorted([(classes.count(c), c) for c in nub(classes)])
class_label = ','.join(
[tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]])
if title is None:
# mention software name/version in PR title (only first 3)
names_and_versions = [
"%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']
]
if len(names_and_versions) <= 3:
main_title = ', '.join(names_and_versions)
else:
main_title = ', '.join(names_and_versions[:3] + ['...'])
title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title)
full_descr = "(created using `eb --new-pr`)\n"
if descr is not None:
full_descr += descr
# create PR
pr_target_branch = build_option('pr_target_branch')
dry_run = build_option('dry_run') or build_option('extended_dry_run')
msg = '\n'.join([
'',
"Opening pull request%s" % ('', " [DRY RUN]")[dry_run],
"* target: %s/%s:%s" %
(pr_target_account, pr_target_repo, pr_target_branch),
"* from: %s/%s:%s" % (github_user, pr_target_repo, branch),
"* title: \"%s\"" % title,
"* description:",
'"""',
full_descr,
'"""',
"* overview of changes:\n%s" % diff_stat,
'',
])
print_msg(msg, log=_log, prefix=False)
if not dry_run:
g = RestClient(GITHUB_API_URL,
username=github_user,
token=github_token)
pulls_url = g.repos[pr_target_account][pr_target_repo].pulls
body = {
'base': pr_target_branch,
'head': '%s:%s' % (github_user, branch),
'title': title,
'body': full_descr,
}
status, data = pulls_url.post(body=body)
if not status == HTTP_STATUS_CREATED:
raise EasyBuildError(
"Failed to open PR for branch %s; status %s, data: %s", branch,
status, data)
print_msg("Opened pull request: %s" % data['html_url'],
log=_log,
prefix=False)
def main():
opts = {
'dry-run': ("Dry run, don't actually post/push/merge anything", None,
'store_true', False, 'x'),
'force': ("Use force to execute the specified action", None,
'store_true', False, 'f'),
'github-account': ("GitHub account where repository is located", None,
'store', 'easybuilders', 'a'),
'github-user': ("GitHub user to use (for authenticated access)", None,
'store', 'boegel', 'u'),
'owner':
("Owner of the bot account that is used", None, 'store', None),
'repository':
("Repository to use", None, 'store', 'easybuild-easyconfigs', 'r'),
# actions
'comment':
("Post a comment in the pull request", None, 'store', None, 'C'),
'merge': ("Merge the pull request", None, 'store_true', False, 'M'),
'review': ("Review the pull request", None, 'store_true', False, 'R'),
'test':
("Submit job to upload test report", None, 'store_or_None', None, 'T'),
'travis': ("Scan Travis test results, notify of failed tests in PRs",
None, 'store_true', False),
}
actions = ['comment', 'merge', 'review', 'test', 'travis']
go = simple_option(go_dict=opts)
init_build_options()
# determine which action should be taken
selected_action = None
for action in sorted(actions):
action_value = getattr(go.options, action)
if isinstance(action_value, bool):
if action_value:
selected_action = (action, action_value)
break
elif action_value is not None:
selected_action = (action, action_value)
break # FIXME: support multiple actions, loop over them (e.g. -C :jok,lgtm -T)
if selected_action is None:
avail_actions = ', '.join(
["%s (-%s)" % (a, a[0].upper()) for a in sorted(actions)])
error("No action specified, pick one: %s" % avail_actions)
else:
info("Selected action: %s" % selected_action[0])
global DRY_RUN
DRY_RUN = go.options.dry_run
force = go.options.force
github_account = go.options.github_account
github_user = go.options.github_user
owner = go.options.owner
repository = go.options.repository
pr = None
check_msg = None
github_token = fetch_github_token(github_user)
# prepare using GitHub API
github = RestClient(GITHUB_API_URL,
username=github_user,
token=github_token,
user_agent='eb-pr-check')
if selected_action[0] == 'travis':
res = travis(github_account, repository, github_token, owner=owner)
if res:
for pr, pr_comment, check_msg in res:
pr_data = fetch_pr_data(github, github_account, repository, pr)
comment(github,
github_user,
repository,
pr_data,
pr_comment,
check_msg=check_msg,
verbose=DRY_RUN)
else:
print "Found no PRs to notify, all done here!"
else:
if len(go.args) == 1:
pr = go.args[0]
else:
usage()
print "Fetching PR information ",
print "(using GitHub token for user '%s': %s)... " % (github_user, (
'no', 'yes')[bool(github_token)]),
sys.stdout.flush()
pr_data = fetch_pr_data(github, github_account, repository, pr)
print ''
#print_raw_pr_info(pr_data)
print_pr_summary(pr_data)
if selected_action[0] == 'comment':
comment(github,
github_user,
repository,
pr_data,
selected_action[1],
check_msg=check_msg)
elif selected_action[0] == 'merge':
merge(github,
github_user,
github_account,
repository,
pr_data,
force=force)
elif selected_action[0] == 'review':
review(pr_data)
elif selected_action[0] == 'test':
test(pr_data, selected_action[1])
else:
error("Handling action '%s' not implemented yet" %
selected_action[0])
def setUp(self):
"""setup"""
super(RestClientTest, self).setUp()
self.client = RestClient('https://api.github.com',
username=GITHUB_LOGIN,
token=GITHUB_TOKEN)
class RestClientTest(TestCase):
""" small test for The RestClient
This should not be to much, since there is an hourly limit of requests for the github api
"""
def setUp(self):
"""setup"""
super(RestClientTest, self).setUp()
self.client = RestClient('https://api.github.com',
username=GITHUB_LOGIN,
token=GITHUB_TOKEN)
def test_client(self):
"""Do a test api call"""
if GITHUB_TOKEN is None:
print("Skipping test_client, since no GitHub token is available")
return
status, body = self.client.repos[GITHUB_USER][
GITHUB_REPO].contents.a_directory['a_file.txt'].get()
self.assertEqual(status, 200)
# dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo= == 'this is a line of text' in base64 encoding
self.assertEqual(body['content'].strip(),
u"dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo=")
status, body = self.client.repos['hpcugent'][
'easybuild-framework'].pulls[1].get()
self.assertEqual(status, 200)
self.assertEqual(body['merge_commit_sha'],
u'fba3e13815f3d2a9dfbd2f89f1cf678dd58bb1f1')
def test_request_methods(self):
"""Test all request methods"""
if GITHUB_TOKEN is None:
print(
"Skipping test_request_methods, since no GitHub token is available"
)
return
status, headers = self.client.head()
self.assertEqual(status, 200)
self.assertTrue(headers)
self.assertTrue('X-GitHub-Media-Type' in headers)
try:
status, body = self.client.user.emails.post(
body='*****@*****.**')
self.assertTrue(
False,
'posting to unauthorized endpoint did not trhow a http error')
except HTTPError:
pass
try:
status, body = self.client.user.emails.delete(
body='*****@*****.**')
self.assertTrue(
False,
'deleting to unauthorized endpoint did not trhow a http error')
except HTTPError:
pass
def test_get_method(self):
"""A quick test of a GET to the github API"""
status, body = self.client.users['hpcugent'].get()
self.assertEqual(status, 200)
self.assertEqual(body['login'], 'hpcugent')
self.assertEqual(body['id'], 1515263)
def test_get_connection(self):
"""Test for Client.get_connection."""
client = Client('https://api.github.com')
url = '/repos/hpcugent/vsc-base/pulls/296/comments'
try:
client.get_connection(Client.POST,
url,
body="{'body': 'test'}",
headers={})
self.assertTrue(
False,
"Trying to post a comment unauthorized should result in HTTPError"
)
except HTTPError:
pass
def setUp(self):
"""setup"""
super(RestClientTest, self).setUp()
self.client = RestClient('https://api.github.com', username=GITHUB_LOGIN, token=GITHUB_TOKEN)