예제 #1
0
class RestClientTest(TestCase):
    """ small test for The RestClient
    This should not be to much, since there is an hourly limit of requests for the github api
    """

    def setUp(self):
        """setup"""
        super(RestClientTest, self).setUp()
        self.client = RestClient('https://api.github.com', username=GITHUB_LOGIN, token=GITHUB_TOKEN)

    def test_client(self):
        """Do a test api call"""
        status, body = self.client.repos[GITHUB_USER][GITHUB_REPO].contents.a_directory['a_file.txt'].get()
        self.assertEqual(status, 200)
        # dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo= == 'this is a line of text' in base64 encoding
        self.assertEqual(body['content'].strip(), u"dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo=")

        status, body = self.client.repos['hpcugent']['easybuild-framework'].pulls[1].get()
        self.assertEqual(status, 200)
        self.assertEqual(body['merge_commit_sha'], u'fba3e13815f3d2a9dfbd2f89f1cf678dd58bb1f1')

    def test_request_methods(self):
        """Test all request methods"""
        status, body = self.client.head()
        self.assertEqual(status, 200)
        try:
            status, body = self.client.user.emails.post(body='*****@*****.**')
            self.assertTrue(False, 'posting to unauthorized endpoint did not trhow a http error')
        except HTTPError:
            pass
        try:
            status, body = self.client.user.emails.delete(body='*****@*****.**')
            self.assertTrue(False, 'deleting to unauthorized endpoint did not trhow a http error')
        except HTTPError:
            pass
예제 #2
0
 def __init__(self,
              githubuser,
              reponame,
              branchname="master",
              username=None,
              password=None,
              token=None):
     """Construct a new githubfs object
     @param githubuser: the github user's repo we want to use.
     @param reponame: The name of the repository we want to use.
     @param branchname: Then name of the branch to use (defaults to master)
     @param username: (optional) your github username.
     @param password: (optional) your github password.
     @param token:    (optional) a github api token.
     """
     if token is None:
         token = fetch_github_token(username)
     self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
     self.gh = RestClient(GITHUB_API_URL,
                          username=username,
                          password=password,
                          token=token)
     self.githubuser = githubuser
     self.reponame = reponame
     self.branchname = branchname
예제 #3
0
def github_api_get_request(request_f, github_user=None, **kwargs):
    """
    Helper method, for performing get requests to GitHub API.
    @param request_f: function that should be called to compose request, providing a RestClient instance
    @param github_user: GitHub user name (to try and obtain matching GitHub token)
    @return: tuple with return status and data
    """
    if github_user is None:
        github_user = build_option('github_user')

    token = fetch_github_token(github_user)
    url = request_f(
        RestClient(GITHUB_API_URL, username=github_user, token=token))

    try:
        status, data = url.get(**kwargs)
    except socket.gaierror, err:
        _log.warning("Error occured while performing get request: %s" % err)
        status, data = 0, None
예제 #4
0
def create_gist(txt, fn, descr=None, github_user=None):
    """Create a gist with the provided text."""
    if descr is None:
        descr = "(none)"
    github_token = fetch_github_token(github_user)

    body = {
        "description": descr,
        "public": True,
        "files": {
            fn: {
                "content": txt,
            }
        }
    }
    g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
    status, data = g.gists.post(body=body)

    if not status == HTTP_STATUS_CREATED:
        _log.error("Failed to create gist; status %s, data: %s" % (status, data))

    return data['html_url']
예제 #5
0
def post_comment_in_issue(issue,
                          txt,
                          repo=GITHUB_EASYCONFIGS_REPO,
                          github_user=None):
    """Post a comment in the specified PR."""
    if not isinstance(issue, int):
        try:
            issue = int(issue)
        except ValueError, err:
            raise EasyBuildError(
                "Failed to parse specified pull request number '%s' as an int: %s; ",
                issue, err)
    github_token = fetch_github_token(github_user)

    g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
    pr_url = g.repos[GITHUB_EB_MAIN][repo].issues[issue]

    status, data = pr_url.comments.post(body={'body': txt})
    if not status == HTTP_STATUS_CREATED:
        raise EasyBuildError(
            "Failed to create comment in PR %s#%d; status %s, data: %s", repo,
            issue, status, data)


class GithubToken(object):
    """Representation of a GitHub token."""

    # singleton metaclass: only one instance is created
    __metaclass__ = Singleton
예제 #6
0
def main():
    """the main function"""
    fancylogger.logToScreen(enable=True, stdout=True)
    fancylogger.setLogLevelInfo()

    options = {
        'github-user': ('Your github username to use', None, 'store', None, 'g'),
        'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'),
        'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'),
        'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'),
    }

    go = simple_option(options)
    log = go.log

    if not (go.options.all or go.options.closed_pr or go.options.orphans):
        log.error("Please tell me what to do?")

    if go.options.github_user is None:
        eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[])
        username = eb_go.options.github_user
        log.debug("Fetch github username from easybuild, found: %s", username)
    else:
        username = go.options.github_user

    if username is None:
        log.error("Could not find a github username")
    else:
        log.info("Using username = %s", username)

    token = fetch_github_token(username)

    gh = RestClient(GITHUB_API_URL, username=username, token=token)
    # ToDo: add support for pagination
    status, gists = gh.gists.get(per_page=100)

    if status != HTTP_STATUS_OK:
        log.error("Failed to get a lists of gists for user %s: error code %s, message = %s",
                  username, status, gists)
    else:
        log.info("Found %s gists", len(gists))

    regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P<PR>[0-9]+))?\)?$")

    pr_cache = {}
    num_deleted = 0

    for gist in gists:
        if not gist["description"]:
            continue
        re_pr_num = regex.search(gist["description"])
        delete_gist = False

        if re_pr_num:
            log.debug("Found a Easybuild gist (id=%s)", gist["id"])
            pr_num = re_pr_num.group("PR")
            if go.options.all:
                delete_gist = True
            elif pr_num and go.options.closed_pr:
                log.debug("Found Easybuild test report for PR #%s", pr_num)

                if pr_num not in pr_cache:
                    status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get()
                    if status != HTTP_STATUS_OK:
                        log.error("Failed to get pull-request #%s: error code %s, message = %s",
                                  pr_num, status, pr)
                    pr_cache[pr_num] = pr["state"]

                if pr_cache[pr_num] == "closed":
                    log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"])
                    delete_gist = True

            elif not pr_num and go.options.orphans:
                log.debug("Found Easybuild test report without PR (id=%s)", gist["id"])
                delete_gist = True

        if delete_gist:
            status, del_gist = gh.gists[gist["id"]].delete()

            if status != HTTP_DELETE_OK:
                log.error("Unable to remove gist (id=%s): error code %s, message = %s",
                          gist["id"], status, del_gist)
            else:
                log.info("Delete gist with id=%s", gist["id"])
                num_deleted += 1

    log.info("Deleted %s gists", num_deleted)
예제 #7
0
def new_pr(paths, title=None, descr=None, commit_msg=None):
    """Open new pull request using specified files."""

    _log.experimental("Opening new pull request for: %s", ', '.join(paths))

    pr_branch_name = build_option('pr_branch_name')
    pr_target_account = build_option('pr_target_account')
    pr_target_repo = build_option('pr_target_repo')

    # collect GitHub info we'll need
    # * GitHub username to push branch to repo
    # * GitHub token to open PR
    github_user = build_option('github_user')
    if github_user is None:
        raise EasyBuildError("GitHub user must be specified to use --new-pr")

    github_token = fetch_github_token(github_user)
    if github_token is None:
        raise EasyBuildError(
            "GitHub token for user '%s' must be available to use --new-pr",
            github_user)

    # create branch, commit files to it & push to GitHub
    file_info, git_repo, branch, diff_stat = _easyconfigs_pr_common(
        paths,
        pr_branch=pr_branch_name,
        target_account=pr_target_account,
        commit_msg=commit_msg)

    # only use most common toolchain(s) in toolchain label of PR title
    toolchains = [
        '%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']
    ]
    toolchains_counted = sorted([(toolchains.count(tc), tc)
                                 for tc in nub(toolchains)])
    toolchain_label = ','.join([
        tc for (cnt, tc) in toolchains_counted
        if cnt == toolchains_counted[-1][0]
    ])

    # only use most common module class(es) in moduleclass label of PR title
    classes = [ec['moduleclass'] for ec in file_info['ecs']]
    classes_counted = sorted([(classes.count(c), c) for c in nub(classes)])
    class_label = ','.join(
        [tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]])

    if title is None:
        # mention software name/version in PR title (only first 3)
        names_and_versions = [
            "%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']
        ]
        if len(names_and_versions) <= 3:
            main_title = ', '.join(names_and_versions)
        else:
            main_title = ', '.join(names_and_versions[:3] + ['...'])

        title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title)

    full_descr = "(created using `eb --new-pr`)\n"
    if descr is not None:
        full_descr += descr

    # create PR
    pr_target_branch = build_option('pr_target_branch')
    dry_run = build_option('dry_run') or build_option('extended_dry_run')

    msg = '\n'.join([
        '',
        "Opening pull request%s" % ('', " [DRY RUN]")[dry_run],
        "* target: %s/%s:%s" %
        (pr_target_account, pr_target_repo, pr_target_branch),
        "* from: %s/%s:%s" % (github_user, pr_target_repo, branch),
        "* title: \"%s\"" % title,
        "* description:",
        '"""',
        full_descr,
        '"""',
        "* overview of changes:\n%s" % diff_stat,
        '',
    ])
    print_msg(msg, log=_log, prefix=False)

    if not dry_run:
        g = RestClient(GITHUB_API_URL,
                       username=github_user,
                       token=github_token)
        pulls_url = g.repos[pr_target_account][pr_target_repo].pulls
        body = {
            'base': pr_target_branch,
            'head': '%s:%s' % (github_user, branch),
            'title': title,
            'body': full_descr,
        }
        status, data = pulls_url.post(body=body)
        if not status == HTTP_STATUS_CREATED:
            raise EasyBuildError(
                "Failed to open PR for branch %s; status %s, data: %s", branch,
                status, data)

        print_msg("Opened pull request: %s" % data['html_url'],
                  log=_log,
                  prefix=False)
예제 #8
0
def main():

    opts = {
        'dry-run': ("Dry run, don't actually post/push/merge anything", None,
                    'store_true', False, 'x'),
        'force': ("Use force to execute the specified action", None,
                  'store_true', False, 'f'),
        'github-account': ("GitHub account where repository is located", None,
                           'store', 'easybuilders', 'a'),
        'github-user': ("GitHub user to use (for authenticated access)", None,
                        'store', 'boegel', 'u'),
        'owner':
        ("Owner of the bot account that is used", None, 'store', None),
        'repository':
        ("Repository to use", None, 'store', 'easybuild-easyconfigs', 'r'),
        # actions
        'comment':
        ("Post a comment in the pull request", None, 'store', None, 'C'),
        'merge': ("Merge the pull request", None, 'store_true', False, 'M'),
        'review': ("Review the pull request", None, 'store_true', False, 'R'),
        'test':
        ("Submit job to upload test report", None, 'store_or_None', None, 'T'),
        'travis': ("Scan Travis test results, notify of failed tests in PRs",
                   None, 'store_true', False),
    }

    actions = ['comment', 'merge', 'review', 'test', 'travis']

    go = simple_option(go_dict=opts)
    init_build_options()

    # determine which action should be taken
    selected_action = None
    for action in sorted(actions):
        action_value = getattr(go.options, action)
        if isinstance(action_value, bool):
            if action_value:
                selected_action = (action, action_value)
                break
        elif action_value is not None:
            selected_action = (action, action_value)
            break  # FIXME: support multiple actions, loop over them (e.g. -C :jok,lgtm -T)

    if selected_action is None:
        avail_actions = ', '.join(
            ["%s (-%s)" % (a, a[0].upper()) for a in sorted(actions)])
        error("No action specified, pick one: %s" % avail_actions)
    else:
        info("Selected action: %s" % selected_action[0])

    global DRY_RUN
    DRY_RUN = go.options.dry_run
    force = go.options.force
    github_account = go.options.github_account
    github_user = go.options.github_user
    owner = go.options.owner
    repository = go.options.repository

    pr = None
    check_msg = None
    github_token = fetch_github_token(github_user)

    # prepare using GitHub API
    github = RestClient(GITHUB_API_URL,
                        username=github_user,
                        token=github_token,
                        user_agent='eb-pr-check')

    if selected_action[0] == 'travis':
        res = travis(github_account, repository, github_token, owner=owner)
        if res:
            for pr, pr_comment, check_msg in res:
                pr_data = fetch_pr_data(github, github_account, repository, pr)
                comment(github,
                        github_user,
                        repository,
                        pr_data,
                        pr_comment,
                        check_msg=check_msg,
                        verbose=DRY_RUN)
        else:
            print "Found no PRs to notify, all done here!"

    else:
        if len(go.args) == 1:
            pr = go.args[0]
        else:
            usage()

        print "Fetching PR information ",
        print "(using GitHub token for user '%s': %s)... " % (github_user, (
            'no', 'yes')[bool(github_token)]),
        sys.stdout.flush()
        pr_data = fetch_pr_data(github, github_account, repository, pr)
        print ''

        #print_raw_pr_info(pr_data)

        print_pr_summary(pr_data)

        if selected_action[0] == 'comment':
            comment(github,
                    github_user,
                    repository,
                    pr_data,
                    selected_action[1],
                    check_msg=check_msg)
        elif selected_action[0] == 'merge':
            merge(github,
                  github_user,
                  github_account,
                  repository,
                  pr_data,
                  force=force)
        elif selected_action[0] == 'review':
            review(pr_data)
        elif selected_action[0] == 'test':
            test(pr_data, selected_action[1])
        else:
            error("Handling action '%s' not implemented yet" %
                  selected_action[0])
예제 #9
0
 def setUp(self):
     """setup"""
     super(RestClientTest, self).setUp()
     self.client = RestClient('https://api.github.com',
                              username=GITHUB_LOGIN,
                              token=GITHUB_TOKEN)
예제 #10
0
class RestClientTest(TestCase):
    """ small test for The RestClient
    This should not be to much, since there is an hourly limit of requests for the github api
    """
    def setUp(self):
        """setup"""
        super(RestClientTest, self).setUp()
        self.client = RestClient('https://api.github.com',
                                 username=GITHUB_LOGIN,
                                 token=GITHUB_TOKEN)

    def test_client(self):
        """Do a test api call"""
        if GITHUB_TOKEN is None:
            print("Skipping test_client, since no GitHub token is available")
            return

        status, body = self.client.repos[GITHUB_USER][
            GITHUB_REPO].contents.a_directory['a_file.txt'].get()
        self.assertEqual(status, 200)
        # dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo= == 'this is a line of text' in base64 encoding
        self.assertEqual(body['content'].strip(),
                         u"dGhpcyBpcyBhIGxpbmUgb2YgdGV4dAo=")

        status, body = self.client.repos['hpcugent'][
            'easybuild-framework'].pulls[1].get()
        self.assertEqual(status, 200)
        self.assertEqual(body['merge_commit_sha'],
                         u'fba3e13815f3d2a9dfbd2f89f1cf678dd58bb1f1')

    def test_request_methods(self):
        """Test all request methods"""
        if GITHUB_TOKEN is None:
            print(
                "Skipping test_request_methods, since no GitHub token is available"
            )
            return

        status, headers = self.client.head()
        self.assertEqual(status, 200)
        self.assertTrue(headers)
        self.assertTrue('X-GitHub-Media-Type' in headers)
        try:
            status, body = self.client.user.emails.post(
                body='*****@*****.**')
            self.assertTrue(
                False,
                'posting to unauthorized endpoint did not trhow a http error')
        except HTTPError:
            pass
        try:
            status, body = self.client.user.emails.delete(
                body='*****@*****.**')
            self.assertTrue(
                False,
                'deleting to unauthorized endpoint did not trhow a http error')
        except HTTPError:
            pass

    def test_get_method(self):
        """A quick test of a GET to the github API"""

        status, body = self.client.users['hpcugent'].get()
        self.assertEqual(status, 200)
        self.assertEqual(body['login'], 'hpcugent')
        self.assertEqual(body['id'], 1515263)

    def test_get_connection(self):
        """Test for Client.get_connection."""

        client = Client('https://api.github.com')

        url = '/repos/hpcugent/vsc-base/pulls/296/comments'
        try:
            client.get_connection(Client.POST,
                                  url,
                                  body="{'body': 'test'}",
                                  headers={})
            self.assertTrue(
                False,
                "Trying to post a comment unauthorized should result in HTTPError"
            )
        except HTTPError:
            pass
예제 #11
0
파일: rest.py 프로젝트: Iepoev/vsc-base
 def setUp(self):
     """setup"""
     super(RestClientTest, self).setUp()
     self.client = RestClient('https://api.github.com', username=GITHUB_LOGIN, token=GITHUB_TOKEN)