Example #1
0
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        build_log.EXPERIMENTAL = self.options.experimental

        # set strictness of run module
        if self.options.strict:
            run.strictness = self.options.strict

        # override current version of EasyBuild with version specified to --deprecated
        if self.options.deprecated:
            build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)

        # log to specified value of --unittest-file
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        # set tmpdir
        self.tmpdir = set_tmpdir(self.options.tmpdir)

        # take --include options into account
        self._postprocess_include()

        # prepare for --list/--avail
        if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
                self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants,
                self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
                self.options.avail_repositories, self.options.show_default_moduleclasses,
                self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
                self.options.show_default_configfiles,
                ]):
            build_easyconfig_constants_dict()  # runs the easyconfig constants sanity check
            self._postprocess_list_avail()

        # fail early if required dependencies for functionality requiring using GitHub API are not available:
        if self.options.from_pr or self.options.upload_test_report:
            if not HAVE_GITHUB_API:
                raise EasyBuildError("Required support for using GitHub API is not available (see warnings).")

        if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
            raise EasyBuildError("Generating Lua module files requires Lmod as modules tool.")

        # make sure a GitHub token is available when it's required
        if self.options.upload_test_report:
            if not HAVE_KEYRING:
                raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.")
            if self.options.github_user is None:
                raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.")
            token = fetch_github_token(self.options.github_user)
            if token is None:
                raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user)

        # make sure autopep8 is available when it needs to be
        if self.options.dump_autopep8:
            if not HAVE_AUTOPEP8:
                raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested")

        self._postprocess_external_modules_metadata()

        self._postprocess_config()
Example #2
0
 def setUp(self):
     """setup"""
     super(GithubTest, self).setUp()
     self.github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT)
     if self.github_token is None:
         self.ghfs = None
     else:
         self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, GITHUB_TEST_ACCOUNT, None, self.github_token)
Example #3
0
 def setUp(self):
     """setup"""
     super(GithubTest, self).setUp()
     github_user = GITHUB_TEST_ACCOUNT
     github_token = fetch_github_token(github_user)
     if github_token is None:
         self.ghfs = None
     else:
         self.ghfs = Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, github_user, None, github_token)
Example #4
0
    def setUp(self):
        """setup"""
        super(GithubTest, self).setUp()
        self.github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT)
        if self.github_token is None:
            self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, None, None, None)
        else:
            self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, GITHUB_TEST_ACCOUNT,
                                    None, self.github_token)

        self.skip_github_tests = self.github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None
Example #5
0
    def test_install_github_token(self):
        """Test for install_github_token function."""
        if self.skip_github_tests:
            print "Skipping test_install_github_token, no GitHub token available?"
            return

        if not HAVE_KEYRING:
            print "Skipping test_install_github_token, keyring module not available"
            return

        random_user = ''.join(random.choice(string.letters) for _ in range(10))
        self.assertEqual(gh.fetch_github_token(random_user), None)

        # poor mans mocking of getpass
        # inject leading/trailing spaces to verify stripping of provided value
        def fake_getpass(*args, **kwargs):
            return ' ' + self.github_token + '  '

        orig_getpass = gh.getpass.getpass
        gh.getpass.getpass = fake_getpass

        token_installed = False
        try:
            gh.install_github_token(random_user, silent=True)
            token_installed = True
        except Exception as err:
            print err

        gh.getpass.getpass = orig_getpass

        token = gh.fetch_github_token(random_user)

        # cleanup
        if token_installed:
            keyring.delete_password(gh.KEYRING_GITHUB_TOKEN, random_user)

        # deliberately not using assertEqual, keep token secret!
        self.assertTrue(token_installed)
        self.assertTrue(token == self.github_token)
Example #6
0
def main():

    opts = {
        'github-account': ("GitHub account where repository is located", None, 'store', 'hpcugent', 'a'),
        'github-user': ("GitHub user to use (for authenticated access)", None, 'store', 'boegel', 'u'),
        'repository': ("Repository to use", None, 'store', 'easybuild-easyconfigs', 'r'),
    }
    go = simple_option(go_dict=opts, descr="Script to print overview of pull requests for a GitHub repository")

    github_token = fetch_github_token(go.options.github_user)
    github = RestClient(GITHUB_API_URL, username=go.options.github_user, token=github_token, user_agent='eb-pr-overview')

    downloading_msg = "Downloading PR data for %s/%s repo..." % (go.options.github_account, go.options.repository)
    print(downloading_msg)

    prs_data = fetch_prs_data(github, go.options.github_account, go.options.repository, downloading_msg)
    gh_repo = github.repos[go.options.github_account][go.options.repository]
    create_pr_overview(prs_data, gh_repo)
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        build_log.EXPERIMENTAL = self.options.experimental
        config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig

        # set strictness of run module
        if self.options.strict:
            run.strictness = self.options.strict

        # override current version of EasyBuild with version specified to --deprecated
        if self.options.deprecated:
            build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)

        # log to specified value of --unittest-file
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        # prepare for --list/--avail
        if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
                self.options.list_easyblocks, self.options.list_toolchains,
                self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
                self.options.avail_repositories, self.options.show_default_moduleclasses,
                self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
               ]):
            build_easyconfig_constants_dict()  # runs the easyconfig constants sanity check
            self._postprocess_list_avail()

        # fail early if required dependencies for functionality requiring using GitHub API are not available:
        if self.options.from_pr or self.options.upload_test_report:
            if not HAVE_GITHUB_API:
                self.log.error("Required support for using GitHub API is not available (see warnings).")

        # make sure a GitHub token is available when it's required
        if self.options.upload_test_report:
            if not HAVE_KEYRING:
                self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.")
            if self.options.github_user is None:
                self.log.error("No GitHub user name provided, required for fetching GitHub token.")
            token = fetch_github_token(self.options.github_user)
            if token is None:
                self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user)

        self._postprocess_config()
Example #8
0
def fetch_pr_data(pickle_file, github_user, github_account, repository):
    """Fetch PR data; either download or load from pickle file."""
    if pickle_file:
        print("Loading PR data from %s" % pickle_file)
        prs = cPickle.load(open(pickle_file, "r"))

    else:
        github_token = fetch_github_token(github_user)
        github = RestClient(GITHUB_API_URL, username=github_user, token=github_token, user_agent="eb-pr-stats")
        gh_repo = github.repos[github_account][repository]

        downloading_msg = "Downloading PR data for %s/%s repo..." % (github_account, repository)
        print(downloading_msg)

        prs = fetch_prs_data(github, github_account, repository, downloading_msg)
        pickle_file = PICKLE_FILE % repository
        cPickle.dump(prs, open(pickle_file, "w"))
        print("PR data dumped to %s" % pickle_file)

    return prs
Example #9
0
 def setUp(self):
     """Set up test."""
     super(RobotTest, self).setUp()
     self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT)
     self.orig_experimental = easybuild.framework.easyconfig.tools._log.experimental
     self.orig_modtool = self.modtool
Example #10
0
 def setUp(self):
     """Set up test."""
     super(RobotTest, self).setUp()
     self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT)
     self.orig_experimental = easybuild.framework.easyconfig.tools._log.experimental
Example #11
0
def main():

    opts = {
        'core-cnt': ("Default core count to use for jobs", None, 'store', None),
        'github-account': ("GitHub account where repository is located", None, 'store', 'easybuilders', 'a'),
        'github-user': ("GitHub user to use (for authenticated access)", None, 'store', 'boegel', 'u'),
        'mode': ("Mode to run in", 'choice', 'store', MODE_CHECK_TRAVIS,
                 [MODE_CHECK_GITHUB_ACTIONS, MODE_CHECK_TRAVIS, MODE_TEST_PR]),
        'owner': ("Owner of the bot account that is used", None, 'store', 'boegel'),
        'repository': ("Repository to use", None, 'store', 'easybuild-easyconfigs', 'r'),
        'host': ("Label for current host (used to filter comments asking to test a PR)", None, 'store', ''),
        'pr-test-cmd': ("Command to use for testing easyconfig pull requests (should include '%(pr)s' template value)",
                        None, 'store', ''),
    }

    go = simple_option(go_dict=opts)
    init_build_options()

    github_account = go.options.github_account
    github_user = go.options.github_user
    mode = go.options.mode
    owner = go.options.owner
    owner = go.options.owner
    repository = go.options.repository
    host = go.options.host
    pr_test_cmd = go.options.pr_test_cmd
    core_cnt = go.options.core_cnt

    github_token = fetch_github_token(github_user)

    # prepare using GitHub API
    github = RestClient(GITHUB_API_URL, username=github_user, token=github_token, user_agent='eb-pr-check')

    if mode in [MODE_CHECK_GITHUB_ACTIONS, MODE_CHECK_TRAVIS]:

        if mode == MODE_CHECK_TRAVIS:
            res = fetch_travis_failed_builds(github_account, repository, owner, github_token)
        elif mode == MODE_CHECK_GITHUB_ACTIONS:
            res = fetch_github_failed_workflows(github, github_account, repository, github_user, owner)
        else:
            error("Unknown mode: %s" % mode)

        for pr, pr_comment, check_msg in res:
            params = {'per_page': GITHUB_MAX_PER_PAGE}
            pr_data, _ = fetch_pr_data(pr, github_account, repository, github_user, full=True, **params)
            if pr_data['state'] == GITHUB_PR_STATE_OPEN:
                comment(github, github_user, repository, pr_data, pr_comment, check_msg=check_msg, verbose=DRY_RUN)
            else:
                print("Not posting comment in already closed %s PR #%s" % (repository, pr))

    elif mode == MODE_TEST_PR:
        if not host:
            error("--host is required when using '--mode %s' !" % MODE_TEST_PR)

        if '%(pr)s' not in pr_test_cmd or '%(eb_args)s' not in pr_test_cmd:
            error("--pr-test-cmd should include '%%(pr)s' and '%%(eb_args)s', found '%s'" % (pr_test_cmd))

        if core_cnt is None:
            error("--core-cnt must be used to specify the default number of cores to request per submitted job!")

        notifications = check_notifications(github, github_user, github_account, repository)
        process_notifications(notifications, github, github_user, github_account, repository, host, pr_test_cmd,
                              core_cnt)
    else:
        error("Unknown mode: %s" % mode)
Example #12
0
def fetch_github_failed_workflows(github, github_account, repository, github_user, owner):
    """Scan GitHub Actions for failed workflow runs."""

    res = []

    # only consider failed workflows triggered by pull requests
    params = {
        'event': 'pull_request',
        # filtering based on status='failure' no longer works correctly?!
        # also with status='completed' some workflow runs are not included in result...
        # 'status': 'failure',
        'per_page': GITHUB_MAX_PER_PAGE,
    }

    try:
        status, run_data = github.repos[github_account][repository].actions.runs.get(**params)
    except socket.gaierror as err:
        error("Failed to download GitHub Actions workflow runs data: %s" % err)

    if status == 200:
        run_data = list(run_data['workflow_runs'])
        print("Found %s failed workflow runs for %s/%s" % (len(run_data), github_account, repository))
    else:
        error("Status for downloading GitHub Actions workflow runs data should be 200, got %s" % status)

    failing_prs = set()

    for idx, entry in enumerate(run_data):

        if entry['status'] != 'completed':
            print("Ignoring incomplete workflow run %s" % entry['html_url'])
            continue

        if entry['conclusion'] == 'success':
            print("Ignoring successful workflow run %s" % entry['html_url'])
            continue

        head_user = entry['head_repository']['owner']['login']
        head = '%s:%s' % (head_user, entry['head_branch'])
        head_sha = entry['head_sha']

        # determine corresponding PR (if any)
        status, pr_data = github.repos[github_account][repository].pulls.get(head=head)
        if status != 200:
            error("Status for downloading data for PR with head %s should be 200, got %s" % (head, status))

        if len(pr_data) == 1:
            pr_data = pr_data[0]
            print("Failed workflow run %s found (PR: %s)" % (entry['html_url'], pr_data['html_url']))

            pr_id = pr_data['number']

            # skip PRs for which a failing workflow was already encountered
            if pr_id in failing_prs:
                print("PR #%s already encountered, so skipping workflow %s" % (pr_id, entry['html_url']))
                continue

            pr_data, _ = fetch_pr_data(pr_id, github_account, repository, github_user, full=True,
                                       per_page=GITHUB_MAX_PER_PAGE)

            if pr_data['state'] == 'open':

                pr_head_sha = pr_data['head']['sha']

                # make sure workflow was run for latest commit in this PR
                if head_sha != pr_head_sha:
                    msg = "Workflow %s was for commit %s, " % (entry['html_url'], head_sha)
                    msg += "not latest commit in PR #%s (%s), so skipping" % (pr_id, pr_head_sha)
                    print(msg)
                    continue

                # check status of most recent commit in this PR,
                # ignore this PR if status is "success" or "pending"
                pr_status = pr_data['status_last_commit']
                print("Status of last commit (%s) in PR #%s: %s" % (pr_head_sha, pr_id, pr_status))

                if pr_status in ['action_required', STATUS_PENDING, STATUS_SUCCESS]:
                    print("Status of last commit in PR #%s is '%s', so ignoring it for now..." % (pr_id, pr_status))
                    continue

                # download list of jobs in workflow
                run_id = entry['id']
                status, jobs_data = github.repos[github_account][repository].actions.runs[run_id].jobs.get()
                if status != 200:
                    error("Failed to download list of jobs for workflow run %s" % entry['html_url'])

                # determine ID of first failing job
                job_id = None
                for job in jobs_data['jobs']:
                    if job['conclusion'] == 'failure':
                        job_id = job['id']
                        print("Found failing job for workflow %s: %s" % (entry['html_url'], job_id))
                        break

                if job_id is None:
                    error("ID of failing job not found for workflow %s" % entry['html_url'])

                try:
                    status, log_txt = github.repos[github_account][repository].actions.jobs[job_id].logs.get()
                except HTTPError as err:
                    status = err.code

                if status == 200:
                    print("Downloaded log for job %s" % job_id)
                else:
                    warning("Failed to download log for job %s" % job_id)
                    log_txt = '(failed to fetch log contents due to HTTP status code %s)' % status

                # strip off timestamp prefixes
                # example timestamp: 2020-07-13T09:54:36.5004935Z
                timestamp_regex = re.compile(r'^[0-9-]{10}T[0-9:]{8}\.[0-9]+Z ')
                log_lines = [timestamp_regex.sub('', x) for x in log_txt.splitlines()]

                # determine line that marks end of output for failing test suite:
                # "ERROR: Not all tests were successful"
                error_line_idx = None
                for idx, line in enumerate(log_lines):
                    if line.startswith("ERROR: Not all tests were successful"):
                        error_line_idx = idx
                        print("Found error line @ index %s" % error_line_idx)
                        break

                if error_line_idx is None:
                    log_txt_clean = '\n'.join(log_lines)
                    warning("Log line that marks end of test suite output not found for job %s!\n%s" % (job_id, log_txt_clean))
                    if is_fluke(log_txt):
                        owner_gh_token = fetch_github_token(owner)
                        if owner_gh_token:
                            github_owner = RestClient(GITHUB_API_URL, username=owner, token=owner_gh_token,
                                                      user_agent='eb-pr-check')
                            print("Fluke found, restarting this workflow using @%s's GitHub account..." % owner)
                            repo_api = github_owner.repos[github_account][repository]
                            status, jobs_data = repo_api.actions.runs[run_id].rerun.post()
                            if status == 201:
                                print("Workflow %s restarted" % entry['html_url'])
                            else:
                                print("Failed to restart workflow %s: status %s" % (entry['html_url'], status))
                        else:
                            warning("Fluke found but can't restart workflow, no token found for @%s" % owner)

                    continue

                # find line that marks start of test output: only dots and 'E'/'F' characters
                start_test_regex = re.compile(r'^[\.EF]+$')
                start_line_idx = error_line_idx
                start_log_line = log_lines[start_line_idx]
                while(start_line_idx >= 0 and not (start_log_line and start_test_regex.match(start_log_line))):
                    start_line_idx -= 1
                    start_log_line = log_lines[start_line_idx]

                log_lines = log_lines[start_line_idx+1:error_line_idx+1]

                # compose comment
                pr_comment = "@%s: Tests failed in GitHub Actions" % pr_data['user']['login']
                pr_comment += ", see %s" % entry['html_url']

                # use first part of comment to check whether comment was already posted
                check_msg = pr_comment

                if len(log_lines) > 100:
                    log_lines = log_lines[-100:]
                    pr_comment += "\nLast 100 lines of output from first failing test suite run:\n\n```"
                else:
                    pr_comment += "\nOutput from first failing test suite run:\n\n```"

                for line in log_lines:
                    pr_comment += line + '\n'

                pr_comment += "```\n"

                pr_comment += "\n*bleep, bloop, I'm just a bot (boegelbot v%s)*\n" % VERSION
                pr_comment += "Please talk to my owner `@%s` if you notice you me acting stupid),\n" % owner
                pr_comment += "or submit a pull request to https://github.com/boegel/boegelbot fix the problem."

                res.append((pr_id, pr_comment, check_msg))
                failing_prs.add(pr_id)
            else:
                print("Ignoring failed workflow run for closed PR %s" % pr_data['html_url'])
        else:
            warning("Expected exactly one PR with head %s, found %s: %s" % (head, len(pr_data), pr_data))

    print("Processed %d failed workflow runs, found %d PRs to report back on" % (len(run_data), len(res)))

    return res
Example #13
0
def main():
    """the main function"""
    fancylogger.logToScreen(enable=True, stdout=True)
    fancylogger.setLogLevelInfo()

    options = {
        'github-user': ('Your github username to use', None, 'store', None, 'g'),
        'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'),
        'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'),
        'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'),
    }

    go = simple_option(options)
    log = go.log

    if not (go.options.all or go.options.closed_pr or go.options.orphans):
        raise EasyBuildError("Please tell me what to do?")

    if go.options.github_user is None:
        eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[])
        username = eb_go.options.github_user
        log.debug("Fetch github username from easybuild, found: %s", username)
    else:
        username = go.options.github_user

    if username is None:
        raise EasyBuildError("Could not find a github username")
    else:
        log.info("Using username = %s", username)

    token = fetch_github_token(username)

    gh = RestClient(GITHUB_API_URL, username=username, token=token)

    all_gists = []
    cur_page = 1
    while True:
        status, gists = gh.gists.get(per_page=100, page=cur_page)

        if status != HTTP_STATUS_OK:
            raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s",
                                 username, status, gists)
        if gists:
            all_gists.extend(gists)
            cur_page += 1
        else:
            break

    log.info("Found %s gists", len(all_gists))
    regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P<PR>[0-9]+))?\)?$")

    pr_cache = {}
    num_deleted = 0

    for gist in all_gists:
        if not gist["description"]:
            continue
        re_pr_num = regex.search(gist["description"])
        delete_gist = False

        if re_pr_num:
            log.debug("Found a Easybuild gist (id=%s)", gist["id"])
            pr_num = re_pr_num.group("PR")
            if go.options.all:
                delete_gist = True
            elif pr_num and go.options.closed_pr:
                log.debug("Found Easybuild test report for PR #%s", pr_num)

                if pr_num not in pr_cache:
                    status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get()
                    if status != HTTP_STATUS_OK:
                        raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s",
                                             pr_num, status, pr)
                    pr_cache[pr_num] = pr["state"]

                if pr_cache[pr_num] == "closed":
                    log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"])
                    delete_gist = True

            elif not pr_num and go.options.orphans:
                log.debug("Found Easybuild test report without PR (id=%s)", gist["id"])
                delete_gist = True

        if delete_gist:
            status, del_gist = gh.gists[gist["id"]].delete()

            if status != HTTP_DELETE_OK:
                raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s",
                                     gist["id"], status, del_gist)
            else:
                log.info("Delete gist with id=%s", gist["id"])
                num_deleted += 1

    log.info("Deleted %s gists", num_deleted)
Example #14
0
 def setUp(self):
     """Set up test."""
     super(RobotTest, self).setUp()
     self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT)
Example #15
0
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        build_log.EXPERIMENTAL = self.options.experimental

        # set strictness of run module
        if self.options.strict:
            run.strictness = self.options.strict

        # override current version of EasyBuild with version specified to --deprecated
        if self.options.deprecated:
            build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)

        # log to specified value of --unittest-file
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        # prepare for --list/--avail
        if any([
                self.options.avail_easyconfig_params,
                self.options.avail_easyconfig_templates,
                self.options.list_easyblocks,
                self.options.list_toolchains,
                self.options.avail_cfgfile_constants,
                self.options.avail_easyconfig_constants,
                self.options.avail_easyconfig_licenses,
                self.options.avail_repositories,
                self.options.show_default_moduleclasses,
                self.options.avail_modules_tools,
                self.options.avail_module_naming_schemes,
                self.options.show_default_configfiles,
        ]):
            build_easyconfig_constants_dict(
            )  # runs the easyconfig constants sanity check
            self._postprocess_list_avail()

        # fail early if required dependencies for functionality requiring using GitHub API are not available:
        if self.options.from_pr or self.options.upload_test_report:
            if not HAVE_GITHUB_API:
                raise EasyBuildError(
                    "Required support for using GitHub API is not available (see warnings)."
                )

        if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
            raise EasyBuildError(
                "Generating Lua module files requires Lmod as modules tool.")

        # make sure a GitHub token is available when it's required
        if self.options.upload_test_report:
            if not HAVE_KEYRING:
                raise EasyBuildError(
                    "Python 'keyring' module required for obtaining GitHub token is not available."
                )
            if self.options.github_user is None:
                raise EasyBuildError(
                    "No GitHub user name provided, required for fetching GitHub token."
                )
            token = fetch_github_token(self.options.github_user)
            if token is None:
                raise EasyBuildError(
                    "Failed to obtain required GitHub token for user '%s'",
                    self.options.github_user)

        self._postprocess_external_modules_metadata()

        self._postprocess_config()
 def setUp(self):
     """Set up test."""
     super(RobotTest, self).setUp()
     self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT)
Example #17
0
def fetch_travis_failed_builds(github_account, repository, owner, github_token):
    """Scan Travis test runs for failures, and return notification to be sent to PR if one is found"""

    if 'travispy' not in globals():
        error("travisy not available?!")

    travis = travispy.TravisPy.github_auth(github_token)

    print("Checking failed Travis builds for %s/%s (using '%s' GitHub account)" % (github_account, repository, owner))

    repo_slug = '%s/%s' % (github_account, repository)
    last_builds = travis.builds(slug=repo_slug, event_type='pull_request')

    done_prs = []

    res = []
    for build in last_builds:
        bid, pr = build.number, build.pull_request_number

        if pr in done_prs:
            print("(skipping test suite run for already processed PR #%s)" % pr)
            continue

        done_prs.append(pr)

        if build.successful:
            print("(skipping successful test suite run %s for PR %s)" % (bid, pr))

        else:
            build_url = os.path.join(TRAVIS_URL, repo_slug, 'builds', str(build.id))
            print("[id: %s] PR #%s - %s - %s" % (bid, pr, build.state, build_url))

            jobs = [(str(job_id), travis.jobs(ids=[job_id])[0]) for job_id in sorted(build.job_ids)]
            jobs_ok = [job.successful for (_, job) in jobs]

            pr_comment = "Travis test report: %d/%d runs failed - " % (jobs_ok.count(False), len(jobs))
            pr_comment += "see %s\n" % build_url
            check_msg = pr_comment.strip()

            jobs = [(job_id, job) for (job_id, job) in jobs if job.unsuccessful]
            print("Found %d unsuccessful jobs" % len(jobs))
            if jobs:

                # detect fluke failures in jobs, and restart them
                flukes = []
                for (job_id, job) in jobs:
                    if is_fluke(job.log.body):
                        flukes.append(job_id)

                if flukes:
                    boegel_gh_token = fetch_github_token('boegel')
                    if boegel_gh_token:
                        travis_boegel = travispy.TravisPy.github_auth(boegel_gh_token)
                        for (job_id, job) in zip(flukes, travis_boegel.jobs(ids=flukes)):
                            print("[id %s] PR #%s - fluke detected in job ID %s, restarting it!" % (bid, pr, job_id))
                            if job.restart():
                                print("Job ID %s restarted" % job_id)
                            else:
                                print("Failed to restart job ID %s!" % job_id)

                        # filter out fluke jobs, we shouldn't report these
                        jobs = [(job_id, job) for (job_id, job) in jobs if job_id not in flukes]
                    else:
                        print("Can't restart Travis jobs that failed due to flukes, no GitHub token found")

            print("Retained %d unsuccessful jobs after filtering out flukes" % len(jobs))
            if jobs:
                job_url = os.path.join(TRAVIS_URL, repo_slug, 'jobs', jobs[0][0])
                pr_comment += "\nOnly showing partial log for 1st failed test suite run %s;\n" % jobs[0][1].number
                pr_comment += "full log at %s\n" % job_url

                # try to filter log to just the stuff that matters
                retained_log_lines = jobs[0][1].log.body.split('\n')
                for idx, log_line in enumerate(retained_log_lines):
                    if repository == 'easybuild-easyconfigs':
                        if log_line.startswith('FAIL:') or log_line.startswith('ERROR:'):
                            retained_log_lines = retained_log_lines[idx:]
                            break
                    elif log_line.strip().endswith("$ python -O -m test.%s.suite" % repository.split('-')[-1]):
                        retained_log_lines = retained_log_lines[idx:]
                        break

                pr_comment += '```\n...\n'
                pr_comment += '\n'.join(retained_log_lines[-100:])
                pr_comment += '\n```\n'

                for (job_id, job) in jobs[1:]:
                    job_url = os.path.join(TRAVIS_URL, repo_slug, 'jobs', job_id)
                    pr_comment += "* %s - %s => %s\n" % (job.number, job.state, job_url)

                pr_comment += "\n*bleep, bloop, I'm just a bot (boegelbot v%s)*" % VERSION
                pr_comment += "Please talk to my owner `@%s` if you notice you me acting stupid)," % owner
                pr_comment += "or submit a pull request to https://github.com/boegel/boegelbot fix the problem."

                res.append((pr, pr_comment, check_msg))

            else:
                print("(no more failed jobs after filtering out flukes for id %s PR #%s)" % (bid, pr))

    print("Processed %d builds, found %d PRs with failed builds to report back on" % (len(last_builds), len(res)))

    return res
def main():
    """the main function"""
    fancylogger.logToScreen(enable=True, stdout=True)
    fancylogger.setLogLevelInfo()

    options = {
        'github-user': ('Your github username to use', None, 'store', None, 'g'),
        'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'),
        'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'),
        'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'),
    }

    go = simple_option(options)
    log = go.log

    if not (go.options.all or go.options.closed_pr or go.options.orphans):
        raise EasyBuildError("Please tell me what to do?")

    if go.options.github_user is None:
        eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[])
        username = eb_go.options.github_user
        log.debug("Fetch github username from easybuild, found: %s", username)
    else:
        username = go.options.github_user

    if username is None:
        raise EasyBuildError("Could not find a github username")
    else:
        log.info("Using username = %s", username)

    token = fetch_github_token(username)

    gh = RestClient(GITHUB_API_URL, username=username, token=token)
    # ToDo: add support for pagination
    status, gists = gh.gists.get(per_page=100)

    if status != HTTP_STATUS_OK:
        raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s",
                             username, status, gists)
    else:
        log.info("Found %s gists", len(gists))

    regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P<PR>[0-9]+))?\)?$")

    pr_cache = {}
    num_deleted = 0

    for gist in gists:
        if not gist["description"]:
            continue
        re_pr_num = regex.search(gist["description"])
        delete_gist = False

        if re_pr_num:
            log.debug("Found a Easybuild gist (id=%s)", gist["id"])
            pr_num = re_pr_num.group("PR")
            if go.options.all:
                delete_gist = True
            elif pr_num and go.options.closed_pr:
                log.debug("Found Easybuild test report for PR #%s", pr_num)

                if pr_num not in pr_cache:
                    status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get()
                    if status != HTTP_STATUS_OK:
                        raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s",
                                             pr_num, status, pr)
                    pr_cache[pr_num] = pr["state"]

                if pr_cache[pr_num] == "closed":
                    log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"])
                    delete_gist = True

            elif not pr_num and go.options.orphans:
                log.debug("Found Easybuild test report without PR (id=%s)", gist["id"])
                delete_gist = True

        if delete_gist:
            status, del_gist = gh.gists[gist["id"]].delete()

            if status != HTTP_DELETE_OK:
                raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s",
                                     gist["id"], status, del_gist)
            else:
                log.info("Delete gist with id=%s", gist["id"])
                num_deleted += 1

    log.info("Deleted %s gists", num_deleted)
Example #19
0
def main():

    opts = {
        'dry-run': ("Dry run, don't actually post/push/merge anything", None, 'store_true', False, 'x'),
        'force': ("Use force to execute the specified action", None, 'store_true', False, 'f'),
        'github-account': ("GitHub account where repository is located", None, 'store', 'hpcugent', 'a'),
        'github-user': ("GitHub user to use (for authenticated access)", None, 'store', 'boegel', 'u'),
        'repository': ("Repository to use", None, 'store', 'easybuild-easyconfigs', 'r'),
        # actions
        'comment': ("Post a comment in the pull request", None, 'store', None, 'C'),
        'merge': ("Merge the pull request", None, 'store_true', False, 'M'),
        'review': ("Review the pull request", None, 'store_true', False, 'R'),
        'test': ("Submit job to upload test report", None, 'store_or_None', None, 'T'),
    }

    actions = ['comment', 'merge', 'review', 'test']

    go = simple_option(go_dict=opts, descr="Script to print overview of pull requests for a GitHub repository")

    # determine which action should be taken
    selected_action = None
    for action in sorted(actions):
        action_value = getattr(go.options, action)
        if isinstance(action_value, bool):
            if action_value:
                selected_action = (action, action_value)
                break
        elif action_value is not None:
            selected_action = (action, action_value)
            break  # FIXME: support multiple actions, loop over them (e.g. -C :jok,lgtm -T)

    if selected_action is None:
        avail_actions = ', '.join(["%s (-%s)" % (a, a[0].upper()) for a in sorted(actions)])
        error("No action specified, pick one: %s" % avail_actions)
    else:
        info("Selected action: %s" % selected_action[0])

    # prepare using GitHub API
    global DRY_RUN
    DRY_RUN = go.options.dry_run
    force = go.options.force
    github_account = go.options.github_account
    github_user = go.options.github_user
    repository = go.options.repository

    github_token = fetch_github_token(github_user)
    github = RestClient(GITHUB_API_URL, username=github_user, token=github_token, user_agent='eb-pr-check')

    if len(go.args) == 1:
        pr = go.args[0]
    else:
        usage()

    print "Fetching PR information ",
    print "(using GitHub token for user '%s': %s)... " % (github_user, ('no', 'yes')[bool(github_token)]),
    sys.stdout.flush()
    pr_data = fetch_pr_data(github, github_account, repository, pr)
    print ''

    #print_raw_pr_info(pr_data)

    print_pr_summary(pr_data)

    if selected_action[0] == 'comment':
        comment(github, github_user, repository, pr_data, selected_action[1])
    elif selected_action[0] == 'merge':
        merge(github, github_user, github_account, repository, pr_data, force=force)
    elif selected_action[0] == 'review':
        review(pr_data)
    elif selected_action[0] == 'test':
        test(pr_data, selected_action[1])
    else:
        error("Handling action '%s' not implemented yet" % selected_action[0])