Exemplo n.º 1
0
def update_pr_comments(pr_number):
    pr = Issue.get(pr_number)
    comments_response = paginated_github_request(get_issues_base() +
                                                 '/%i/comments' % pr_number,
                                                 oauth_token=oauth_token)
    # TODO: after fixing #32, re-enable etags here: etag=self.comments_etag)
    if comments_response is None:
        return "Comments for PR %i are up-to-date" % pr_number
    else:
        pr.comments_json, pr.comments_etag = comments_response
        pr.cached_commenters = pr._compute_commenters()
        pr.cached_last_jenkins_outcome = None
        pr.last_jenkins_outcome  # force recomputation of Jenkins outcome
        pr.put()  # Write our modifications back to the database

        # Delete out-of-date comments from AmplabJenkins and SparkQA.
        jenkins_comment_to_preserve = pr.last_jenkins_comment
        sparkqa_token = app.config["SPARKQA_GITHUB_OAUTH_KEY"]
        amplabjenkins_token = app.config["AMPLAB_JENKINS_GITHUB_OAUTH_KEY"]
        sparkqa_start_comments = {}  # Map from build ID to build start comment
        build_start_regex = r"Test build #(\d+) has started"
        build_end_regex = r"Test build #(\d+) (has finished|timed out)"
        for comment in (pr.comments_json or []):
            author = comment["user"]["login"]
            # Delete all comments from AmplabJenkins unless they are the comments that should be
            # displayed on the Spark PR dashboard. If we do not know which comment to preserve, then
            # do not delete any comments from AmplabJenkins.
            if jenkins_comment_to_preserve \
                    and author == "AmplabJenkins" \
                    and comment["url"] != jenkins_comment_to_preserve["url"]:
                raw_github_request(comment["url"],
                                   oauth_token=amplabjenkins_token,
                                   method="DELETE")
            elif author == "SparkQA":
                # Only delete build start notification comments from SparkQA and only delete them
                # after we've seen the corresponding build finished message.
                start_regex_match = re.search(build_start_regex,
                                              comment["body"])
                if start_regex_match:
                    sparkqa_start_comments[start_regex_match.groups()
                                           [0]] = comment
                else:
                    end_regex_match = re.search(build_end_regex,
                                                comment["body"])
                    if end_regex_match:
                        start_comment = sparkqa_start_comments.get(
                            end_regex_match.groups()[0])
                        if start_comment:
                            raw_github_request(start_comment["url"],
                                               oauth_token=sparkqa_token,
                                               method="DELETE")
        return "Done updating comments for PR %i" % pr_number
Exemplo n.º 2
0
def update_pr(pr_number):
    logging.debug("Updating pull request %i" % pr_number)
    pr = Issue.get_or_create(pr_number)
    issue_response = raw_github_request(get_pulls_base() + '/%i' % pr_number,
                                        oauth_token=oauth_token, etag=pr.etag)
    if issue_response is None:
        logging.debug("PR %i hasn't changed since last visit; skipping" % pr_number)
        return "Done updating pull request %i (nothing changed)" % pr_number
    pr.pr_json = json.loads(issue_response.content)
    pr.etag = issue_response.headers["ETag"]
    pr.state = pr.pr_json['state']
    pr.user = pr.pr_json['user']['login']
    pr.updated_at = \
        parse_datetime(pr.pr_json['updated_at']).astimezone(tz.tzutc()).replace(tzinfo=None)

    for issue_number in pr.parsed_title['jiras']:
        try:
            link_issue_to_pr("%s-%s" % (app.config['JIRA_PROJECT'], issue_number), pr)
        except:
            logging.exception("Exception when linking to JIRA issue %s-%s" %
                              (app.config['JIRA_PROJECT'], issue_number))
        try:
            start_issue_progress("%s-%s" % (app.config['JIRA_PROJECT'], issue_number))
        except:
            logging.exception(
                "Exception when starting progress on JIRA issue %s-%s" %
                (app.config['JIRA_PROJECT'], issue_number))

    pr.put()  # Write our modifications back to the database

    subtasks = [".update_pr_comments", ".update_pr_review_comments", ".update_pr_files"]
    for task in subtasks:
        taskqueue.add(url=url_for(task, pr_number=pr_number), queue_name='fresh-prs')

    return "Done updating pull request %i" % pr_number
Exemplo n.º 3
0
def update_pr(pr_number):
    logging.debug("Updating pull request %i" % pr_number)
    pr = Issue.get_or_create(pr_number)
    issue_response = raw_github_request(PULLS_BASE + '/%i' % pr_number,
                                        oauth_token=oauth_token, etag=pr.etag)
    if issue_response is None:
        logging.debug("PR %i hasn't changed since last visit; skipping" % pr_number)
        return "Done updating pull request %i (nothing changed)" % pr_number
    pr.pr_json = json.loads(issue_response.content)
    pr.etag = issue_response.headers["ETag"]
    pr.state = pr.pr_json['state']
    pr.user = pr.pr_json['user']['login']
    pr.updated_at = \
        parse_datetime(pr.pr_json['updated_at']).astimezone(tz.tzutc()).replace(tzinfo=None)

    for issue_number in pr.parsed_title['jiras']:
        try:
            link_issue_to_pr("SPARK-%s" % issue_number, pr)
        except:
            logging.exception("Exception when linking to JIRA issue SPARK-%s" % issue_number)
        try:
            start_issue_progress("SPARK-%s" % issue_number)
        except:
            logging.exception(
                "Exception when starting progress on JIRA issue SPARK-%s" % issue_number)

    pr.put()  # Write our modifications back to the database

    subtasks = [".update_pr_comments", ".update_pr_review_comments", ".update_pr_files"]
    for task in subtasks:
        taskqueue.add(url=url_for(task, pr_number=pr_number), queue_name='fresh-prs')

    return "Done updating pull request %i" % pr_number
Exemplo n.º 4
0
 def fetch_and_process(url):
     logging.debug("Following url %s" % url)
     response = raw_github_request(url, oauth_token=oauth_token)
     prs = json.loads(response.content)
     now = datetime.utcnow()
     should_continue_loading = True
     update_time = last_update_time
     for pr in prs:
         updated_at = \
             parse_datetime(pr['updated_at']).astimezone(tz.tzutc()).replace(tzinfo=None)
         update_time = max(update_time, updated_at)
         if updated_at < last_update_time:
             should_continue_loading = False
             break
         is_fresh = (now - updated_at
                     ).total_seconds() < app.config['FRESHNESS_THRESHOLD']
         queue_name = ("fresh-prs" if is_fresh else "old-prs")
         taskqueue.add(url=url_for(".update_pr", pr_number=pr['number']),
                       queue_name=queue_name)
     if should_continue_loading:
         link_header = parse_link_header(response.headers.get('Link', ''))
         for link in link_header.links:
             if link.rel == 'next':
                 fetch_and_process(link.href)
     return update_time
Exemplo n.º 5
0
def update_pr_comments(pr_number):
    pr = Issue.get(pr_number)
    comments_response = paginated_github_request(get_issues_base() + '/%i/comments' % pr_number,
                                                 oauth_token=oauth_token)
    # TODO: after fixing #32, re-enable etags here: etag=self.comments_etag)
    if comments_response is None:
        return "Comments for PR %i are up-to-date" % pr_number
    else:
        pr.comments_json, pr.comments_etag = comments_response
        pr.cached_commenters = pr._compute_commenters()
        pr.cached_last_jenkins_outcome = None
        pr.last_jenkins_outcome  # force recomputation of Jenkins outcome
        pr.put()  # Write our modifications back to the database

        # Delete out-of-date comments from AmplabJenkins and SparkQA.
        jenkins_comment_to_preserve = pr.last_jenkins_comment
        sparkqa_token = app.config["SPARKQA_GITHUB_OAUTH_KEY"]
        amplabjenkins_token = app.config["AMPLAB_JENKINS_GITHUB_OAUTH_KEY"]
        sparkqa_start_comments = {}  # Map from build ID to build start comment
        build_start_regex = r"Test build #(\d+) has started"
        build_end_regex = r"Test build #(\d+) (has finished|timed out)"
        for comment in (pr.comments_json or []):
            author = comment["user"]["login"]
            # Delete all comments from AmplabJenkins unless they are the comments that should be
            # displayed on the Spark PR dashboard. If we do not know which comment to preserve, then
            # do not delete any comments from AmplabJenkins.
            if jenkins_comment_to_preserve \
                    and author == "AmplabJenkins" \
                    and comment["url"] != jenkins_comment_to_preserve["url"]:
                raw_github_request(comment["url"], oauth_token=amplabjenkins_token, method="DELETE")
            elif author == "SparkQA":
                # Only delete build start notification comments from SparkQA and only delete them
                # after we've seen the corresponding build finished message.
                start_regex_match = re.search(build_start_regex, comment["body"])
                if start_regex_match:
                    sparkqa_start_comments[start_regex_match.groups()[0]] = comment
                else:
                    end_regex_match = re.search(build_end_regex, comment["body"])
                    if end_regex_match:
                        start_comment = sparkqa_start_comments.get(end_regex_match.groups()[0])
                        if start_comment:
                            raw_github_request(start_comment["url"], oauth_token=sparkqa_token,
                                               method="DELETE")
        return "Done updating comments for PR %i" % pr_number
Exemplo n.º 6
0
 def fetch_and_process(url):
     logging.debug("Following url %s" % url)
     response = raw_github_request(url, oauth_token=app.config["GITHUB_OAUTH_KEY"])
     link_header = parse_link_header(response.headers.get("Link", ""))
     prs = json.loads(response.content)
     now = datetime.utcnow()
     for pr in prs:
         updated_at = parse_datetime(pr["updated_at"]).astimezone(tz.tzutc()).replace(tzinfo=None)
         is_fresh = (now - updated_at).total_seconds() < app.config["FRESHNESS_THRESHOLD"]
         queue_name = "fresh-prs" if is_fresh else "old-prs"
         taskqueue.add(url="/tasks/update-github-pr/%i" % pr["number"], queue_name=queue_name)
     for link in link_header.links:
         if link.rel == "next":
             fetch_and_process(link.href)
Exemplo n.º 7
0
 def fetch_and_process(url):
     logging.debug("Following url %s" % url)
     response = raw_github_request(url, oauth_token=app.config['GITHUB_OAUTH_KEY'])
     link_header = parse_link_header(response.headers.get('Link', ''))
     prs = json.loads(response.content)
     now = datetime.utcnow()
     for pr in prs:
         updated_at = \
             parse_datetime(pr['updated_at']).astimezone(tz.tzutc()).replace(tzinfo=None)
         is_fresh = (now - updated_at).total_seconds() < app.config['FRESHNESS_THRESHOLD']
         queue_name = ("fresh-prs" if is_fresh else "old-prs")
         taskqueue.add(url=url_for(".update_pr", pr_number=pr['number']), queue_name=queue_name)
     for link in link_header.links:
         if link.rel == 'next':
             fetch_and_process(link.href)
Exemplo n.º 8
0
 def fetch_and_process(url):
     logging.debug("Following url %s" % url)
     response = raw_github_request(url, oauth_token=oauth_token)
     prs = json.loads(response.content)
     now = datetime.utcnow()
     should_continue_loading = True
     update_time = last_update_time
     for pr in prs:
         updated_at = \
             parse_datetime(pr['updated_at']).astimezone(tz.tzutc()).replace(tzinfo=None)
         update_time = max(update_time, updated_at)
         if updated_at < last_update_time:
             should_continue_loading = False
             break
         is_fresh = (now - updated_at).total_seconds() < app.config['FRESHNESS_THRESHOLD']
         queue_name = ("fresh-prs" if is_fresh else "old-prs")
         taskqueue.add(url=url_for(".update_pr", pr_number=pr['number']), queue_name=queue_name)
     if should_continue_loading:
         link_header = parse_link_header(response.headers.get('Link', ''))
         for link in link_header.links:
             if link.rel == 'next':
                 fetch_and_process(link.href)
     return update_time
Exemplo n.º 9
0
def backfill_prs():
    """
    This method attempts to update every PR ever opened against the repository.
    As a result, this method should only be invoked by admins when trying to bootstrap a new
    deployment of the PR board.
    """
    # Determine the number of PRs:
    url = get_pulls_base() + "?sort=created&state=all&direction=desc"
    response = raw_github_request(url, oauth_token=oauth_token)
    latest_prs = json.loads(response.content)
    latest_pr_number = int(latest_prs[0]['number'])
    queue = taskqueue.Queue('old-prs')
    update_tasks = []
    for num in reversed(xrange(1, latest_pr_number + 1)):
        update_tasks.append(taskqueue.Task(url=url_for(".update_pr", pr_number=num)))
    # Can only enqueue up to 100 tasks per API call
    async_call_results = []
    for group_of_tasks in chunked(update_tasks, 100):
        async_call_results.append(queue.add_async(group_of_tasks))
    # Block until the async calls are finished:
    for r in async_call_results:
        r.get_result()
    return "Enqueued tasks to backfill %i PRs" % latest_pr_number
Exemplo n.º 10
0
def backfill_prs():
    """
    This method attempts to update every PR ever opened against the repository.
    As a result, this method should only be invoked by admins when trying to bootstrap a new
    deployment of the PR board.
    """
    # Determine the number of PRs:
    url = get_pulls_base() + "?sort=created&state=all&direction=desc"
    response = raw_github_request(url, oauth_token=oauth_token)
    latest_prs = json.loads(response.content)
    latest_pr_number = int(latest_prs[0]['number'])
    queue = taskqueue.Queue('old-prs')
    update_tasks = []
    for num in reversed(xrange(1, latest_pr_number + 1)):
        update_tasks.append(taskqueue.Task(url=url_for(".update_pr", pr_number=num)))
    # Can only enqueue up to 100 tasks per API call
    async_call_results = []
    for group_of_tasks in chunked(update_tasks, 100):
        async_call_results.append(queue.add_async(group_of_tasks))
    # Block until the async calls are finished:
    for r in async_call_results:
        r.get_result()
    return "Enqueued tasks to backfill %i PRs" % latest_pr_number