def test_nonmatching_captures(self):
ptn = u"(re).*(ger)"
email = u"tony@tiremove_thisger.net"
results = rure.search(ptn, email)
stdlib_results = re.search(ptn, email)
self.assertIsNotNone(results)
self.assertEqual(len(results.groups()), len(stdlib_results.groups()))
ptn = u"(re)|(ger)"
results = rure.search(ptn, email)
stdlib_results = re.search(ptn, email)
self.assertIsNotNone(results)
self.assertEqual(len(results.groups()), len(stdlib_results.groups()))
self.assertEqual(results.group(0), stdlib_results.group(0))
def fix_unbalanced_quotes(vba_code):
"""
Fix lines with missing double quotes.
"""
# Fix invalid string assignments.
uni_vba_code = None
try:
uni_vba_code = vba_code.decode("utf-8")
except UnicodeDecodeError:
# Punt.
return vba_code
if (re2.search(u"(\w+)\s+=\s+\"\r?\n", uni_vba_code) is not None):
vba_code = re.sub(r"(\w+)\s+=\s+\"\r?\n", r'\1 = ""\n', vba_code)
if (re2.search(u"(\w+\s+=\s+\")(:[^\"]+)\r?\n", uni_vba_code) is not None):
vba_code = re.sub(r"(\w+\s+=\s+\")(:[^\"]+)\r?\n", r'\1"\2\n', vba_code)
if (re2.search(u"^\"[^=]*([=>])\s*\"\s+[Tt][Hh][Ee][Nn]", uni_vba_code) is not None):
vba_code = re.sub(r"^\"[^=]*([=>])\s*\"\s+[Tt][Hh][Ee][Nn]", r'\1 "" Then', vba_code)
# Fix ambiguous EOL comment lines like ".foo '' A comment". "''" could be parsed as
# an argument to .foo or as an EOL comment. Here we change things like ".foo '' A comment"
# to ".foo ' A comment" so it is not ambiguous (parse as comment).
vba_code += "\n"
vba_code = re.sub(r"'('[^'^\"]+\n)", r"\1", vba_code, re.DOTALL)
# More ambiguous EOL comments. Something like "a = 12 : 'stuff 'more stuff" could have
# 'stuff ' potentially parsed as a string. Just wipe out the comments in this case
# (ex. "a = 12 : 'stuff 'more stuff" => "a = 12 :").
vba_code = re.sub(r"(\n[^'^\n]+)'[^'^\"^\n]+'[^'^\"^\n]+\n", r"\1\n", vba_code, re.DOTALL)
# Fix Execute statements with no space between the execute and the argument.
vba_code = re.sub(r"\n\s*([Ee][Xx][Ee][Cc][Uu][Tt][Ee])\"", r'\nExecute "', vba_code)
# See if we have lines with unbalanced double quotes.
r = ""
for line in vba_code.split("\n"):
if ('"' not in line):
r += line + "\n"
continue
num_quotes = line.count('"')
if ((num_quotes % 2) != 0):
last_quote = line.rindex('"')
line = line[:last_quote] + '"' + line[last_quote:]
r += line + "\n"
# Return the balanced code.
return r
def is_mixed_wide_ascii_str(the_str):
"""
Test a string to see if it is a mix of wide and ASCII chars.
"""
uni_str = None
try:
uni_str = the_str.decode("utf-8")
except UnicodeDecodeError:
# Punt.
return False
extended_asc_pat = b"[\x80-\xff]"
if (re.search(extended_asc_pat, uni_str) is not None):
return True
return False
def fix_unhandled_array_assigns(vba_code):
"""
Currently things like 'foo(1, 2, 3) = 1' are not handled.
Comment them out.
"""
uni_vba_code = None
try:
uni_vba_code = vba_code.decode("utf-8")
except UnicodeDecodeError:
# Punt.
return vba_code
pat = "\n(\s*\w+\((?:\w+\s*,\s*){2,}\w+\)\s*=)"
if (re2.search(unicode(pat), uni_vba_code) is not None):
vba_code = re.sub(pat, r"\n' UNHANDLED ARRAY ASSIGNMENT \1", vba_code)
return vba_code
def is_mixed_wide_ascii_str(the_str):
"""Test a string to see if a string is a mix of wide and ASCII chars.
@param the_str (str) The string to check.
@return (boolean) True if the string is a mized string, False if
not.
"""
uni_str = None
try:
uni_str = the_str.decode("utf-8")
except UnicodeDecodeError:
# Punt.
return False
extended_asc_pat = b"[\x80-\xff]"
if (re.search(extended_asc_pat, uni_str) is not None):
return True
return False
async def mergeable(
api: PRAPI,
config: Union[config.V1, pydantic.ValidationError, toml.TomlDecodeError],
config_str: str,
config_path: str,
pull_request: PullRequest,
branch_protection: Optional[BranchProtectionRule],
review_requests: List[PRReviewRequest],
reviews: List[PRReview],
contexts: List[StatusContext],
check_runs: List[CheckRun],
commits: List[Commit],
valid_signature: bool,
valid_merge_methods: List[MergeMethod],
repository: RepoInfo,
merging: bool,
is_active_merge: bool,
skippable_check_timeout: int,
api_call_retries_remaining: int,
api_call_errors: Sequence[APICallRetry],
subscription: Optional[Subscription],
app_id: Optional[str] = None,
) -> None:
# TODO(chdsbd): Use structlog bind_contextvars to automatically set useful context (install id, repo, pr number).
log = logger.bind(number=pull_request.number, url=pull_request.url)
# we set is_active_merge when the PR is being merged from the merge queue.
# We don't want to clobber any statuses set by that system, so we take no
# action. If the PR becomes ineligible for merging that logic will handle
# it.
# rebase_fast_forward isn't determined via the GitHub UI and is always
# available.
valid_merge_methods = [
*valid_merge_methods, MergeMethod.rebase_fast_forward
]
async def set_status(msg: str,
markdown_content: Optional[str] = None) -> None:
# don't clobber statuses set via merge loop.
if is_active_merge:
return
await api.set_status(
msg,
latest_commit_sha=pull_request.latest_sha,
markdown_content=markdown_content,
)
if not isinstance(config, V1):
log.warning("problem fetching config")
await set_status(
'⚠️ Invalid configuration (Click "Details" for more info.)',
markdown_content=get_markdown_for_config(config,
config_str=config_str,
git_path=config_path),
)
await api.dequeue()
return
if api_call_retries_remaining == 0:
log.warning("timeout reached for api calls to GitHub")
if api_call_errors:
first_error = api_call_errors[0]
await set_status(
f"⚠️ problem contacting GitHub API with method {first_error.api_name!r}",
markdown_content=get_markdown_for_api_call_errors(
errors=api_call_errors),
)
else:
await set_status("⚠️ problem contacting GitHub API")
return
# if we have an app_id in the config then we only want to work on this repo
# if our app_id from the environment matches the configuration.
if config.app_id is not None and config.app_id != app_id:
log.info("missing required app_id")
await api.dequeue()
return
if branch_protection is None:
await cfg_err(
api,
pull_request,
f"missing branch protection for baseRef: {pull_request.baseRefName!r}",
)
return
merge_method = get_merge_method(
cfg_merge_method=config.merge.method,
valid_merge_methods=valid_merge_methods,
log=log,
labels=pull_request.labels,
)
if (branch_protection.requiresCommitSignatures
and merge_method == MergeMethod.rebase):
await cfg_err(
api,
pull_request,
'"Require signed commits" branch protection is only supported with "squash" or "merge" commits. Rebase is not supported by GitHub.',
)
return
if merge_method not in valid_merge_methods:
valid_merge_methods_str = [
method.value for method in valid_merge_methods
]
await cfg_err(
api,
pull_request,
f"configured merge.method {merge_method.value!r} is invalid. Valid methods for repo are {valid_merge_methods_str!r}",
)
return
if (not config.merge.do_not_merge and branch_protection.restrictsPushes and
missing_push_allowance(branch_protection.pushAllowances.nodes)):
await cfg_err(
api,
pull_request,
"push restriction branch protection setting is missing push allowance for Kodiak",
markdown_content=get_markdown_for_push_allowance_error(
branch_name=pull_request.baseRefName),
)
return
# we keep the configuration errors before the rest of the application logic
# so configuration issues are surfaced as early as possible.
if config.disable_bot_label in pull_request.labels:
await api.dequeue()
await api.set_status(
f"🚨 kodiak disabled by disable_bot_label ({config.disable_bot_label}). Remove label to re-enable Kodiak.",
latest_commit_sha=pull_request.latest_sha,
)
return
if (app_config.SUBSCRIPTIONS_ENABLED and repository.is_private
and subscription is not None
and subscription.subscription_blocker is not None):
# We only count private repositories in our usage calculations. A user
# has an active subscription if a subscription exists in Redis and has
# an empty subscription_blocker.
#
# We also ignore missing subscriptions. The web api will set
# subscription blockers if usage exceeds limits.
status_message = get_paywall_status_for_blocker(
pull_request, subscription.subscription_blocker, log)
if status_message is not None:
await set_status(
f"💳 subscription: {status_message}",
markdown_content=get_markdown_for_paywall(),
)
return
pull_request_labels = set(pull_request.labels)
config_automerge_labels = ({config.merge.automerge_label} if isinstance(
config.merge.automerge_label, str) else set(
config.merge.automerge_label))
pull_request_automerge_labels = config_automerge_labels.intersection(
pull_request_labels)
has_automerge_label = len(pull_request_automerge_labels) > 0
should_dependency_automerge = (
pull_request.author.login
in config.merge.automerge_dependencies.usernames
and dep_version_from_title(pull_request.title)
in config.merge.automerge_dependencies.versions)
# we should trigger mergeability checks whenever we encounter UNKNOWN.
#
# I don't foresee conflicts with checking configuration errors,
# `config.disable_bot_label`, and the paywall before this code.
#
# Previously we had an issue where this code wasn't being entered because
# `merge.blocking_title_regex` was checked first. Which caused
# `update.always` to not operate.
if (pull_request.mergeable == MergeableState.UNKNOWN
and pull_request.state == PullRequestState.OPEN):
# we need to trigger a test commit to fix this. We do that by calling
# GET on the pull request endpoint.
await api.trigger_test_commit()
# queue the PR for evaluation again in case GitHub doesn't send another
# webhook for the commit test.
await api.requeue()
# we don't want to abort the merge if we encounter this status check.
# Just keep polling!
if merging:
raise PollForever
return
is_draft_pull_request = (pull_request.isDraft
or pull_request.mergeStateStatus
== MergeStateStatus.DRAFT)
if (pull_request.author.login in config.approve.auto_approve_usernames
and pull_request.state == PullRequestState.OPEN
and not is_draft_pull_request):
# if the PR was created by an approve author and we have not previously
# given an approval, approve the PR.
sorted_reviews = sorted(reviews, key=lambda x: x.createdAt)
kodiak_reviews = [
review for review in sorted_reviews
if review.author.login == KODIAK_LOGIN
]
status = review_status(kodiak_reviews)
if status != PRReviewState.APPROVED:
await api.approve_pull_request()
else:
log.info("approval already exists, not adding another")
need_branch_update = (branch_protection.requiresStrictStatusChecks
and pull_request.mergeStateStatus
== MergeStateStatus.BEHIND)
update_always = config.update.always and (
has_automerge_label or not config.update.require_automerge_label)
has_autoupdate_label = config.update.autoupdate_label in pull_request_labels
auto_update_enabled = update_always or has_autoupdate_label
# Dequeue pull request if out-of-date and author in
# `update.ignored_usernames`. We cannot update or merge it.
#
# If `update.autoupdate_label` is applied to the pull request, bypass
# `update.ignored_usernames` and let the pull request update.
if need_branch_update and not has_autoupdate_label:
if pull_request.author.login in config.update.blacklist_usernames:
await set_status(
f"🛑 updates blocked by update.blacklist_usernames: {config.update.blacklist_usernames!r}",
markdown_content=
"Apply the `update.autoupdate_label` label to enable updates for this pull request.",
)
await api.dequeue()
return
if pull_request.author.login in config.update.ignored_usernames:
await set_status(
f"🛑 updates blocked by update.ignored_usernames: {config.update.ignored_usernames!r}",
markdown_content=
"Apply the `update.autoupdate_label` label to enable updates for this pull request.",
)
await api.dequeue()
return
if need_branch_update and not merging and auto_update_enabled:
await set_status(
"🔄 updating branch",
markdown_content=
"branch updated because `update.always = true` is configured.",
)
await api.update_branch()
return
if (config.merge.require_automerge_label and not has_automerge_label
and not should_dependency_automerge):
await block_merge(
api,
pull_request,
f"missing automerge_label: {config.merge.automerge_label!r}",
)
return
# We want users to get notified a merge conflict even if the PR matches a
# WIP title via merge.blacklist_title_regex.
if (pull_request.mergeStateStatus == MergeStateStatus.DIRTY
or pull_request.mergeable == MergeableState.CONFLICTING
) and pull_request.state == PullRequestState.OPEN:
await block_merge(api, pull_request, "merge conflict")
# remove label if configured and send message
if (config.merge.notify_on_conflict
and config.merge.require_automerge_label
and has_automerge_label):
automerge_label = config.merge.automerge_label
await asyncio.gather(*[
api.remove_label(label)
for label in pull_request_automerge_labels
])
body = textwrap.dedent(f"""
This PR currently has a merge conflict. Please resolve this and then re-add the `{automerge_label}` label.
""")
await api.create_comment(body)
return
blacklist_labels = set(config.merge.blacklist_labels) & set(
pull_request.labels)
blocking_labels = set(config.merge.blocking_labels) & set(
pull_request.labels)
if blacklist_labels:
await block_merge(api, pull_request,
f"has blacklist_labels: {blacklist_labels!r}")
return
if blocking_labels:
await block_merge(api, pull_request,
f"has merge.blocking_labels: {blocking_labels!r}")
return
title_blocker = get_blocking_title_regex(config)
if (title_blocker.pattern and re.search(title_blocker.pattern,
pull_request.title) is not None):
await block_merge(
api,
pull_request,
f"title matches {title_blocker.config_key}: {title_blocker.pattern!r}",
)
return
if is_draft_pull_request:
await block_merge(api, pull_request, "pull request is in draft state")
return
if config.merge.block_on_reviews_requested and review_requests:
names = [r.name for r in review_requests]
await block_merge(api, pull_request, f"reviews requested: {names!r}")
return
if pull_request.state == PullRequestState.MERGED:
log.info(
"pull request merged. config.merge.delete_branch_on_merge=%r",
config.merge.delete_branch_on_merge,
)
await api.dequeue()
if (not config.merge.delete_branch_on_merge
or pull_request.isCrossRepository
or repository.delete_branch_on_merge):
return
pr_count = await api.pull_requests_for_ref(ref=pull_request.headRefName
)
# if we couldn't access the dependent PR count or we have dependent PRs
# we will abort deleting this branch.
if pr_count is None or pr_count > 0:
log.info("skipping branch deletion because of dependent PRs",
pr_count=pr_count)
return
await api.delete_branch(branch_name=pull_request.headRefName)
return
if pull_request.state == PullRequestState.CLOSED:
await api.dequeue()
return
if pull_request.mergeStateStatus == MergeStateStatus.UNSTABLE:
# TODO(chdsbd): This status means that the pr is mergeable but has failing
# status checks. we may want to handle this via config
pass
wait_for_checks = False
if pull_request.mergeStateStatus in (
MergeStateStatus.BLOCKED,
MergeStateStatus.BEHIND,
):
# figure out why we can't merge. There isn't a way to get this simply from the Github API. We need to find out ourselves.
#
# I think it's possible to find out blockers from branch protection issues
# https://developer.github.com/v4/object/branchprotectionrule/?#fields
#
# - missing reviews
# - blocking reviews
# - missing required status checks
# - failing required status checks
# - branch not up to date (should be handled before this)
# - missing required signature
if (branch_protection.requiresApprovingReviews
and branch_protection.requiredApprovingReviewCount):
reviews_by_author: MutableMapping[
str, List[PRReview]] = defaultdict(list)
for review in sorted(reviews, key=lambda x: x.createdAt):
if review.author.permission not in {
Permission.ADMIN, Permission.WRITE
}:
continue
reviews_by_author[review.author.login].append(review)
successful_reviews = 0
for author_name, review_list in reviews_by_author.items():
review_state = review_status(review_list)
# blocking review
if review_state == PRReviewState.CHANGES_REQUESTED:
await block_merge(api, pull_request,
f"changes requested by {author_name!r}")
return
# successful review
if review_state == PRReviewState.APPROVED:
successful_reviews += 1
# missing required review count
if successful_reviews < branch_protection.requiredApprovingReviewCount:
await block_merge(
api,
pull_request,
f"missing required reviews, have {successful_reviews!r}/{branch_protection.requiredApprovingReviewCount!r}",
)
return
if pull_request.reviewDecision == PullRequestReviewDecision.REVIEW_REQUIRED:
await block_merge(api, pull_request, "missing required reviews")
return
required: Set[str] = set()
passing: Set[str] = set()
if branch_protection.requiresStatusChecks:
skippable_contexts: List[str] = []
failing_contexts: List[str] = []
pending_contexts: List[str] = []
passing_contexts: List[str] = []
required = set(branch_protection.requiredStatusCheckContexts)
for status_context in contexts:
# handle dont_wait_on_status_checks. We want to consider a
# status_check failed if it is incomplete and in the
# configuration.
if (status_context.context
in config.merge.dont_wait_on_status_checks
and status_context.state
in (StatusState.EXPECTED, StatusState.PENDING)):
skippable_contexts.append(status_context.context)
continue
if status_context.state in (StatusState.ERROR,
StatusState.FAILURE):
failing_contexts.append(status_context.context)
elif status_context.state in (
StatusState.EXPECTED,
StatusState.PENDING,
):
pending_contexts.append(status_context.context)
else:
assert status_context.state == StatusState.SUCCESS
passing_contexts.append(status_context.context)
for check_run in check_runs:
if (check_run.name in config.merge.dont_wait_on_status_checks
and check_run.conclusion
in (None, CheckConclusionState.NEUTRAL)):
skippable_contexts.append(check_run.name)
continue
if check_run.conclusion is None:
continue
if check_run.conclusion == CheckConclusionState.SUCCESS:
passing_contexts.append(check_run.name)
if check_run.conclusion in (
CheckConclusionState.ACTION_REQUIRED,
CheckConclusionState.FAILURE,
CheckConclusionState.TIMED_OUT,
CheckConclusionState.CANCELLED,
CheckConclusionState.SKIPPED,
CheckConclusionState.STALE,
):
failing_contexts.append(check_run.name)
passing = set(passing_contexts)
failing = set(failing_contexts)
# we have failing statuses that are required
failing_required_status_checks = failing & required
# GitHub has undocumented logic for travis-ci checks in GitHub
# branch protection rules. GitHub compresses
# "continuous-integration/travis-ci/{pr,push}" to
# "continuous-integration/travis-ci". There is only special handling
# for these specific checks.
if "continuous-integration/travis-ci" in required:
if "continuous-integration/travis-ci/pr" in failing:
failing_required_status_checks.add(
"continuous-integration/travis-ci/pr")
if "continuous-integration/travis-ci/push" in failing:
failing_required_status_checks.add(
"continuous-integration/travis-ci/push")
# either check can satisfy continuous-integration/travis-ci, but
# if either fails they'll also block the merge.
if ("continuous-integration/travis-ci/pr" in passing
or "continuous-integration/travis-ci/push" in passing):
required.remove("continuous-integration/travis-ci")
if failing_required_status_checks:
# NOTE(chdsbd): We need to skip this PR because it would block
# the merge queue. We may be able to bump it to the back of the
# queue, but it's easier just to remove it all together. There
# is a similar question for the review counting.
await block_merge(
api,
pull_request,
f"failing required status checks: {failing_required_status_checks!r}",
)
return
if skippable_contexts:
if merging:
if skippable_check_timeout > 0:
await set_status(
f"⛴ merging PR (waiting a bit for dont_wait_on_status_checks: {skippable_contexts!r})"
)
raise RetryForSkippableChecks
log.warning(
"timeout reached waiting for dont_wait_on_status_checks",
skippable_contexts=skippable_contexts,
)
await set_status(
f"⚠️ timeout reached for dont_wait_on_status_checks: {skippable_contexts!r}"
)
await set_status(
f"🛑 not waiting for dont_wait_on_status_checks: {skippable_contexts!r}"
)
return
missing_required_status_checks = required - passing
wait_for_checks = bool(branch_protection.requiresStatusChecks
and missing_required_status_checks)
if config.merge.update_branch_immediately and need_branch_update:
await set_status(
"🔄 updating branch",
markdown_content=
"branch updated because `merge.update_branch_immediately = true` is configured.",
)
await api.update_branch()
if merging:
raise PollForever
return
if merging:
# prioritize branch updates over waiting for status checks to complete
if config.merge.optimistic_updates:
if need_branch_update:
await set_status("⛴ merging PR (updating branch)")
await api.update_branch()
raise PollForever
if wait_for_checks:
await set_status(
f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})"
)
raise PollForever
# almost the same as the pervious case, but we prioritize status checks
# over branch updates.
else:
if wait_for_checks:
await set_status(
f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})"
)
raise PollForever
if need_branch_update:
await set_status("⛴ merging PR (updating branch)")
await api.update_branch()
raise PollForever
# if we reach this point and we don't need to wait for checks or update a branch we've failed to calculate why the PR is blocked. This should _not_ happen normally.
if not (wait_for_checks or need_branch_update):
await block_merge(api, pull_request,
"Merging blocked by GitHub requirements")
log.warning("merge blocked for unknown reason")
return
ready_to_merge = not (wait_for_checks or need_branch_update)
if config.merge.do_not_merge:
if wait_for_checks:
await set_status(
f"⌛️ waiting for required status checks: {missing_required_status_checks!r}"
)
elif need_branch_update:
await set_status(
"⚠️ need branch update (suggestion: use merge.update_branch_immediately with merge.do_not_merge)",
markdown_content="""\
When `merge.do_not_merge = true` is configured `merge.update_branch_immediately = true` \
is recommended so Kodiak can automatically update branches.
By default, Kodiak is efficient and only update branches when merging a PR, but \
when `merge.do_not_merge` is enabled, Kodiak never has that opportunity to \
update a branch during merge. `merge.update_branch_immediately = true` will \
trigger Kodiak to update branches whenever a PR is outdated and not failing any \
branch protection requirements.
""",
)
else:
await set_status("✅ okay to merge")
log.info(
"eligible to merge, stopping because config.merge.do_not_merge is enabled."
)
return
# okay to merge if we reach this point.
if (config.merge.prioritize_ready_to_merge and ready_to_merge) or merging:
merge_args = get_merge_body(config,
merge_method,
pull_request,
commits=commits)
await set_status("⛴ attempting to merge PR (merging)")
try:
# Use the Git Refs API to rebase merge.
#
# This preserves the rebased commits and their hashes. Using the
# GitHub Pull Request API to rebase merge rewrites the rebased
# commits, so the commit hashes change.
#
# For build systems that depend on commit hashes instead of tree
# hashes, it's desirable to not rewrite commits.
if merge_args.merge_method is MergeMethod.rebase_fast_forward:
await api.update_ref(ref=pull_request.baseRefName,
sha=pull_request.latest_sha)
else:
await api.merge(
merge_method=merge_args.merge_method,
commit_title=merge_args.commit_title,
commit_message=merge_args.commit_message,
)
# if we encounter an internal server error (status code 500), it is
# _not_ safe to retry. Instead we mark the pull request as unmergable
# and require a user to re-enable Kodiak on the pull request.
except GitHubApiInternalServerError:
logger.warning(
"kodiak encountered GitHub API error merging pull request",
exc_info=True,
)
# We add the disable_bot_label to disable Kodiak from taking any
# action to update, approve, comment, label, or merge.
disable_bot_label = config.disable_bot_label
await api.add_label(disable_bot_label)
await block_merge(api, pull_request,
"Cannot merge due to GitHub API failure.")
body = messages.format(
textwrap.dedent(f"""
This PR could not be merged because the GitHub API returned an internal server error. To enable Kodiak on this pull request please remove the `{disable_bot_label}` label.
When the GitHub API returns an internal server error (HTTP status code 500), it is not safe for Kodiak to retry merging.
For more information please see https://kodiakhq.com/docs/troubleshooting#merge-errors
"""))
await api.create_comment(body)
else:
await set_status("merge complete 🎉")
else:
priority_merge = config.merge.priority_merge_label in pull_request.labels
position_in_queue = await api.queue_for_merge(first=priority_merge)
if position_in_queue is None:
# this case should be rare/impossible.
log.warning("couldn't find position for enqueued PR")
return
ordinal_position = inflection.ordinalize(position_in_queue + 1)
if not is_active_merge:
await set_status(
f"📦 enqueued for merge (position={ordinal_position})")
else:
log.info(
"not setting status message for enqueued job because is_active_merge=True"
)
return
async def mergeable(
api: PRAPI,
config: Union[config.V1, pydantic.ValidationError, toml.TomlDecodeError],
config_str: str,
config_path: str,
pull_request: PullRequest,
branch_protection: Optional[BranchProtectionRule],
review_requests: List[PRReviewRequest],
reviews: List[PRReview],
contexts: List[StatusContext],
check_runs: List[CheckRun],
valid_signature: bool,
valid_merge_methods: List[MergeMethod],
merging: bool,
is_active_merge: bool,
skippable_check_timeout: int,
api_call_retry_timeout: int,
api_call_retry_method_name: Optional[str],
app_id: Optional[str] = None,
) -> None:
log = logger.bind(
config=config,
pull_request=pull_request,
branch_protection=branch_protection,
review_requests=review_requests,
reviews=reviews,
contexts=contexts,
valid_signature=valid_signature,
valid_merge_methods=valid_merge_methods,
)
# we set is_active_merge when the PR is being merged from the merge queue.
# We don't want to clobber any statuses set by that system, so we take no
# action. If the PR becomes ineligible for merging that logic will handle
# it.
async def set_status(msg: str,
markdown_content: Optional[str] = None) -> None:
# don't clobber statuses set via merge loop.
if is_active_merge:
return
await api.set_status(
msg,
latest_commit_sha=pull_request.latest_sha,
markdown_content=markdown_content,
)
if not isinstance(config, V1):
log.warning("problem fetching config")
await set_status(
'⚠️ Invalid configuration (Click "Details" for more info.)',
markdown_content=get_markdown_for_config(config,
config_str=config_str,
git_path=config_path),
)
await api.dequeue()
return
if api_call_retry_timeout == 0:
log.warning("timeout reached for api calls to GitHub")
if api_call_retry_method_name is not None:
await set_status(
f"⚠️ problem contacting GitHub API with method {api_call_retry_method_name!r}"
)
else:
await set_status("⚠️ problem contacting GitHub API")
return
# if we have an app_id in the config then we only want to work on this repo
# if our app_id from the environment matches the configuration.
if config.app_id is not None and config.app_id != app_id:
log.info("missing required app_id")
await api.dequeue()
return
if branch_protection is None:
await cfg_err(
api,
pull_request,
f"missing branch protection for baseRef: {pull_request.baseRefName!r}",
)
return
if branch_protection.requiresCommitSignatures and config.merge.method in (
MergeMethod.rebase,
MergeMethod.squash,
):
await cfg_err(
api,
pull_request,
'"Require signed commits" branch protection is only supported with merge commits. Squash and rebase are not supported by GitHub.',
)
return
if config.merge.method not in valid_merge_methods:
valid_merge_methods_str = [
method.value for method in valid_merge_methods
]
await cfg_err(
api,
pull_request,
f"configured merge.method {config.merge.method.value!r} is invalid. Valid methods for repo are {valid_merge_methods_str!r}",
)
return
# we keep the configuration errors before the rest of the application logic
# so configuration issues are surfaced as early as possible.
if (pull_request.author.login in config.approve.auto_approve_usernames
and pull_request.state == PullRequestState.OPEN
and pull_request.mergeStateStatus != MergeStateStatus.DRAFT):
# if the PR was created by an approve author and we have not previously
# given an approval, approve the PR.
sorted_reviews = sorted(reviews, key=lambda x: x.createdAt)
kodiak_reviews = [
review for review in sorted_reviews
if review.author.login == KODIAK_LOGIN
]
status = review_status(kodiak_reviews)
if status != PRReviewState.APPROVED:
await api.approve_pull_request()
else:
log.info("approval already exists, not adding another")
need_branch_update = (branch_protection.requiresStrictStatusChecks
and pull_request.mergeStateStatus
== MergeStateStatus.BEHIND)
meets_label_requirement = (config.merge.automerge_label
in pull_request.labels
or not config.update.require_automerge_label)
if (need_branch_update and not merging and config.update.always
and meets_label_requirement):
await set_status(
"🔄 updating branch",
markdown_content=
"branch updated because `update.always = true` is configured.",
)
await api.update_branch()
return
if (config.merge.require_automerge_label
and config.merge.automerge_label not in pull_request.labels):
await block_merge(
api,
pull_request,
f"missing automerge_label: {config.merge.automerge_label!r}",
)
return
blacklist_labels = set(config.merge.blacklist_labels) & set(
pull_request.labels)
if blacklist_labels:
await block_merge(api, pull_request,
f"has blacklist_labels: {blacklist_labels!r}")
return
if (config.merge.blacklist_title_regex
and re.search(config.merge.blacklist_title_regex,
pull_request.title) is not None):
await block_merge(
api,
pull_request,
f"title matches blacklist_title_regex: {config.merge.blacklist_title_regex!r}",
)
return
if pull_request.mergeStateStatus == MergeStateStatus.DRAFT:
await block_merge(api, pull_request, "pull request is in draft state")
return
if config.merge.block_on_reviews_requested and review_requests:
names = [r.name for r in review_requests]
await block_merge(api, pull_request, f"reviews requested: {names!r}")
return
if pull_request.state == PullRequestState.MERGED:
log.info(
"pull request merged. config.merge.delete_branch_on_merge=%r",
config.merge.delete_branch_on_merge,
)
await api.dequeue()
if not config.merge.delete_branch_on_merge or pull_request.isCrossRepository:
return
pr_count = await api.pull_requests_for_ref(ref=pull_request.headRefName
)
# if we couldn't access the dependent PR count or we have dependent PRs
# we will abort deleting this branch.
if pr_count is None or pr_count > 0:
log.info("skipping branch deletion because of dependent PRs",
pr_count=pr_count)
return
await api.delete_branch(branch_name=pull_request.headRefName)
return
if pull_request.state == PullRequestState.CLOSED:
await api.dequeue()
return
if (pull_request.mergeStateStatus == MergeStateStatus.DIRTY
or pull_request.mergeable == MergeableState.CONFLICTING):
await block_merge(api, pull_request, "merge conflict")
# remove label if configured and send message
if config.merge.notify_on_conflict and config.merge.require_automerge_label:
automerge_label = config.merge.automerge_label
await api.remove_label(automerge_label)
body = textwrap.dedent(f"""
This PR currently has a merge conflict. Please resolve this and then re-add the `{automerge_label}` label.
""")
await api.create_comment(body)
return
if pull_request.mergeStateStatus == MergeStateStatus.UNSTABLE:
# TODO(chdsbd): This status means that the pr is mergeable but has failing
# status checks. we may want to handle this via config
pass
if pull_request.mergeable == MergeableState.UNKNOWN:
# we need to trigger a test commit to fix this. We do that by calling
# GET on the pull request endpoint.
await api.trigger_test_commit()
return
wait_for_checks = False
if pull_request.mergeStateStatus in (
MergeStateStatus.BLOCKED,
MergeStateStatus.BEHIND,
):
# figure out why we can't merge. There isn't a way to get this simply from the Github API. We need to find out ourselves.
#
# I think it's possible to find out blockers from branch protection issues
# https://developer.github.com/v4/object/branchprotectionrule/?#fields
#
# - missing reviews
# - blocking reviews
# - missing required status checks
# - failing required status checks
# - branch not up to date (should be handled before this)
# - missing required signature
if (branch_protection.requiresApprovingReviews
and branch_protection.requiredApprovingReviewCount):
reviews_by_author: MutableMapping[
str, List[PRReview]] = defaultdict(list)
for review in sorted(reviews, key=lambda x: x.createdAt):
if review.author.permission not in {
Permission.ADMIN, Permission.WRITE
}:
continue
reviews_by_author[review.author.login].append(review)
successful_reviews = 0
for author_name, review_list in reviews_by_author.items():
review_state = review_status(review_list)
# blocking review
if review_state == PRReviewState.CHANGES_REQUESTED:
await block_merge(api, pull_request,
f"changes requested by {author_name!r}")
return
# successful review
if review_state == PRReviewState.APPROVED:
successful_reviews += 1
# missing required review count
if successful_reviews < branch_protection.requiredApprovingReviewCount:
await block_merge(
api,
pull_request,
f"missing required reviews, have {successful_reviews!r}/{branch_protection.requiredApprovingReviewCount!r}",
)
return
required: Set[str] = set()
passing: Set[str] = set()
if branch_protection.requiresStatusChecks:
skippable_contexts: List[str] = []
failing_contexts: List[str] = []
pending_contexts: List[str] = []
passing_contexts: List[str] = []
required = set(branch_protection.requiredStatusCheckContexts)
for status_context in contexts:
# handle dont_wait_on_status_checks. We want to consider a
# status_check failed if it is incomplete and in the
# configuration.
if (status_context.context
in config.merge.dont_wait_on_status_checks
and status_context.state
in (StatusState.EXPECTED, StatusState.PENDING)):
skippable_contexts.append(status_context.context)
continue
if status_context.state in (StatusState.ERROR,
StatusState.FAILURE):
failing_contexts.append(status_context.context)
elif status_context.state in (
StatusState.EXPECTED,
StatusState.PENDING,
):
pending_contexts.append(status_context.context)
else:
assert status_context.state == StatusState.SUCCESS
passing_contexts.append(status_context.context)
for check_run in check_runs:
if (check_run.name in config.merge.dont_wait_on_status_checks
and check_run.conclusion
in (None, CheckConclusionState.NEUTRAL)):
skippable_contexts.append(check_run.name)
continue
if check_run.conclusion is None:
continue
if check_run.conclusion == CheckConclusionState.SUCCESS:
passing_contexts.append(check_run.name)
if check_run.conclusion in (
CheckConclusionState.ACTION_REQUIRED,
CheckConclusionState.FAILURE,
CheckConclusionState.TIMED_OUT,
):
failing_contexts.append(check_run.name)
passing = set(passing_contexts)
failing = set(failing_contexts)
# we have failing statuses that are required
failing_required_status_checks = failing & required
# GitHub has undocumented logic for travis-ci checks in GitHub
# branch protection rules. GitHub compresses
# "continuous-integration/travis-ci/{pr,push}" to
# "continuous-integration/travis-ci". There is only special handling
# for these specific checks.
if "continuous-integration/travis-ci" in required:
if "continuous-integration/travis-ci/pr" in failing:
failing_required_status_checks.add(
"continuous-integration/travis-ci/pr")
if "continuous-integration/travis-ci/push" in failing:
failing_required_status_checks.add(
"continuous-integration/travis-ci/push")
# either check can satisfy continuous-integration/travis-ci, but
# if either fails they'll also block the merge.
if ("continuous-integration/travis-ci/pr" in passing
or "continuous-integration/travis-ci/push" in passing):
required.remove("continuous-integration/travis-ci")
if failing_required_status_checks:
# NOTE(chdsbd): We need to skip this PR because it would block
# the merge queue. We may be able to bump it to the back of the
# queue, but it's easier just to remove it all together. There
# is a similar question for the review counting.
await block_merge(
api,
pull_request,
f"failing required status checks: {failing_required_status_checks!r}",
)
return
if skippable_contexts:
if merging:
if skippable_check_timeout > 0:
await set_status(
f"⛴ merging PR (waiting a bit for dont_wait_on_status_checks: {skippable_contexts!r})"
)
raise RetryForSkippableChecks
log.warning(
"timeout reached waiting for dont_wait_on_status_checks",
skippable_contexts=skippable_contexts,
)
await set_status(
f"⚠️ timeout reached for dont_wait_on_status_checks: {skippable_contexts!r}"
)
await set_status(
f"🛑 not waiting for dont_wait_on_status_checks: {skippable_contexts!r}"
)
return
missing_required_status_checks = required - passing
wait_for_checks = bool(branch_protection.requiresStatusChecks
and missing_required_status_checks)
if config.merge.update_branch_immediately and need_branch_update:
await set_status(
"🔄 updating branch",
markdown_content=
"branch updated because `merge.update_branch_immediately = true` is configured.",
)
await api.update_branch()
return
if merging:
# prioritize branch updates over waiting for status checks to complete
if config.merge.optimistic_updates:
if need_branch_update:
await set_status("⛴ merging PR (updating branch)")
await api.update_branch()
raise PollForever
if wait_for_checks:
await set_status(
f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})"
)
raise PollForever
# almost the same as the pervious case, but we prioritize status checks
# over branch updates.
else:
if wait_for_checks:
await set_status(
f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})"
)
raise PollForever
if need_branch_update:
await set_status("⛴ merging PR (updating branch)")
await api.update_branch()
raise PollForever
# if we reach this point and we don't need to wait for checks or update a branch we've failed to calculate why the PR is blocked. This should _not_ happen normally.
if not (wait_for_checks or need_branch_update):
await block_merge(api, pull_request,
"Merging blocked by GitHub requirements")
log.warning("merge blocked for unknown reason")
return
ready_to_merge = not (wait_for_checks or need_branch_update)
if config.merge.do_not_merge:
if wait_for_checks:
await set_status(
f"⌛️ waiting for required status checks: {missing_required_status_checks!r}"
)
elif need_branch_update:
await set_status(
"⚠️ need branch update (suggestion: use merge.update_branch_immediately with merge.do_not_merge)",
markdown_content="""\
When `merge.do_not_merge = true` is configured `merge.update_branch_immediately = true` \
is recommended so Kodiak can automatically update branches.
By default, Kodiak is efficient and only update branches when merging a PR, but \
when `merge.do_not_merge` is enabled, Kodiak never has that opportunity to \
update a branch during merge. `merge.update_branch_immediately = true` will \
trigger Kodiak to update branches whenever a PR is outdated and not failing any \
branch protection requirements.
""",
)
else:
await set_status("✅ okay to merge")
log.info(
"eligible to merge, stopping because config.merge.do_not_merge is enabled."
)
return
# okay to merge if we reach this point.
if (config.merge.prioritize_ready_to_merge and ready_to_merge) or merging:
merge_args = get_merge_body(config, pull_request)
await set_status("⛴ attempting to merge PR (merging)")
await api.merge(
merge_method=merge_args.merge_method,
commit_title=merge_args.commit_title,
commit_message=merge_args.commit_message,
)
else:
position_in_queue = await api.queue_for_merge()
if position_in_queue is None:
# this case should be rare/impossible.
log.warning("couldn't find position for enqueued PR")
return
ordinal_position = inflection.ordinalize(position_in_queue + 1)
if not is_active_merge:
await set_status(
f"📦 enqueued for merge (position={ordinal_position})")
else:
log.info(
"not setting status message for enqueued job because is_active_merge=True"
)
return
def test_start_end(self):
m = rure.search(u"remove_this", u"tony@tiremove_thisger.net")
self.assertEqual(
(m.string[:m.start()] + m.string[m.end():]).decode('utf8'),
u'*****@*****.**')
def fix_vba_code(vba_code):
"""
Fix up some substrings that ViperMonkey has problems parsing.
"""
# Strip comment lines from the code.
vba_code = strip_comments(vba_code)
# Fix dumb typo in some maldocs VBA.
vba_code = vba_code.replace("End SubPrivate", "End Sub\nPrivate")
# No null bytes in VB to process.
vba_code = vba_code.replace("\x00", "")
# Make "End Try" in try/catch blocks easier to parse.
vba_code = re.sub(r"End\s+Try", "##End ##Try", vba_code)
# We don't handle Line Input constructs for now. Delete them.
# TODO: Actually handle Line Input consructs.
linputs = re.findall(r"Line\s+Input\s+#\d+\s*,\s*\w+", vba_code, re.DOTALL)
if (len(linputs) > 0):
log.warning("VB Line Input constructs are not currently handled. Stripping them from code...")
for linput in linputs:
vba_code = vba_code.replace(linput, "")
# We don't handle Property constructs for now. Delete them.
# TODO: Actually handle Property consructs.
props = re.findall(r"(?:Public\s+|Private\s+|Friend\s+)?Property\s+.+?End\s+Property", vba_code, re.DOTALL)
if (len(props) > 0):
log.warning("VB Property constructs are not currently handled. Stripping them from code...")
for prop in props:
vba_code = vba_code.replace(prop, "")
# We don't handle Implements constructs for now. Delete them.
# TODO: Figure out if we need to worry about Implements.
implements = re.findall(r"Implements \w+", vba_code, re.DOTALL)
if (len(implements) > 0):
log.warning("VB Implements constructs are not currently handled. Stripping them from code...")
for imp in implements:
vba_code = vba_code.replace(imp, "")
# We don't handle Enum constructs for now. Delete them.
# TODO: Actually handle Enum consructs.
enums = re.findall(r"(?:(?:Public|Private)\s+)?Enum\s+.+?End\s+Enum", vba_code, re.DOTALL)
if (len(enums) > 0):
log.warning("VB Enum constructs are not currently handled. Stripping them from code...")
for enum in enums:
vba_code = vba_code.replace(enum, "")
# We don't handle ([a1]) constructs for now. Delete them.
# TODO: Actually handle these things.
brackets = re.findall(r"\(\[[^\]]+\]\)", vba_code, re.DOTALL)
if (len(brackets) > 0):
log.warning("([a1]) style constructs are not currently handled. Rewriting them...")
for bracket in brackets:
vba_code = vba_code.replace(bracket, "(" + bracket[2:-2] + ")")
# Clear out lines broken up on multiple lines.
vba_code = re.sub(r" _ *\r?\n", "", vba_code)
vba_code = re.sub(r"&_ *\r?\n", "&", vba_code)
vba_code = re.sub(r"\(_ *\r?\n", "(", vba_code)
#vba_code = re.sub(r":\s*[Ee]nd\s+[Ss]ub", r"\nEnd Sub", vba_code)
vba_code = "\n" + vba_code
vba_code = re.sub(r"\n:", "\n", vba_code)
# Some maldocs have single line member access expressions that end with a '.'.
# Comment those out.
dumb_member_exps = re.findall(r"\n(?:\w+\.)+\n", vba_code)
for dumb_exp in dumb_member_exps:
log.warning("Commenting out bad line '" + dumb_exp.replace("\n", "") + "'.")
safe_exp = "\n'" + dumb_exp[1:]
vba_code = vba_code.replace(dumb_exp, safe_exp)
# How about maldocs with Subs with spaces in their names?
space_subs = re.findall(r"\n\s*Sub\s*\w+\s+\w+\s*\(", vba_code)
for space_sub in space_subs:
start = space_sub.index("Sub") + len("Sub")
end = space_sub.rindex("(")
sub_name = space_sub[start:end]
new_name = sub_name.replace(" ", "_")
log.warning("Replacing bad sub name '" + sub_name + "' with '" + new_name + "'.")
vba_code = vba_code.replace(sub_name, new_name)
# Clear out some garbage characters.
#vba_code = vba_code.replace('\x0b', '')
#vba_code = vba_code.replace('\x88', '')
# It looks like VBA supports variable and function names containing
# non-ASCII characters. Parsing these with pyparsing would be difficult
# (or impossible), so convert the non-ASCII names to ASCII.
#
# Break up lines with multiple statements onto their own lines.
vba_code = fix_difficult_code(vba_code)
# Fix function calls with a skipped 1st argument.
vba_code = fix_skipped_1st_arg(vba_code)
# Fix lines with missing double quotes.
vba_code = fix_unbalanced_quotes(vba_code)
# For each const integer defined, replace it inline in the code to reduce lookups
vba_code = replace_constant_int_inline(vba_code)
# Skip the next part if unnneeded.
uni_vba_code = None
try:
uni_vba_code = vba_code.decode("utf-8")
except UnicodeDecodeError:
# Punt.
return vba_code
got_multassign = (re2.search(u"(?:\w+\s*=\s*){2}", uni_vba_code) is not None)
if ((" if+" not in vba_code) and
(" If+" not in vba_code) and
("\nif+" not in vba_code) and
("\nIf+" not in vba_code) and
(not got_multassign)):
return vba_code
# Change things like 'If+foo > 12 ..." to "If foo > 12 ...".
r = ""
for line in vba_code.split("\n"):
# Fix up assignments like 'cat = dog = frog = 12'.
line = fix_multiple_assignments(line)
# Do we have an "if+..."?
if ("if+" not in line.lower()):
# No. No change.
r += line + "\n"
continue
# Yes we do. Figure out if it is in a string.
in_str = False
window = " "
new_line = ""
for c in line:
# Start/End of string?
if (c == '"'):
in_str = not in_str
# Have we seen an if+ ?
if ((not in_str) and (c == "+") and (window.lower() == " if")):
# Replace the '+' with a ' '.
new_line += " "
# No if+ .
else:
new_line += c
# Advance the viewing window.
window = window[1:] + c
# Save the updated line.
r += new_line + "\n"
# Return the updated code.
return r
