def test_nonmatching_captures(self): ptn = u"(re).*(ger)" email = u"tony@tiremove_thisger.net" results = rure.search(ptn, email) stdlib_results = re.search(ptn, email) self.assertIsNotNone(results) self.assertEqual(len(results.groups()), len(stdlib_results.groups())) ptn = u"(re)|(ger)" results = rure.search(ptn, email) stdlib_results = re.search(ptn, email) self.assertIsNotNone(results) self.assertEqual(len(results.groups()), len(stdlib_results.groups())) self.assertEqual(results.group(0), stdlib_results.group(0))
def fix_unbalanced_quotes(vba_code): """ Fix lines with missing double quotes. """ # Fix invalid string assignments. uni_vba_code = None try: uni_vba_code = vba_code.decode("utf-8") except UnicodeDecodeError: # Punt. return vba_code if (re2.search(u"(\w+)\s+=\s+\"\r?\n", uni_vba_code) is not None): vba_code = re.sub(r"(\w+)\s+=\s+\"\r?\n", r'\1 = ""\n', vba_code) if (re2.search(u"(\w+\s+=\s+\")(:[^\"]+)\r?\n", uni_vba_code) is not None): vba_code = re.sub(r"(\w+\s+=\s+\")(:[^\"]+)\r?\n", r'\1"\2\n', vba_code) if (re2.search(u"^\"[^=]*([=>])\s*\"\s+[Tt][Hh][Ee][Nn]", uni_vba_code) is not None): vba_code = re.sub(r"^\"[^=]*([=>])\s*\"\s+[Tt][Hh][Ee][Nn]", r'\1 "" Then', vba_code) # Fix ambiguous EOL comment lines like ".foo '' A comment". "''" could be parsed as # an argument to .foo or as an EOL comment. Here we change things like ".foo '' A comment" # to ".foo ' A comment" so it is not ambiguous (parse as comment). vba_code += "\n" vba_code = re.sub(r"'('[^'^\"]+\n)", r"\1", vba_code, re.DOTALL) # More ambiguous EOL comments. Something like "a = 12 : 'stuff 'more stuff" could have # 'stuff ' potentially parsed as a string. Just wipe out the comments in this case # (ex. "a = 12 : 'stuff 'more stuff" => "a = 12 :"). vba_code = re.sub(r"(\n[^'^\n]+)'[^'^\"^\n]+'[^'^\"^\n]+\n", r"\1\n", vba_code, re.DOTALL) # Fix Execute statements with no space between the execute and the argument. vba_code = re.sub(r"\n\s*([Ee][Xx][Ee][Cc][Uu][Tt][Ee])\"", r'\nExecute "', vba_code) # See if we have lines with unbalanced double quotes. r = "" for line in vba_code.split("\n"): if ('"' not in line): r += line + "\n" continue num_quotes = line.count('"') if ((num_quotes % 2) != 0): last_quote = line.rindex('"') line = line[:last_quote] + '"' + line[last_quote:] r += line + "\n" # Return the balanced code. return r
def is_mixed_wide_ascii_str(the_str): """ Test a string to see if it is a mix of wide and ASCII chars. """ uni_str = None try: uni_str = the_str.decode("utf-8") except UnicodeDecodeError: # Punt. return False extended_asc_pat = b"[\x80-\xff]" if (re.search(extended_asc_pat, uni_str) is not None): return True return False
def fix_unhandled_array_assigns(vba_code): """ Currently things like 'foo(1, 2, 3) = 1' are not handled. Comment them out. """ uni_vba_code = None try: uni_vba_code = vba_code.decode("utf-8") except UnicodeDecodeError: # Punt. return vba_code pat = "\n(\s*\w+\((?:\w+\s*,\s*){2,}\w+\)\s*=)" if (re2.search(unicode(pat), uni_vba_code) is not None): vba_code = re.sub(pat, r"\n' UNHANDLED ARRAY ASSIGNMENT \1", vba_code) return vba_code
def is_mixed_wide_ascii_str(the_str): """Test a string to see if a string is a mix of wide and ASCII chars. @param the_str (str) The string to check. @return (boolean) True if the string is a mized string, False if not. """ uni_str = None try: uni_str = the_str.decode("utf-8") except UnicodeDecodeError: # Punt. return False extended_asc_pat = b"[\x80-\xff]" if (re.search(extended_asc_pat, uni_str) is not None): return True return False
async def mergeable( api: PRAPI, config: Union[config.V1, pydantic.ValidationError, toml.TomlDecodeError], config_str: str, config_path: str, pull_request: PullRequest, branch_protection: Optional[BranchProtectionRule], review_requests: List[PRReviewRequest], reviews: List[PRReview], contexts: List[StatusContext], check_runs: List[CheckRun], commits: List[Commit], valid_signature: bool, valid_merge_methods: List[MergeMethod], repository: RepoInfo, merging: bool, is_active_merge: bool, skippable_check_timeout: int, api_call_retries_remaining: int, api_call_errors: Sequence[APICallRetry], subscription: Optional[Subscription], app_id: Optional[str] = None, ) -> None: # TODO(chdsbd): Use structlog bind_contextvars to automatically set useful context (install id, repo, pr number). log = logger.bind(number=pull_request.number, url=pull_request.url) # we set is_active_merge when the PR is being merged from the merge queue. # We don't want to clobber any statuses set by that system, so we take no # action. If the PR becomes ineligible for merging that logic will handle # it. # rebase_fast_forward isn't determined via the GitHub UI and is always # available. valid_merge_methods = [ *valid_merge_methods, MergeMethod.rebase_fast_forward ] async def set_status(msg: str, markdown_content: Optional[str] = None) -> None: # don't clobber statuses set via merge loop. if is_active_merge: return await api.set_status( msg, latest_commit_sha=pull_request.latest_sha, markdown_content=markdown_content, ) if not isinstance(config, V1): log.warning("problem fetching config") await set_status( '⚠️ Invalid configuration (Click "Details" for more info.)', markdown_content=get_markdown_for_config(config, config_str=config_str, git_path=config_path), ) await api.dequeue() return if api_call_retries_remaining == 0: log.warning("timeout reached for api calls to GitHub") if api_call_errors: first_error = api_call_errors[0] await set_status( f"⚠️ problem contacting GitHub API with method {first_error.api_name!r}", markdown_content=get_markdown_for_api_call_errors( errors=api_call_errors), ) else: await set_status("⚠️ problem contacting GitHub API") return # if we have an app_id in the config then we only want to work on this repo # if our app_id from the environment matches the configuration. if config.app_id is not None and config.app_id != app_id: log.info("missing required app_id") await api.dequeue() return if branch_protection is None: await cfg_err( api, pull_request, f"missing branch protection for baseRef: {pull_request.baseRefName!r}", ) return merge_method = get_merge_method( cfg_merge_method=config.merge.method, valid_merge_methods=valid_merge_methods, log=log, labels=pull_request.labels, ) if (branch_protection.requiresCommitSignatures and merge_method == MergeMethod.rebase): await cfg_err( api, pull_request, '"Require signed commits" branch protection is only supported with "squash" or "merge" commits. Rebase is not supported by GitHub.', ) return if merge_method not in valid_merge_methods: valid_merge_methods_str = [ method.value for method in valid_merge_methods ] await cfg_err( api, pull_request, f"configured merge.method {merge_method.value!r} is invalid. Valid methods for repo are {valid_merge_methods_str!r}", ) return if (not config.merge.do_not_merge and branch_protection.restrictsPushes and missing_push_allowance(branch_protection.pushAllowances.nodes)): await cfg_err( api, pull_request, "push restriction branch protection setting is missing push allowance for Kodiak", markdown_content=get_markdown_for_push_allowance_error( branch_name=pull_request.baseRefName), ) return # we keep the configuration errors before the rest of the application logic # so configuration issues are surfaced as early as possible. if config.disable_bot_label in pull_request.labels: await api.dequeue() await api.set_status( f"🚨 kodiak disabled by disable_bot_label ({config.disable_bot_label}). Remove label to re-enable Kodiak.", latest_commit_sha=pull_request.latest_sha, ) return if (app_config.SUBSCRIPTIONS_ENABLED and repository.is_private and subscription is not None and subscription.subscription_blocker is not None): # We only count private repositories in our usage calculations. A user # has an active subscription if a subscription exists in Redis and has # an empty subscription_blocker. # # We also ignore missing subscriptions. The web api will set # subscription blockers if usage exceeds limits. status_message = get_paywall_status_for_blocker( pull_request, subscription.subscription_blocker, log) if status_message is not None: await set_status( f"💳 subscription: {status_message}", markdown_content=get_markdown_for_paywall(), ) return pull_request_labels = set(pull_request.labels) config_automerge_labels = ({config.merge.automerge_label} if isinstance( config.merge.automerge_label, str) else set( config.merge.automerge_label)) pull_request_automerge_labels = config_automerge_labels.intersection( pull_request_labels) has_automerge_label = len(pull_request_automerge_labels) > 0 should_dependency_automerge = ( pull_request.author.login in config.merge.automerge_dependencies.usernames and dep_version_from_title(pull_request.title) in config.merge.automerge_dependencies.versions) # we should trigger mergeability checks whenever we encounter UNKNOWN. # # I don't foresee conflicts with checking configuration errors, # `config.disable_bot_label`, and the paywall before this code. # # Previously we had an issue where this code wasn't being entered because # `merge.blocking_title_regex` was checked first. Which caused # `update.always` to not operate. if (pull_request.mergeable == MergeableState.UNKNOWN and pull_request.state == PullRequestState.OPEN): # we need to trigger a test commit to fix this. We do that by calling # GET on the pull request endpoint. await api.trigger_test_commit() # queue the PR for evaluation again in case GitHub doesn't send another # webhook for the commit test. await api.requeue() # we don't want to abort the merge if we encounter this status check. # Just keep polling! if merging: raise PollForever return is_draft_pull_request = (pull_request.isDraft or pull_request.mergeStateStatus == MergeStateStatus.DRAFT) if (pull_request.author.login in config.approve.auto_approve_usernames and pull_request.state == PullRequestState.OPEN and not is_draft_pull_request): # if the PR was created by an approve author and we have not previously # given an approval, approve the PR. sorted_reviews = sorted(reviews, key=lambda x: x.createdAt) kodiak_reviews = [ review for review in sorted_reviews if review.author.login == KODIAK_LOGIN ] status = review_status(kodiak_reviews) if status != PRReviewState.APPROVED: await api.approve_pull_request() else: log.info("approval already exists, not adding another") need_branch_update = (branch_protection.requiresStrictStatusChecks and pull_request.mergeStateStatus == MergeStateStatus.BEHIND) update_always = config.update.always and ( has_automerge_label or not config.update.require_automerge_label) has_autoupdate_label = config.update.autoupdate_label in pull_request_labels auto_update_enabled = update_always or has_autoupdate_label # Dequeue pull request if out-of-date and author in # `update.ignored_usernames`. We cannot update or merge it. # # If `update.autoupdate_label` is applied to the pull request, bypass # `update.ignored_usernames` and let the pull request update. if need_branch_update and not has_autoupdate_label: if pull_request.author.login in config.update.blacklist_usernames: await set_status( f"🛑 updates blocked by update.blacklist_usernames: {config.update.blacklist_usernames!r}", markdown_content= "Apply the `update.autoupdate_label` label to enable updates for this pull request.", ) await api.dequeue() return if pull_request.author.login in config.update.ignored_usernames: await set_status( f"🛑 updates blocked by update.ignored_usernames: {config.update.ignored_usernames!r}", markdown_content= "Apply the `update.autoupdate_label` label to enable updates for this pull request.", ) await api.dequeue() return if need_branch_update and not merging and auto_update_enabled: await set_status( "🔄 updating branch", markdown_content= "branch updated because `update.always = true` is configured.", ) await api.update_branch() return if (config.merge.require_automerge_label and not has_automerge_label and not should_dependency_automerge): await block_merge( api, pull_request, f"missing automerge_label: {config.merge.automerge_label!r}", ) return # We want users to get notified a merge conflict even if the PR matches a # WIP title via merge.blacklist_title_regex. if (pull_request.mergeStateStatus == MergeStateStatus.DIRTY or pull_request.mergeable == MergeableState.CONFLICTING ) and pull_request.state == PullRequestState.OPEN: await block_merge(api, pull_request, "merge conflict") # remove label if configured and send message if (config.merge.notify_on_conflict and config.merge.require_automerge_label and has_automerge_label): automerge_label = config.merge.automerge_label await asyncio.gather(*[ api.remove_label(label) for label in pull_request_automerge_labels ]) body = textwrap.dedent(f""" This PR currently has a merge conflict. Please resolve this and then re-add the `{automerge_label}` label. """) await api.create_comment(body) return blacklist_labels = set(config.merge.blacklist_labels) & set( pull_request.labels) blocking_labels = set(config.merge.blocking_labels) & set( pull_request.labels) if blacklist_labels: await block_merge(api, pull_request, f"has blacklist_labels: {blacklist_labels!r}") return if blocking_labels: await block_merge(api, pull_request, f"has merge.blocking_labels: {blocking_labels!r}") return title_blocker = get_blocking_title_regex(config) if (title_blocker.pattern and re.search(title_blocker.pattern, pull_request.title) is not None): await block_merge( api, pull_request, f"title matches {title_blocker.config_key}: {title_blocker.pattern!r}", ) return if is_draft_pull_request: await block_merge(api, pull_request, "pull request is in draft state") return if config.merge.block_on_reviews_requested and review_requests: names = [r.name for r in review_requests] await block_merge(api, pull_request, f"reviews requested: {names!r}") return if pull_request.state == PullRequestState.MERGED: log.info( "pull request merged. config.merge.delete_branch_on_merge=%r", config.merge.delete_branch_on_merge, ) await api.dequeue() if (not config.merge.delete_branch_on_merge or pull_request.isCrossRepository or repository.delete_branch_on_merge): return pr_count = await api.pull_requests_for_ref(ref=pull_request.headRefName ) # if we couldn't access the dependent PR count or we have dependent PRs # we will abort deleting this branch. if pr_count is None or pr_count > 0: log.info("skipping branch deletion because of dependent PRs", pr_count=pr_count) return await api.delete_branch(branch_name=pull_request.headRefName) return if pull_request.state == PullRequestState.CLOSED: await api.dequeue() return if pull_request.mergeStateStatus == MergeStateStatus.UNSTABLE: # TODO(chdsbd): This status means that the pr is mergeable but has failing # status checks. we may want to handle this via config pass wait_for_checks = False if pull_request.mergeStateStatus in ( MergeStateStatus.BLOCKED, MergeStateStatus.BEHIND, ): # figure out why we can't merge. There isn't a way to get this simply from the Github API. We need to find out ourselves. # # I think it's possible to find out blockers from branch protection issues # https://developer.github.com/v4/object/branchprotectionrule/?#fields # # - missing reviews # - blocking reviews # - missing required status checks # - failing required status checks # - branch not up to date (should be handled before this) # - missing required signature if (branch_protection.requiresApprovingReviews and branch_protection.requiredApprovingReviewCount): reviews_by_author: MutableMapping[ str, List[PRReview]] = defaultdict(list) for review in sorted(reviews, key=lambda x: x.createdAt): if review.author.permission not in { Permission.ADMIN, Permission.WRITE }: continue reviews_by_author[review.author.login].append(review) successful_reviews = 0 for author_name, review_list in reviews_by_author.items(): review_state = review_status(review_list) # blocking review if review_state == PRReviewState.CHANGES_REQUESTED: await block_merge(api, pull_request, f"changes requested by {author_name!r}") return # successful review if review_state == PRReviewState.APPROVED: successful_reviews += 1 # missing required review count if successful_reviews < branch_protection.requiredApprovingReviewCount: await block_merge( api, pull_request, f"missing required reviews, have {successful_reviews!r}/{branch_protection.requiredApprovingReviewCount!r}", ) return if pull_request.reviewDecision == PullRequestReviewDecision.REVIEW_REQUIRED: await block_merge(api, pull_request, "missing required reviews") return required: Set[str] = set() passing: Set[str] = set() if branch_protection.requiresStatusChecks: skippable_contexts: List[str] = [] failing_contexts: List[str] = [] pending_contexts: List[str] = [] passing_contexts: List[str] = [] required = set(branch_protection.requiredStatusCheckContexts) for status_context in contexts: # handle dont_wait_on_status_checks. We want to consider a # status_check failed if it is incomplete and in the # configuration. if (status_context.context in config.merge.dont_wait_on_status_checks and status_context.state in (StatusState.EXPECTED, StatusState.PENDING)): skippable_contexts.append(status_context.context) continue if status_context.state in (StatusState.ERROR, StatusState.FAILURE): failing_contexts.append(status_context.context) elif status_context.state in ( StatusState.EXPECTED, StatusState.PENDING, ): pending_contexts.append(status_context.context) else: assert status_context.state == StatusState.SUCCESS passing_contexts.append(status_context.context) for check_run in check_runs: if (check_run.name in config.merge.dont_wait_on_status_checks and check_run.conclusion in (None, CheckConclusionState.NEUTRAL)): skippable_contexts.append(check_run.name) continue if check_run.conclusion is None: continue if check_run.conclusion == CheckConclusionState.SUCCESS: passing_contexts.append(check_run.name) if check_run.conclusion in ( CheckConclusionState.ACTION_REQUIRED, CheckConclusionState.FAILURE, CheckConclusionState.TIMED_OUT, CheckConclusionState.CANCELLED, CheckConclusionState.SKIPPED, CheckConclusionState.STALE, ): failing_contexts.append(check_run.name) passing = set(passing_contexts) failing = set(failing_contexts) # we have failing statuses that are required failing_required_status_checks = failing & required # GitHub has undocumented logic for travis-ci checks in GitHub # branch protection rules. GitHub compresses # "continuous-integration/travis-ci/{pr,push}" to # "continuous-integration/travis-ci". There is only special handling # for these specific checks. if "continuous-integration/travis-ci" in required: if "continuous-integration/travis-ci/pr" in failing: failing_required_status_checks.add( "continuous-integration/travis-ci/pr") if "continuous-integration/travis-ci/push" in failing: failing_required_status_checks.add( "continuous-integration/travis-ci/push") # either check can satisfy continuous-integration/travis-ci, but # if either fails they'll also block the merge. if ("continuous-integration/travis-ci/pr" in passing or "continuous-integration/travis-ci/push" in passing): required.remove("continuous-integration/travis-ci") if failing_required_status_checks: # NOTE(chdsbd): We need to skip this PR because it would block # the merge queue. We may be able to bump it to the back of the # queue, but it's easier just to remove it all together. There # is a similar question for the review counting. await block_merge( api, pull_request, f"failing required status checks: {failing_required_status_checks!r}", ) return if skippable_contexts: if merging: if skippable_check_timeout > 0: await set_status( f"⛴ merging PR (waiting a bit for dont_wait_on_status_checks: {skippable_contexts!r})" ) raise RetryForSkippableChecks log.warning( "timeout reached waiting for dont_wait_on_status_checks", skippable_contexts=skippable_contexts, ) await set_status( f"⚠️ timeout reached for dont_wait_on_status_checks: {skippable_contexts!r}" ) await set_status( f"🛑 not waiting for dont_wait_on_status_checks: {skippable_contexts!r}" ) return missing_required_status_checks = required - passing wait_for_checks = bool(branch_protection.requiresStatusChecks and missing_required_status_checks) if config.merge.update_branch_immediately and need_branch_update: await set_status( "🔄 updating branch", markdown_content= "branch updated because `merge.update_branch_immediately = true` is configured.", ) await api.update_branch() if merging: raise PollForever return if merging: # prioritize branch updates over waiting for status checks to complete if config.merge.optimistic_updates: if need_branch_update: await set_status("⛴ merging PR (updating branch)") await api.update_branch() raise PollForever if wait_for_checks: await set_status( f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})" ) raise PollForever # almost the same as the pervious case, but we prioritize status checks # over branch updates. else: if wait_for_checks: await set_status( f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})" ) raise PollForever if need_branch_update: await set_status("⛴ merging PR (updating branch)") await api.update_branch() raise PollForever # if we reach this point and we don't need to wait for checks or update a branch we've failed to calculate why the PR is blocked. This should _not_ happen normally. if not (wait_for_checks or need_branch_update): await block_merge(api, pull_request, "Merging blocked by GitHub requirements") log.warning("merge blocked for unknown reason") return ready_to_merge = not (wait_for_checks or need_branch_update) if config.merge.do_not_merge: if wait_for_checks: await set_status( f"⌛️ waiting for required status checks: {missing_required_status_checks!r}" ) elif need_branch_update: await set_status( "⚠️ need branch update (suggestion: use merge.update_branch_immediately with merge.do_not_merge)", markdown_content="""\ When `merge.do_not_merge = true` is configured `merge.update_branch_immediately = true` \ is recommended so Kodiak can automatically update branches. By default, Kodiak is efficient and only update branches when merging a PR, but \ when `merge.do_not_merge` is enabled, Kodiak never has that opportunity to \ update a branch during merge. `merge.update_branch_immediately = true` will \ trigger Kodiak to update branches whenever a PR is outdated and not failing any \ branch protection requirements. """, ) else: await set_status("✅ okay to merge") log.info( "eligible to merge, stopping because config.merge.do_not_merge is enabled." ) return # okay to merge if we reach this point. if (config.merge.prioritize_ready_to_merge and ready_to_merge) or merging: merge_args = get_merge_body(config, merge_method, pull_request, commits=commits) await set_status("⛴ attempting to merge PR (merging)") try: # Use the Git Refs API to rebase merge. # # This preserves the rebased commits and their hashes. Using the # GitHub Pull Request API to rebase merge rewrites the rebased # commits, so the commit hashes change. # # For build systems that depend on commit hashes instead of tree # hashes, it's desirable to not rewrite commits. if merge_args.merge_method is MergeMethod.rebase_fast_forward: await api.update_ref(ref=pull_request.baseRefName, sha=pull_request.latest_sha) else: await api.merge( merge_method=merge_args.merge_method, commit_title=merge_args.commit_title, commit_message=merge_args.commit_message, ) # if we encounter an internal server error (status code 500), it is # _not_ safe to retry. Instead we mark the pull request as unmergable # and require a user to re-enable Kodiak on the pull request. except GitHubApiInternalServerError: logger.warning( "kodiak encountered GitHub API error merging pull request", exc_info=True, ) # We add the disable_bot_label to disable Kodiak from taking any # action to update, approve, comment, label, or merge. disable_bot_label = config.disable_bot_label await api.add_label(disable_bot_label) await block_merge(api, pull_request, "Cannot merge due to GitHub API failure.") body = messages.format( textwrap.dedent(f""" This PR could not be merged because the GitHub API returned an internal server error. To enable Kodiak on this pull request please remove the `{disable_bot_label}` label. When the GitHub API returns an internal server error (HTTP status code 500), it is not safe for Kodiak to retry merging. For more information please see https://kodiakhq.com/docs/troubleshooting#merge-errors """)) await api.create_comment(body) else: await set_status("merge complete 🎉") else: priority_merge = config.merge.priority_merge_label in pull_request.labels position_in_queue = await api.queue_for_merge(first=priority_merge) if position_in_queue is None: # this case should be rare/impossible. log.warning("couldn't find position for enqueued PR") return ordinal_position = inflection.ordinalize(position_in_queue + 1) if not is_active_merge: await set_status( f"📦 enqueued for merge (position={ordinal_position})") else: log.info( "not setting status message for enqueued job because is_active_merge=True" ) return
async def mergeable( api: PRAPI, config: Union[config.V1, pydantic.ValidationError, toml.TomlDecodeError], config_str: str, config_path: str, pull_request: PullRequest, branch_protection: Optional[BranchProtectionRule], review_requests: List[PRReviewRequest], reviews: List[PRReview], contexts: List[StatusContext], check_runs: List[CheckRun], valid_signature: bool, valid_merge_methods: List[MergeMethod], merging: bool, is_active_merge: bool, skippable_check_timeout: int, api_call_retry_timeout: int, api_call_retry_method_name: Optional[str], app_id: Optional[str] = None, ) -> None: log = logger.bind( config=config, pull_request=pull_request, branch_protection=branch_protection, review_requests=review_requests, reviews=reviews, contexts=contexts, valid_signature=valid_signature, valid_merge_methods=valid_merge_methods, ) # we set is_active_merge when the PR is being merged from the merge queue. # We don't want to clobber any statuses set by that system, so we take no # action. If the PR becomes ineligible for merging that logic will handle # it. async def set_status(msg: str, markdown_content: Optional[str] = None) -> None: # don't clobber statuses set via merge loop. if is_active_merge: return await api.set_status( msg, latest_commit_sha=pull_request.latest_sha, markdown_content=markdown_content, ) if not isinstance(config, V1): log.warning("problem fetching config") await set_status( '⚠️ Invalid configuration (Click "Details" for more info.)', markdown_content=get_markdown_for_config(config, config_str=config_str, git_path=config_path), ) await api.dequeue() return if api_call_retry_timeout == 0: log.warning("timeout reached for api calls to GitHub") if api_call_retry_method_name is not None: await set_status( f"⚠️ problem contacting GitHub API with method {api_call_retry_method_name!r}" ) else: await set_status("⚠️ problem contacting GitHub API") return # if we have an app_id in the config then we only want to work on this repo # if our app_id from the environment matches the configuration. if config.app_id is not None and config.app_id != app_id: log.info("missing required app_id") await api.dequeue() return if branch_protection is None: await cfg_err( api, pull_request, f"missing branch protection for baseRef: {pull_request.baseRefName!r}", ) return if branch_protection.requiresCommitSignatures and config.merge.method in ( MergeMethod.rebase, MergeMethod.squash, ): await cfg_err( api, pull_request, '"Require signed commits" branch protection is only supported with merge commits. Squash and rebase are not supported by GitHub.', ) return if config.merge.method not in valid_merge_methods: valid_merge_methods_str = [ method.value for method in valid_merge_methods ] await cfg_err( api, pull_request, f"configured merge.method {config.merge.method.value!r} is invalid. Valid methods for repo are {valid_merge_methods_str!r}", ) return # we keep the configuration errors before the rest of the application logic # so configuration issues are surfaced as early as possible. if (pull_request.author.login in config.approve.auto_approve_usernames and pull_request.state == PullRequestState.OPEN and pull_request.mergeStateStatus != MergeStateStatus.DRAFT): # if the PR was created by an approve author and we have not previously # given an approval, approve the PR. sorted_reviews = sorted(reviews, key=lambda x: x.createdAt) kodiak_reviews = [ review for review in sorted_reviews if review.author.login == KODIAK_LOGIN ] status = review_status(kodiak_reviews) if status != PRReviewState.APPROVED: await api.approve_pull_request() else: log.info("approval already exists, not adding another") need_branch_update = (branch_protection.requiresStrictStatusChecks and pull_request.mergeStateStatus == MergeStateStatus.BEHIND) meets_label_requirement = (config.merge.automerge_label in pull_request.labels or not config.update.require_automerge_label) if (need_branch_update and not merging and config.update.always and meets_label_requirement): await set_status( "🔄 updating branch", markdown_content= "branch updated because `update.always = true` is configured.", ) await api.update_branch() return if (config.merge.require_automerge_label and config.merge.automerge_label not in pull_request.labels): await block_merge( api, pull_request, f"missing automerge_label: {config.merge.automerge_label!r}", ) return blacklist_labels = set(config.merge.blacklist_labels) & set( pull_request.labels) if blacklist_labels: await block_merge(api, pull_request, f"has blacklist_labels: {blacklist_labels!r}") return if (config.merge.blacklist_title_regex and re.search(config.merge.blacklist_title_regex, pull_request.title) is not None): await block_merge( api, pull_request, f"title matches blacklist_title_regex: {config.merge.blacklist_title_regex!r}", ) return if pull_request.mergeStateStatus == MergeStateStatus.DRAFT: await block_merge(api, pull_request, "pull request is in draft state") return if config.merge.block_on_reviews_requested and review_requests: names = [r.name for r in review_requests] await block_merge(api, pull_request, f"reviews requested: {names!r}") return if pull_request.state == PullRequestState.MERGED: log.info( "pull request merged. config.merge.delete_branch_on_merge=%r", config.merge.delete_branch_on_merge, ) await api.dequeue() if not config.merge.delete_branch_on_merge or pull_request.isCrossRepository: return pr_count = await api.pull_requests_for_ref(ref=pull_request.headRefName ) # if we couldn't access the dependent PR count or we have dependent PRs # we will abort deleting this branch. if pr_count is None or pr_count > 0: log.info("skipping branch deletion because of dependent PRs", pr_count=pr_count) return await api.delete_branch(branch_name=pull_request.headRefName) return if pull_request.state == PullRequestState.CLOSED: await api.dequeue() return if (pull_request.mergeStateStatus == MergeStateStatus.DIRTY or pull_request.mergeable == MergeableState.CONFLICTING): await block_merge(api, pull_request, "merge conflict") # remove label if configured and send message if config.merge.notify_on_conflict and config.merge.require_automerge_label: automerge_label = config.merge.automerge_label await api.remove_label(automerge_label) body = textwrap.dedent(f""" This PR currently has a merge conflict. Please resolve this and then re-add the `{automerge_label}` label. """) await api.create_comment(body) return if pull_request.mergeStateStatus == MergeStateStatus.UNSTABLE: # TODO(chdsbd): This status means that the pr is mergeable but has failing # status checks. we may want to handle this via config pass if pull_request.mergeable == MergeableState.UNKNOWN: # we need to trigger a test commit to fix this. We do that by calling # GET on the pull request endpoint. await api.trigger_test_commit() return wait_for_checks = False if pull_request.mergeStateStatus in ( MergeStateStatus.BLOCKED, MergeStateStatus.BEHIND, ): # figure out why we can't merge. There isn't a way to get this simply from the Github API. We need to find out ourselves. # # I think it's possible to find out blockers from branch protection issues # https://developer.github.com/v4/object/branchprotectionrule/?#fields # # - missing reviews # - blocking reviews # - missing required status checks # - failing required status checks # - branch not up to date (should be handled before this) # - missing required signature if (branch_protection.requiresApprovingReviews and branch_protection.requiredApprovingReviewCount): reviews_by_author: MutableMapping[ str, List[PRReview]] = defaultdict(list) for review in sorted(reviews, key=lambda x: x.createdAt): if review.author.permission not in { Permission.ADMIN, Permission.WRITE }: continue reviews_by_author[review.author.login].append(review) successful_reviews = 0 for author_name, review_list in reviews_by_author.items(): review_state = review_status(review_list) # blocking review if review_state == PRReviewState.CHANGES_REQUESTED: await block_merge(api, pull_request, f"changes requested by {author_name!r}") return # successful review if review_state == PRReviewState.APPROVED: successful_reviews += 1 # missing required review count if successful_reviews < branch_protection.requiredApprovingReviewCount: await block_merge( api, pull_request, f"missing required reviews, have {successful_reviews!r}/{branch_protection.requiredApprovingReviewCount!r}", ) return required: Set[str] = set() passing: Set[str] = set() if branch_protection.requiresStatusChecks: skippable_contexts: List[str] = [] failing_contexts: List[str] = [] pending_contexts: List[str] = [] passing_contexts: List[str] = [] required = set(branch_protection.requiredStatusCheckContexts) for status_context in contexts: # handle dont_wait_on_status_checks. We want to consider a # status_check failed if it is incomplete and in the # configuration. if (status_context.context in config.merge.dont_wait_on_status_checks and status_context.state in (StatusState.EXPECTED, StatusState.PENDING)): skippable_contexts.append(status_context.context) continue if status_context.state in (StatusState.ERROR, StatusState.FAILURE): failing_contexts.append(status_context.context) elif status_context.state in ( StatusState.EXPECTED, StatusState.PENDING, ): pending_contexts.append(status_context.context) else: assert status_context.state == StatusState.SUCCESS passing_contexts.append(status_context.context) for check_run in check_runs: if (check_run.name in config.merge.dont_wait_on_status_checks and check_run.conclusion in (None, CheckConclusionState.NEUTRAL)): skippable_contexts.append(check_run.name) continue if check_run.conclusion is None: continue if check_run.conclusion == CheckConclusionState.SUCCESS: passing_contexts.append(check_run.name) if check_run.conclusion in ( CheckConclusionState.ACTION_REQUIRED, CheckConclusionState.FAILURE, CheckConclusionState.TIMED_OUT, ): failing_contexts.append(check_run.name) passing = set(passing_contexts) failing = set(failing_contexts) # we have failing statuses that are required failing_required_status_checks = failing & required # GitHub has undocumented logic for travis-ci checks in GitHub # branch protection rules. GitHub compresses # "continuous-integration/travis-ci/{pr,push}" to # "continuous-integration/travis-ci". There is only special handling # for these specific checks. if "continuous-integration/travis-ci" in required: if "continuous-integration/travis-ci/pr" in failing: failing_required_status_checks.add( "continuous-integration/travis-ci/pr") if "continuous-integration/travis-ci/push" in failing: failing_required_status_checks.add( "continuous-integration/travis-ci/push") # either check can satisfy continuous-integration/travis-ci, but # if either fails they'll also block the merge. if ("continuous-integration/travis-ci/pr" in passing or "continuous-integration/travis-ci/push" in passing): required.remove("continuous-integration/travis-ci") if failing_required_status_checks: # NOTE(chdsbd): We need to skip this PR because it would block # the merge queue. We may be able to bump it to the back of the # queue, but it's easier just to remove it all together. There # is a similar question for the review counting. await block_merge( api, pull_request, f"failing required status checks: {failing_required_status_checks!r}", ) return if skippable_contexts: if merging: if skippable_check_timeout > 0: await set_status( f"⛴ merging PR (waiting a bit for dont_wait_on_status_checks: {skippable_contexts!r})" ) raise RetryForSkippableChecks log.warning( "timeout reached waiting for dont_wait_on_status_checks", skippable_contexts=skippable_contexts, ) await set_status( f"⚠️ timeout reached for dont_wait_on_status_checks: {skippable_contexts!r}" ) await set_status( f"🛑 not waiting for dont_wait_on_status_checks: {skippable_contexts!r}" ) return missing_required_status_checks = required - passing wait_for_checks = bool(branch_protection.requiresStatusChecks and missing_required_status_checks) if config.merge.update_branch_immediately and need_branch_update: await set_status( "🔄 updating branch", markdown_content= "branch updated because `merge.update_branch_immediately = true` is configured.", ) await api.update_branch() return if merging: # prioritize branch updates over waiting for status checks to complete if config.merge.optimistic_updates: if need_branch_update: await set_status("⛴ merging PR (updating branch)") await api.update_branch() raise PollForever if wait_for_checks: await set_status( f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})" ) raise PollForever # almost the same as the pervious case, but we prioritize status checks # over branch updates. else: if wait_for_checks: await set_status( f"⛴ merging PR (waiting for status checks: {missing_required_status_checks!r})" ) raise PollForever if need_branch_update: await set_status("⛴ merging PR (updating branch)") await api.update_branch() raise PollForever # if we reach this point and we don't need to wait for checks or update a branch we've failed to calculate why the PR is blocked. This should _not_ happen normally. if not (wait_for_checks or need_branch_update): await block_merge(api, pull_request, "Merging blocked by GitHub requirements") log.warning("merge blocked for unknown reason") return ready_to_merge = not (wait_for_checks or need_branch_update) if config.merge.do_not_merge: if wait_for_checks: await set_status( f"⌛️ waiting for required status checks: {missing_required_status_checks!r}" ) elif need_branch_update: await set_status( "⚠️ need branch update (suggestion: use merge.update_branch_immediately with merge.do_not_merge)", markdown_content="""\ When `merge.do_not_merge = true` is configured `merge.update_branch_immediately = true` \ is recommended so Kodiak can automatically update branches. By default, Kodiak is efficient and only update branches when merging a PR, but \ when `merge.do_not_merge` is enabled, Kodiak never has that opportunity to \ update a branch during merge. `merge.update_branch_immediately = true` will \ trigger Kodiak to update branches whenever a PR is outdated and not failing any \ branch protection requirements. """, ) else: await set_status("✅ okay to merge") log.info( "eligible to merge, stopping because config.merge.do_not_merge is enabled." ) return # okay to merge if we reach this point. if (config.merge.prioritize_ready_to_merge and ready_to_merge) or merging: merge_args = get_merge_body(config, pull_request) await set_status("⛴ attempting to merge PR (merging)") await api.merge( merge_method=merge_args.merge_method, commit_title=merge_args.commit_title, commit_message=merge_args.commit_message, ) else: position_in_queue = await api.queue_for_merge() if position_in_queue is None: # this case should be rare/impossible. log.warning("couldn't find position for enqueued PR") return ordinal_position = inflection.ordinalize(position_in_queue + 1) if not is_active_merge: await set_status( f"📦 enqueued for merge (position={ordinal_position})") else: log.info( "not setting status message for enqueued job because is_active_merge=True" ) return
def test_start_end(self): m = rure.search(u"remove_this", u"tony@tiremove_thisger.net") self.assertEqual( (m.string[:m.start()] + m.string[m.end():]).decode('utf8'), u'*****@*****.**')
def fix_vba_code(vba_code): """ Fix up some substrings that ViperMonkey has problems parsing. """ # Strip comment lines from the code. vba_code = strip_comments(vba_code) # Fix dumb typo in some maldocs VBA. vba_code = vba_code.replace("End SubPrivate", "End Sub\nPrivate") # No null bytes in VB to process. vba_code = vba_code.replace("\x00", "") # Make "End Try" in try/catch blocks easier to parse. vba_code = re.sub(r"End\s+Try", "##End ##Try", vba_code) # We don't handle Line Input constructs for now. Delete them. # TODO: Actually handle Line Input consructs. linputs = re.findall(r"Line\s+Input\s+#\d+\s*,\s*\w+", vba_code, re.DOTALL) if (len(linputs) > 0): log.warning("VB Line Input constructs are not currently handled. Stripping them from code...") for linput in linputs: vba_code = vba_code.replace(linput, "") # We don't handle Property constructs for now. Delete them. # TODO: Actually handle Property consructs. props = re.findall(r"(?:Public\s+|Private\s+|Friend\s+)?Property\s+.+?End\s+Property", vba_code, re.DOTALL) if (len(props) > 0): log.warning("VB Property constructs are not currently handled. Stripping them from code...") for prop in props: vba_code = vba_code.replace(prop, "") # We don't handle Implements constructs for now. Delete them. # TODO: Figure out if we need to worry about Implements. implements = re.findall(r"Implements \w+", vba_code, re.DOTALL) if (len(implements) > 0): log.warning("VB Implements constructs are not currently handled. Stripping them from code...") for imp in implements: vba_code = vba_code.replace(imp, "") # We don't handle Enum constructs for now. Delete them. # TODO: Actually handle Enum consructs. enums = re.findall(r"(?:(?:Public|Private)\s+)?Enum\s+.+?End\s+Enum", vba_code, re.DOTALL) if (len(enums) > 0): log.warning("VB Enum constructs are not currently handled. Stripping them from code...") for enum in enums: vba_code = vba_code.replace(enum, "") # We don't handle ([a1]) constructs for now. Delete them. # TODO: Actually handle these things. brackets = re.findall(r"\(\[[^\]]+\]\)", vba_code, re.DOTALL) if (len(brackets) > 0): log.warning("([a1]) style constructs are not currently handled. Rewriting them...") for bracket in brackets: vba_code = vba_code.replace(bracket, "(" + bracket[2:-2] + ")") # Clear out lines broken up on multiple lines. vba_code = re.sub(r" _ *\r?\n", "", vba_code) vba_code = re.sub(r"&_ *\r?\n", "&", vba_code) vba_code = re.sub(r"\(_ *\r?\n", "(", vba_code) #vba_code = re.sub(r":\s*[Ee]nd\s+[Ss]ub", r"\nEnd Sub", vba_code) vba_code = "\n" + vba_code vba_code = re.sub(r"\n:", "\n", vba_code) # Some maldocs have single line member access expressions that end with a '.'. # Comment those out. dumb_member_exps = re.findall(r"\n(?:\w+\.)+\n", vba_code) for dumb_exp in dumb_member_exps: log.warning("Commenting out bad line '" + dumb_exp.replace("\n", "") + "'.") safe_exp = "\n'" + dumb_exp[1:] vba_code = vba_code.replace(dumb_exp, safe_exp) # How about maldocs with Subs with spaces in their names? space_subs = re.findall(r"\n\s*Sub\s*\w+\s+\w+\s*\(", vba_code) for space_sub in space_subs: start = space_sub.index("Sub") + len("Sub") end = space_sub.rindex("(") sub_name = space_sub[start:end] new_name = sub_name.replace(" ", "_") log.warning("Replacing bad sub name '" + sub_name + "' with '" + new_name + "'.") vba_code = vba_code.replace(sub_name, new_name) # Clear out some garbage characters. #vba_code = vba_code.replace('\x0b', '') #vba_code = vba_code.replace('\x88', '') # It looks like VBA supports variable and function names containing # non-ASCII characters. Parsing these with pyparsing would be difficult # (or impossible), so convert the non-ASCII names to ASCII. # # Break up lines with multiple statements onto their own lines. vba_code = fix_difficult_code(vba_code) # Fix function calls with a skipped 1st argument. vba_code = fix_skipped_1st_arg(vba_code) # Fix lines with missing double quotes. vba_code = fix_unbalanced_quotes(vba_code) # For each const integer defined, replace it inline in the code to reduce lookups vba_code = replace_constant_int_inline(vba_code) # Skip the next part if unnneeded. uni_vba_code = None try: uni_vba_code = vba_code.decode("utf-8") except UnicodeDecodeError: # Punt. return vba_code got_multassign = (re2.search(u"(?:\w+\s*=\s*){2}", uni_vba_code) is not None) if ((" if+" not in vba_code) and (" If+" not in vba_code) and ("\nif+" not in vba_code) and ("\nIf+" not in vba_code) and (not got_multassign)): return vba_code # Change things like 'If+foo > 12 ..." to "If foo > 12 ...". r = "" for line in vba_code.split("\n"): # Fix up assignments like 'cat = dog = frog = 12'. line = fix_multiple_assignments(line) # Do we have an "if+..."? if ("if+" not in line.lower()): # No. No change. r += line + "\n" continue # Yes we do. Figure out if it is in a string. in_str = False window = " " new_line = "" for c in line: # Start/End of string? if (c == '"'): in_str = not in_str # Have we seen an if+ ? if ((not in_str) and (c == "+") and (window.lower() == " if")): # Replace the '+' with a ' '. new_line += " " # No if+ . else: new_line += c # Advance the viewing window. window = window[1:] + c # Save the updated line. r += new_line + "\n" # Return the updated code. return r