def _report_error(self, content):
content = sanitize_sensitive_data(content)
if self.context.pr_id:
pr = PullRequest(bitbucket, self.context.repository)
pr.comment(self.context.pr_id, content)
else:
cs = Changesets(bitbucket, self.context.repository)
cs.comment(self.commit_hash, content)
def handle_pull_request_approved(payload):
if not current_app.config['AUTO_MERGE_ENABLED']:
return
repo = payload['repository']
pr = payload['pullrequest']
pr_id = pr['id']
title = pr['title'].lower()
description = (pr['description'] or '').lower()
for keyword in ('wip', 'merge skip', 'working in progress'):
if keyword in title or keyword in description:
logger.info('%s found, ignore auto merge.', keyword)
return
pull_request = PullRequest(
bitbucket,
repo['full_name']
)
try:
pr_info = pull_request.get(pr_id)
except BitbucketAPIError as exc:
logger.exception('Error calling Bitbucket API')
if exc.code != 404:
sentry.captureException()
return
if pr_info['state'] != 'OPEN':
return
participants = pr_info['participants']
approved_users = [u for u in participants if u['approved']]
if len(approved_users) < current_app.config['AUTO_MERGE_APPROVAL_COUNT']:
return
commit_hash = pr_info['source']['commit']['hash']
build_status = BuildStatus(
bitbucket,
pr_info['source']['repository']['full_name'],
commit_hash,
'badwolf/test',
url_for('log.build_log', sha=commit_hash, _external=True)
)
message = 'Auto merge pull request #{}: {}'.format(pr_id, pr['title'])
if description:
message += '\n\n{}'.format(pr['description'])
try:
status = build_status.get()
if status['state'] == 'SUCCESSFUL':
pull_request.merge(pr_id, message)
except BitbucketAPIError as exc:
logger.exception('Error calling Bitbucket API')
if exc.code != 404:
sentry.captureException()
def __init__(self, context, spec, working_dir=None):
self.context = context
self.spec = spec
self.working_dir = working_dir or context.clone_path
self.problems = Problems()
self.pr = PullRequest(bitbucket, context.repository)
commit_hash = context.source['commit']['hash']
self.build_status = BuildStatus(
bitbucket, context.source['repository']['full_name'], commit_hash,
'badwolf/lint', 'https://bitbucket.org/{}/pull-requests/{}'.format(
context.repository, context.pr_id))
def __init__(self, context, spec, working_dir):
self.context = context
self.spec = spec
self.working_dir = working_dir
self.problems = Problems()
self.pr = PullRequest(bitbucket, context.repository)
commit_hash = context.source['commit']['hash']
self.build_status = BuildStatus(
bitbucket, context.source['repository']['full_name'], commit_hash,
'badwolf/lint',
url_for('log.lint_log', sha=commit_hash, _external=True))
def _report_git_error(self, exc):
self.build_status.update('FAILED',
description='Git clone repository failed')
content = ':broken_heart: **Git error**: {}'.format(to_text(exc))
content = sanitize_sensitive_data(content)
if self.context.pr_id:
pr = PullRequest(bitbucket, self.context.repository)
pr.comment(self.context.pr_id, content)
else:
cs = Changesets(bitbucket, self.context.repository)
cs.comment(self.commit_hash, content)
def check_pr_mergeable(context):
repo = bitbucket.get('2.0/repositories/{}'.format(context.repository))
main_branch = repo['mainbranch']['name']
current_branch = context.source['branch']['name']
if current_branch != 'master' and current_branch != main_branch:
logger.info('Current branch %s is not main branch %s', current_branch,
main_branch)
return
time.sleep(5) # wait for Bitbucket merge process ready
pr = PullRequest(bitbucket, context.repository)
open_prs = pr.list(state='OPEN')['values']
if not open_prs:
logger.debug('No opening pull requests found')
return
for open_pr in open_prs:
check_mergeable(context, pr, open_pr)
class LintProcessor(object):
LINTERS = {
'eslint': ESLinter,
'flake8': Flake8Linter,
'pep8': PyCodeStyleLinter,
'pycodestyle': PyCodeStyleLinter,
'csslint': CSSLinter,
'shellcheck': ShellCheckLinter,
'jsonlint': JSONLinter,
'yamllint': YAMLLinter,
'bandit': BanditLinter,
'rstlint': RestructuredTextLinter,
'pylint': PylintLinter,
'sasslint': SassLinter,
'stylelint': StyleLinter,
'mypy': MypyLinter,
}
def __init__(self, context, spec, working_dir=None):
self.context = context
self.spec = spec
self.working_dir = working_dir or context.clone_path
self.problems = Problems()
self.pr = PullRequest(bitbucket, context.repository)
commit_hash = context.source['commit']['hash']
self.build_status = BuildStatus(
bitbucket, context.source['repository']['full_name'], commit_hash,
'badwolf/lint', 'https://bitbucket.org/{}/pull-requests/{}'.format(
context.repository, context.pr_id))
def load_changes(self):
try:
changes = self.pr.diff(self.context.pr_id)
except (BitbucketAPIError, UnidiffParseError):
logger.exception('Error getting pull request diff from API')
sentry.captureException()
return
self.problems.set_changes(changes)
return changes
def process(self):
if not self.spec.linters:
logger.info('No linters configured, ignore lint.')
return
logger.info('Running code linting')
patch = self.load_changes()
if not patch:
logger.info('Load changes failed, ignore lint.')
return
lint_files = patch.added_files + patch.modified_files
if not lint_files:
logger.info('No changed files found, ignore lint')
return
self.update_build_status('INPROGRESS', 'Lint in progress')
files = [f.path for f in lint_files]
self._execute_linters(files)
total_problems = len(self.problems)
self.problems.limit_to_changes()
in_diff_problems = len(self.problems)
# Report error and cleanup outdated lint comments
submitted_problems, fixed_problems = self._report()
if total_problems > 0:
if in_diff_problems == total_problems:
description = 'Found {} new issues'.format(total_problems)
else:
description = 'Found {} issues'.format(total_problems)
description += ', {} issues in diff'.format(in_diff_problems)
if submitted_problems > 0:
description += ', {} new issues'.format(submitted_problems)
if fixed_problems > 0:
description += ' {} issues fixed'.format(fixed_problems)
else:
description = 'No code issues found'
has_error = any(p for p in self.problems if p.is_error)
if has_error:
logger.info('Lint failed: %s', description)
self.update_build_status('FAILED', description)
else:
logger.info('Lint successful: %s', description)
self.update_build_status('SUCCESSFUL', description)
def _execute_linters(self, files):
for linter_option in self.spec.linters:
name = linter_option.name
linter_cls = self.LINTERS.get(name)
if not linter_cls:
logger.info('Linter %s not found, ignore.', name)
continue
linter = linter_cls(self.working_dir, self.problems, linter_option)
if not linter.is_usable():
logger.info('Linter %s is not usable, ignore.', name)
continue
logger.info('Running %s code linter', name)
linter.execute(files)
def _report(self):
try:
comments = self.pr.all_comments(self.context.pr_id)
except BitbucketAPIError:
logger.exception('Error fetching all comments for pull request')
sentry.captureException()
comments = []
existing_comments_ids = {}
for comment in comments:
inline = comment.get('inline')
if not inline:
continue
raw = comment['content']['raw']
if not raw.startswith(':broken_heart: **'):
continue
filename = inline['path']
line = inline['to'] or inline['from']
if line is None:
continue
existing_comments_ids[(filename, line, raw)] = comment['id']
if len(self.problems) == 0:
return 0, 0
revision_before = self.context.target['commit']['hash']
revision_after = self.context.source['commit']['hash']
lint_comments = set()
problem_count = 0
for problem in self.problems:
content = ':broken_heart: **{}**: {}'.format(
problem.linter, problem.message)
comment_tuple = (problem.filename, problem.line, content)
lint_comments.add(comment_tuple)
if comment_tuple in existing_comments_ids:
continue
comment_kwargs = {
'filename': problem.filename,
'anchor': revision_after,
'dest_rev': revision_before,
}
if problem.has_line_change:
comment_kwargs['line_to'] = problem.line
else:
comment_kwargs['line_from'] = problem.line
try:
self.pr.comment(self.context.pr_id, content, **comment_kwargs)
except BitbucketAPIError:
logger.exception(
'Error creating inline comment for pull request')
sentry.captureException()
else:
problem_count += 1
logger.info('Code lint result: %d problems found, %d submitted',
len(self.problems), problem_count)
outdated_cleaned = 0
outdated_comments = set(existing_comments_ids.keys()) - lint_comments
logger.info('%d outdated lint comments found', len(outdated_comments))
for comment in outdated_comments:
# Delete comment
try:
self.pr.delete_comment(self.context.pr_id,
existing_comments_ids[comment])
outdated_cleaned += 1
except BitbucketAPIError:
logger.exception('Error deleting pull request comment')
sentry.captureException()
return problem_count, outdated_cleaned
def update_build_status(self, state, description=None):
try:
self.build_status.update(state, description=description)
except BitbucketAPIError:
logger.exception('Error calling Bitbucket API')
sentry.captureException()
class LintProcessor(object):
LINTERS = {
'eslint': ESLinter,
'flake8': Flake8Linter,
'jscs': JSCSLinter,
'pep8': PEP8Linter,
'csslint': CSSLinter,
'shellcheck': ShellCheckLinter,
'jsonlint': JSONLinter,
'yamllint': YAMLLinter,
'bandit': BanditLinter,
'rstlint': RestructuredTextLinter,
'pylint': PylintLinter,
'sasslint': SassLinter,
'stylelint': StyleLinter,
}
def __init__(self, context, spec, working_dir):
self.context = context
self.spec = spec
self.working_dir = working_dir
self.problems = Problems()
self.pr = PullRequest(bitbucket, context.repository)
commit_hash = context.source['commit']['hash']
self.build_status = BuildStatus(
bitbucket, context.source['repository']['full_name'], commit_hash,
'badwolf/lint',
url_for('log.lint_log', sha=commit_hash, _external=True))
def load_changes(self):
try:
changes = self.pr.diff(self.context.pr_id)
except (BitbucketAPIError, UnidiffParseError):
logger.exception('Error getting pull request diff from API')
return
self.problems.set_changes(changes)
return changes
def process(self):
if not self.spec.linters:
logger.info('No linters configured, ignore lint.')
return
logger.info('Running code linting')
patch = self.load_changes()
if not patch:
logger.info('Load changes failed, ignore lint.')
return
lint_files = patch.added_files + patch.modified_files
if not lint_files:
logger.info('No changed files found, ignore lint')
return
self.update_build_status('INPROGRESS', 'Lint in progress')
files = [f.path for f in lint_files]
self._execute_linters(files)
logger.info('%d problems found before limit to changes',
len(self.problems))
self.problems.limit_to_changes()
has_error = any(p for p in self.problems if p.is_error)
if len(self.problems):
description = 'Found {} code issues'.format(len(self.problems))
else:
description = 'No code issues found'
logger.info('No problems found when linting codes')
# Report error or cleanup lint
self._report()
if has_error:
self.update_build_status('FAILED', description)
else:
self.update_build_status('SUCCESSFUL', description)
def _execute_linters(self, files):
for linter_option in self.spec.linters:
name = linter_option.name
linter_cls = self.LINTERS.get(name)
if not linter_cls:
logger.info('Linter %s not found, ignore.', name)
continue
linter = linter_cls(self.working_dir, self.problems, linter_option)
if not linter.is_usable():
logger.info('Linter %s is not usable, ignore.', name)
continue
logger.info('Running %s code linter', name)
linter.execute(files)
def _report(self):
try:
comments = self.pr.all_comments(self.context.pr_id)
except BitbucketAPIError:
logger.exception('Error fetching all comments for pull request')
comments = []
hash_set = set()
for comment in comments:
inline = comment.get('inline')
if not inline:
continue
raw = comment['content']['raw']
if self.context.cleanup_lint and raw.startswith(':broken_heart:'):
# Delete comment
try:
self.pr.delete_comment(self.context.pr_id, comment['id'])
except BitbucketAPIError:
logger.exception('Error deleting pull request comment')
else:
filename = inline['path']
line_to = inline['to']
hash_set.add(hash('{}{}{}'.format(filename, line_to, raw)))
if len(self.problems) == 0:
return
revision_before = self.context.target['commit']['hash']
revision_after = self.context.source['commit']['hash']
problem_count = 0
for problem in self.problems:
content = ':broken_heart: **{}**: {}'.format(
problem.linter, problem.message)
comment_hash = hash('{}{}{}'.format(
problem.filename,
problem.line,
content,
))
if comment_hash in hash_set:
continue
try:
self.pr.comment(
self.context.pr_id,
content,
line_to=problem.line,
filename=problem.filename,
anchor=revision_after,
dest_rev=revision_before,
)
except BitbucketAPIError:
logger.exception(
'Error creating inline comment for pull request')
else:
problem_count += 1
logger.info('Code lint result: %d problems found, %d submited',
len(self.problems), problem_count)
return problem_count
def update_build_status(self, state, description=None):
try:
self.build_status.update(state, description=description)
except BitbucketAPIError:
logger.exception('Error calling Bitbucket API')
def run(self):
start_time = time.time()
self.branch = self.context.source['branch']['name']
try:
self.clone_repository()
except git.GitCommandError as e:
logger.exception('Git command error')
self.update_build_status('FAILED', 'Git clone repository failed')
content = ':broken_heart: **Git error**: {}'.format(to_text(e))
if self.context.pr_id:
pr = PullRequest(bitbucket, self.repo_full_name)
pr.comment(self.context.pr_id, content)
else:
cs = Changesets(bitbucket, self.repo_full_name)
cs.comment(self.commit_hash, content)
self.cleanup()
return
if not self.validate_settings():
self.cleanup()
return
context = {
'context':
self.context,
'task_id':
self.task_id,
'build_log_url':
url_for('log.build_log', sha=self.commit_hash, _external=True),
'branch':
self.branch,
'scripts':
self.spec.scripts,
}
if self.spec.scripts:
self.update_build_status('INPROGRESS', 'Test in progress')
docker_image_name, build_output = self.get_docker_image()
context['build_logs'] = to_text(build_output)
context.update({
'build_logs': to_text(build_output),
'elapsed_time': int(time.time() - start_time),
})
if not docker_image_name:
self.update_build_status('FAILED',
'Build or get Docker image failed')
context['exit_code'] = -1
self.send_notifications(context)
self.cleanup()
return
exit_code, output = self.run_tests_in_container(docker_image_name)
if exit_code == 0:
# Success
logger.info('Test succeed for repo: %s', self.repo_full_name)
self.update_build_status('SUCCESSFUL', '1 of 1 test succeed')
else:
# Failed
logger.info('Test failed for repo: %s, exit code: %s',
self.repo_full_name, exit_code)
self.update_build_status('FAILED', '1 of 1 test failed')
context.update({
'logs': to_text(output),
'exit_code': exit_code,
'elapsed_time': int(time.time() - start_time),
})
self.send_notifications(context)
# Code linting
if self.context.pr_id and self.spec.linters:
lint = LintProcessor(self.context, self.spec, self.clone_path)
lint.process()
self.cleanup()
