def run_tests_in_container(self, docker_image_name):
command = '/bin/sh -c badwolf-run'
environment = {}
if self.spec.environments:
# TODO: Support run in multiple environments
environment = self.spec.environments[0]
# TODO: Add more test context related env vars
environment.update({
'DEBIAN_FRONTEND': 'noninteractive',
'CI': 'true',
'CI_NAME': 'badwolf',
'BADWOLF_BRANCH': self.branch,
'BADWOLF_COMMIT': self.commit_hash,
'BADWOLF_BUILD_DIR': '/mnt/src',
'BADWOLF_REPO_SLUG': self.repo_full_name,
})
if self.context.pr_id:
environment['BADWOLF_PULL_REQUEST'] = to_text(self.context.pr_id)
container = self.docker.create_container(
docker_image_name,
command=command,
environment=environment,
working_dir='/mnt/src',
volumes=['/mnt/src'],
host_config=self.docker.create_host_config(
privileged=self.spec.privileged,
binds={
self.clone_path: {
'bind': '/mnt/src',
'mode': 'rw',
},
}))
container_id = container['Id']
logger.info('Created container %s from image %s', container_id,
docker_image_name)
output = []
try:
self.docker.start(container_id)
self.update_build_status('INPROGRESS',
'Running tests in Docker container')
for line in self.docker.logs(container_id, stream=True):
output.append(to_text(line))
exit_code = self.docker.wait(
container_id, current_app.config['DOCKER_RUN_TIMEOUT'])
except (APIError, DockerException, ReadTimeout) as e:
exit_code = -1
output.append(to_text(e))
logger.exception('Docker error')
finally:
try:
self.docker.remove_container(container_id, force=True)
except (APIError, DockerException):
logger.exception('Error removing docker container')
return exit_code, ''.join(output)
def __init__(self,
repository,
actor,
type,
message,
source,
target=None,
rebuild=False,
pr_id=None,
nocache=False,
clone_depth=50):
self.task_id = to_text(uuid.uuid4())
self.repository = repository
self.repo_name = repository.split('/')[-1]
self.actor = actor
self.type = type
self.message = message
self.source = source
self.target = target
self.rebuild = rebuild
self.pr_id = pr_id
# Don't use cache when build Docker image
self.nocache = nocache
self.clone_depth = clone_depth
if 'repository' not in self.source:
self.source['repository'] = {'full_name': repository}
self.clone_path = os.path.join(
current_app.config['BADWOLF_REPO_DIR'],
self.repo_name,
self.task_id,
)
def __init__(self, repository, actor, type, message, source,
target=None, rebuild=False, pr_id=None, cleanup_lint=False,
nocache=False, clone_depth=50):
self.task_id = to_text(uuid.uuid4())
self.repository = repository
self.repo_name = repository.split('/')[-1]
self.actor = actor
self.type = type
self.message = message
self.source = source
self.target = target
self.rebuild = rebuild
self.pr_id = pr_id
self.cleanup_lint = cleanup_lint
# Don't use cache when build Docker image
self.nocache = nocache
self.clone_depth = clone_depth
if 'repository' not in self.source:
self.source['repository'] = {'full_name': repository}
self.clone_path = os.path.join(
TMP_PATH,
'badwolf',
self.task_id,
self.repo_name,
)
def test_repo_push_ci_skip_found(test_client):
payload = json.dumps({
'repository': {
'full_name': 'deepanalyzer/badwolf',
'scm': 'git',
},
'push': {
'changes': [
{
'new': {
'type': 'branch',
},
'commits': [
{
'hash': '2cedc1af762',
'message': 'Test [ci skip]',
}
],
}
]
}
})
res = test_client.post(
url_for('webhook.webhook_push'),
data=payload,
headers={
'User-Agent': 'Bitbucket-Webhooks/2.0',
'X-Event-Key': 'repo:push',
}
)
assert res.status_code == 200
assert to_text(res.data) == ''
def test_pr_updated_state_not_open(test_client):
payload = json.dumps({
'repository': {
'full_name': 'deepanalyzer/badwolf',
'scm': 'git',
},
'pullrequest': {
'title': 'Test PR',
'description': '',
'state': 'MERGED',
},
'source': {
'repository': {'full_name': 'deepanalyzer/badwolf'},
'branch': {'name': 'develop'},
'commit': {'hash': 'abc'}
},
'target': {
'repository': {'full_name': 'deepanalyzer/badwolf'},
'branch': {'name': 'master'},
'commit': {'hash': 'abc'}
}
})
res = test_client.post(
url_for('webhook.webhook_push'),
data=payload,
headers={
'X-Event-Key': 'pullrequest:updated',
}
)
assert res.status_code == 200
assert to_text(res.data) == ''
def test_repo_push_unsupported_scm(test_client):
payload = json.dumps({
'repository': {
'full_name': 'deepanalyzer/badwolf',
'scm': 'hg',
},
'push': {
'changes': [
{
'new': {
'type': 'branch',
}
}
]
}
})
res = test_client.post(
url_for('webhook.webhook_push'),
data=payload,
headers={
'User-Agent': 'Bitbucket-Webhooks/2.0',
'X-Event-Key': 'repo:push',
}
)
assert res.status_code == 200
assert to_text(res.data) == ''
def test_repo_push_no_new_changes(test_client):
payload = json.dumps({'push': {'changes': []}})
res = test_client.post(url_for('webhook.webhook_push'),
data=payload,
headers={
'X-Event-Key': 'repo:push',
})
assert res.status_code == 200
assert to_text(res.data) == ''
def test_parse_secure_env(app):
s = """env:
- secure: {}""".format(to_text(SecureToken.encrypt('X=1 Y=2 Z=3')))
f = io.StringIO(s)
spec = Specification.parse_file(f)
assert len(spec.environments) == 1
env0 = spec.environments[0]
assert env0['X'] == '1'
assert env0['Y'] == '2'
assert env0['Z'] == '3'
def _report_git_error(self, exc):
self.build_status.update('FAILED',
description='Git clone repository failed')
content = ':broken_heart: **Git error**: {}'.format(to_text(exc))
content = sanitize_sensitive_data(content)
if self.context.pr_id:
pr = PullRequest(bitbucket, self.context.repository)
pr.comment(self.context.pr_id, content)
else:
cs = Changesets(bitbucket, self.context.repository)
cs.comment(self.commit_hash, content)
def test_pr_created_unsupported_scm(test_client):
payload = json.dumps({
'repository': {
'full_name': 'deepanalyzer/badwolf',
'scm': 'hg',
},
})
res = test_client.post(url_for('webhook.webhook_push'),
data=payload,
headers={
'X-Event-Key': 'pullrequest:created',
})
assert res.status_code == 200
assert to_text(res.data) == ''
def __init__(self,
repository,
actor,
type,
message,
source,
target=None,
rebuild=False,
pr_id=None,
nocache=False,
clone_depth=50,
skip_lint=False):
self.task_id = to_text(uuid.uuid4())
self.repository = repository
self.repo_owner, self.repo_name = repository.split('/')
self.actor = actor
self.type = type
self.message = message
self.source = source
self.target = target
self.rebuild = rebuild
self.pr_id = pr_id
# Don't use cache when build Docker image
self.nocache = nocache
self.clone_depth = clone_depth
self.skip_lint = skip_lint
if 'repository' not in self.source:
self.source['repository'] = {'full_name': repository}
self.clone_path = os.path.join(
current_app.config['BADWOLF_REPO_DIR'],
self.repo_name,
self.task_id,
)
self.environment = {
'DEBIAN_FRONTEND': 'noninteractive',
'CI': 'true',
'CI_NAME': 'badwolf',
'BADWOLF_BUILD_DIR': self.clone_path,
'BADWOLF_REPO_SLUG': repository,
'BADWOLF_COMMIT': source['commit']['hash'],
}
if type == 'tag':
self.environment['BADWOLF_TAG'] = source['branch']['name']
else:
self.environment['BADWOLF_BRANCH'] = source['branch']['name']
if pr_id:
self.environment['BADWOLF_PULL_REQUEST'] = str(pr_id)
def get_docker_image(self):
docker_image_name = self.repo_full_name.replace('/', '-')
output = []
with self.lock:
docker_image = self.docker.images(docker_image_name)
if not docker_image or self.context.rebuild:
dockerfile = os.path.join(self.clone_path,
self.spec.dockerfile)
build_options = {
'tag': docker_image_name,
'rm': True,
}
if not os.path.exists(dockerfile):
logger.warning(
'No Dockerfile: %s found for repo: %s, using simple runner image',
dockerfile, self.repo_full_name)
dockerfile_content = 'FROM messense/badwolf-test-runner\n'
fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
build_options['fileobj'] = fileobj
else:
build_options['dockerfile'] = self.spec.dockerfile
build_success = False
logger.info('Building Docker image %s', docker_image_name)
self.update_build_status('INPROGRESS', 'Building Docker image')
res = self.docker.build(self.clone_path, **build_options)
for line in res:
if b'Successfully built' in line:
build_success = True
log = to_text(json.loads(to_text(line))['stream'])
output.append(log)
logger.info('`docker build` : %s', log.strip())
if not build_success:
return None, ''.join(output)
return docker_image_name, ''.join(output)
def test_unhandled_event(test_client):
payload = json.dumps({
'push': {
'changes': [
]
}
})
res = test_client.post(
url_for('webhook.webhook_push'),
data=payload,
headers={
'User-Agent': 'Bitbucket-Webhooks/2.0',
'X-Event-Key': 'repo:created',
}
)
assert res.status_code == 200
assert to_text(res.data) == ''
def test_pr_updated_ci_skip_found(test_client):
payload = json.dumps({
'repository': {
'full_name': 'deepanalyzer/badwolf',
'scm': 'git',
},
'pullrequest': {
'title': 'Test PR',
'description': 'ci skip',
}
})
res = test_client.post(url_for('webhook.webhook_push'),
data=payload,
headers={
'X-Event-Key': 'pullrequest:updated',
})
assert res.status_code == 200
assert to_text(res.data) == ''
def lint_files(self, files):
command = [
self.python_name, '-m', 'pylint', '-r', 'n', '-f', 'parseable'
]
command += files
_, output = run_command(command,
split=True,
include_errors=True,
cwd=self.working_dir)
if not output:
raise StopIteration()
for line in output:
parsed = self._parse_line(to_text(line))
if parsed is None:
continue
filename, line, message = parsed
yield Problem(filename, line, message, self.name)
def test_repo_push_unsupported_push_type(test_client):
payload = json.dumps({
'repository': {
'full_name': 'deepanalyzer/badwolf',
'scm': 'git',
},
'push': {
'changes': [{
'new': {
'type': 'wrong_push_type',
}
}]
}
})
res = test_client.post(url_for('webhook.webhook_push'),
data=payload,
headers={
'X-Event-Key': 'repo:push',
})
assert res.status_code == 200
assert to_text(res.data) == ''
def shell_script(self):
def _trace(command):
return 'echo + {}\n{} '.format(shlex.quote(command), command)
commands = []
after_success = [_trace(cmd) for cmd in self.after_success]
after_failure = [_trace(cmd) for cmd in self.after_failure]
for service in self.services:
commands.append(_trace('service {} start'.format(service)))
for script in self.scripts:
commands.append(_trace(script))
command_encoded = shlex.quote(
to_text(base64.b64encode(to_binary('\n'.join(commands)))))
context = {
'command': command_encoded,
'after_success': ' \n'.join(after_success),
'after_failure': ' \n'.join(after_failure),
}
script = render_template('script.sh', **context)
logger.debug('Build script: \n%s', script)
return script
def run(self):
start_time = time.time()
self.branch = self.context.source['branch']['name']
try:
self.clone_repository()
except git.GitCommandError as e:
logger.exception('Git command error')
self.update_build_status('FAILED', 'Git clone repository failed')
content = ':broken_heart: **Git error**: {}'.format(to_text(e))
if self.context.pr_id:
pr = PullRequest(bitbucket, self.repo_full_name)
pr.comment(self.context.pr_id, content)
else:
cs = Changesets(bitbucket, self.repo_full_name)
cs.comment(self.commit_hash, content)
self.cleanup()
return
if not self.validate_settings():
self.cleanup()
return
context = {
'context':
self.context,
'task_id':
self.task_id,
'build_log_url':
url_for('log.build_log', sha=self.commit_hash, _external=True),
'branch':
self.branch,
'scripts':
self.spec.scripts,
}
if self.spec.scripts:
self.update_build_status('INPROGRESS', 'Test in progress')
docker_image_name, build_output = self.get_docker_image()
context['build_logs'] = to_text(build_output)
context.update({
'build_logs': to_text(build_output),
'elapsed_time': int(time.time() - start_time),
})
if not docker_image_name:
self.update_build_status('FAILED',
'Build or get Docker image failed')
context['exit_code'] = -1
self.send_notifications(context)
self.cleanup()
return
exit_code, output = self.run_tests_in_container(docker_image_name)
if exit_code == 0:
# Success
logger.info('Test succeed for repo: %s', self.repo_full_name)
self.update_build_status('SUCCESSFUL', '1 of 1 test succeed')
else:
# Failed
logger.info('Test failed for repo: %s, exit code: %s',
self.repo_full_name, exit_code)
self.update_build_status('FAILED', '1 of 1 test failed')
context.update({
'logs': to_text(output),
'exit_code': exit_code,
'elapsed_time': int(time.time() - start_time),
})
self.send_notifications(context)
# Code linting
if self.context.pr_id and self.spec.linters:
lint = LintProcessor(self.context, self.spec, self.clone_path)
lint.process()
self.cleanup()
def decrypt(encrypted):
fernet = Fernet(to_binary(current_app.config['SECURE_TOKEN_KEY']))
text = fernet.decrypt(to_binary(encrypted))
return to_text(text)
def run_in_container(self, docker_image_name):
environment = {}
if self.spec.environments:
# TODO: Support run in multiple environments
environment = self.spec.environments[0]
# TODO: Add more test context related env vars
script = shlex.quote(
to_text(base64.b64encode(to_binary(self.spec.shell_script))))
environment.update({
'DEBIAN_FRONTEND': 'noninteractive',
'HOME': '/root',
'SHELL': '/bin/sh',
'CI': 'true',
'CI_NAME': 'badwolf',
'BADWOLF_COMMIT': self.commit_hash,
'BADWOLF_BUILD_DIR': self.context.clone_path,
'BADWOLF_REPO_SLUG': self.context.repository,
'BADWOLF_SCRIPT': script,
})
environment.setdefault('TERM', 'xterm-256color')
branch = self.context.source['branch']
labels = {
'repo': self.context.repository,
'commit': self.commit_hash,
'task_id': self.context.task_id,
}
if self.context.type == 'tag':
environment['BADWOLF_TAG'] = branch['name']
labels['tag'] = branch['name']
else:
environment['BADWOLF_BRANCH'] = branch['name']
labels['branch'] = branch['name']
if self.context.pr_id:
environment['BADWOLF_PULL_REQUEST'] = str(self.context.pr_id)
labels['pull_request'] = str(self.context.pr_id)
volumes = {
self.context.clone_path: {
'bind': self.context.clone_path,
'mode': 'rw',
},
}
if self.spec.docker:
volumes['/var/run/docker.sock'] = {
'bind': '/var/run/docker.sock',
'mode': 'ro',
}
environment.setdefault('DOCKER_HOST',
'unix:///var/run/docker.sock')
self._populate_more_envvars(environment)
logger.debug('Docker container environment: \n %r', environment)
container = self.docker.containers.create(
docker_image_name,
entrypoint=['/bin/sh', '-c'],
command=['echo $BADWOLF_SCRIPT | base64 --decode | /bin/sh'],
environment=environment,
working_dir=self.context.clone_path,
volumes=volumes,
privileged=self.spec.privileged,
stdin_open=False,
tty=True,
labels=labels,
)
container_id = container.id
logger.info('Created container %s from image %s', container_id,
docker_image_name)
output = []
try:
container.start()
self.update_build_status('INPROGRESS',
'Running tests in Docker container')
exit_code = container.wait(
timeout=current_app.config['DOCKER_RUN_TIMEOUT'])
except (APIError, DockerException, ReadTimeout) as e:
exit_code = -1
output.append(str(e) + '\n')
logger.exception('Docker error')
finally:
try:
output.append(to_text(container.logs()))
container.remove(force=True)
except NotFound:
pass
except APIError as api_e:
if 'can not get logs from container which is dead or marked for removal' in str(
api_e):
output.append('Build cancelled')
else:
logger.exception('Error removing docker container')
except (DockerException, ReadTimeout):
logger.exception('Error removing docker container')
return exit_code, ''.join(output)
