def run_tests_in_container(self, docker_image_name): command = '/bin/sh -c badwolf-run' environment = {} if self.spec.environments: # TODO: Support run in multiple environments environment = self.spec.environments[0] # TODO: Add more test context related env vars environment.update({ 'DEBIAN_FRONTEND': 'noninteractive', 'CI': 'true', 'CI_NAME': 'badwolf', 'BADWOLF_BRANCH': self.branch, 'BADWOLF_COMMIT': self.commit_hash, 'BADWOLF_BUILD_DIR': '/mnt/src', 'BADWOLF_REPO_SLUG': self.repo_full_name, }) if self.context.pr_id: environment['BADWOLF_PULL_REQUEST'] = to_text(self.context.pr_id) container = self.docker.create_container( docker_image_name, command=command, environment=environment, working_dir='/mnt/src', volumes=['/mnt/src'], host_config=self.docker.create_host_config( privileged=self.spec.privileged, binds={ self.clone_path: { 'bind': '/mnt/src', 'mode': 'rw', }, })) container_id = container['Id'] logger.info('Created container %s from image %s', container_id, docker_image_name) output = [] try: self.docker.start(container_id) self.update_build_status('INPROGRESS', 'Running tests in Docker container') for line in self.docker.logs(container_id, stream=True): output.append(to_text(line)) exit_code = self.docker.wait( container_id, current_app.config['DOCKER_RUN_TIMEOUT']) except (APIError, DockerException, ReadTimeout) as e: exit_code = -1 output.append(to_text(e)) logger.exception('Docker error') finally: try: self.docker.remove_container(container_id, force=True) except (APIError, DockerException): logger.exception('Error removing docker container') return exit_code, ''.join(output)
def __init__(self, repository, actor, type, message, source, target=None, rebuild=False, pr_id=None, nocache=False, clone_depth=50): self.task_id = to_text(uuid.uuid4()) self.repository = repository self.repo_name = repository.split('/')[-1] self.actor = actor self.type = type self.message = message self.source = source self.target = target self.rebuild = rebuild self.pr_id = pr_id # Don't use cache when build Docker image self.nocache = nocache self.clone_depth = clone_depth if 'repository' not in self.source: self.source['repository'] = {'full_name': repository} self.clone_path = os.path.join( current_app.config['BADWOLF_REPO_DIR'], self.repo_name, self.task_id, )
def __init__(self, repository, actor, type, message, source, target=None, rebuild=False, pr_id=None, cleanup_lint=False, nocache=False, clone_depth=50): self.task_id = to_text(uuid.uuid4()) self.repository = repository self.repo_name = repository.split('/')[-1] self.actor = actor self.type = type self.message = message self.source = source self.target = target self.rebuild = rebuild self.pr_id = pr_id self.cleanup_lint = cleanup_lint # Don't use cache when build Docker image self.nocache = nocache self.clone_depth = clone_depth if 'repository' not in self.source: self.source['repository'] = {'full_name': repository} self.clone_path = os.path.join( TMP_PATH, 'badwolf', self.task_id, self.repo_name, )
def test_repo_push_ci_skip_found(test_client): payload = json.dumps({ 'repository': { 'full_name': 'deepanalyzer/badwolf', 'scm': 'git', }, 'push': { 'changes': [ { 'new': { 'type': 'branch', }, 'commits': [ { 'hash': '2cedc1af762', 'message': 'Test [ci skip]', } ], } ] } }) res = test_client.post( url_for('webhook.webhook_push'), data=payload, headers={ 'User-Agent': 'Bitbucket-Webhooks/2.0', 'X-Event-Key': 'repo:push', } ) assert res.status_code == 200 assert to_text(res.data) == ''
def test_pr_updated_state_not_open(test_client): payload = json.dumps({ 'repository': { 'full_name': 'deepanalyzer/badwolf', 'scm': 'git', }, 'pullrequest': { 'title': 'Test PR', 'description': '', 'state': 'MERGED', }, 'source': { 'repository': {'full_name': 'deepanalyzer/badwolf'}, 'branch': {'name': 'develop'}, 'commit': {'hash': 'abc'} }, 'target': { 'repository': {'full_name': 'deepanalyzer/badwolf'}, 'branch': {'name': 'master'}, 'commit': {'hash': 'abc'} } }) res = test_client.post( url_for('webhook.webhook_push'), data=payload, headers={ 'X-Event-Key': 'pullrequest:updated', } ) assert res.status_code == 200 assert to_text(res.data) == ''
def test_repo_push_unsupported_scm(test_client): payload = json.dumps({ 'repository': { 'full_name': 'deepanalyzer/badwolf', 'scm': 'hg', }, 'push': { 'changes': [ { 'new': { 'type': 'branch', } } ] } }) res = test_client.post( url_for('webhook.webhook_push'), data=payload, headers={ 'User-Agent': 'Bitbucket-Webhooks/2.0', 'X-Event-Key': 'repo:push', } ) assert res.status_code == 200 assert to_text(res.data) == ''
def test_repo_push_no_new_changes(test_client): payload = json.dumps({'push': {'changes': []}}) res = test_client.post(url_for('webhook.webhook_push'), data=payload, headers={ 'X-Event-Key': 'repo:push', }) assert res.status_code == 200 assert to_text(res.data) == ''
def test_parse_secure_env(app): s = """env: - secure: {}""".format(to_text(SecureToken.encrypt('X=1 Y=2 Z=3'))) f = io.StringIO(s) spec = Specification.parse_file(f) assert len(spec.environments) == 1 env0 = spec.environments[0] assert env0['X'] == '1' assert env0['Y'] == '2' assert env0['Z'] == '3'
def _report_git_error(self, exc): self.build_status.update('FAILED', description='Git clone repository failed') content = ':broken_heart: **Git error**: {}'.format(to_text(exc)) content = sanitize_sensitive_data(content) if self.context.pr_id: pr = PullRequest(bitbucket, self.context.repository) pr.comment(self.context.pr_id, content) else: cs = Changesets(bitbucket, self.context.repository) cs.comment(self.commit_hash, content)
def test_pr_created_unsupported_scm(test_client): payload = json.dumps({ 'repository': { 'full_name': 'deepanalyzer/badwolf', 'scm': 'hg', }, }) res = test_client.post(url_for('webhook.webhook_push'), data=payload, headers={ 'X-Event-Key': 'pullrequest:created', }) assert res.status_code == 200 assert to_text(res.data) == ''
def __init__(self, repository, actor, type, message, source, target=None, rebuild=False, pr_id=None, nocache=False, clone_depth=50, skip_lint=False): self.task_id = to_text(uuid.uuid4()) self.repository = repository self.repo_owner, self.repo_name = repository.split('/') self.actor = actor self.type = type self.message = message self.source = source self.target = target self.rebuild = rebuild self.pr_id = pr_id # Don't use cache when build Docker image self.nocache = nocache self.clone_depth = clone_depth self.skip_lint = skip_lint if 'repository' not in self.source: self.source['repository'] = {'full_name': repository} self.clone_path = os.path.join( current_app.config['BADWOLF_REPO_DIR'], self.repo_name, self.task_id, ) self.environment = { 'DEBIAN_FRONTEND': 'noninteractive', 'CI': 'true', 'CI_NAME': 'badwolf', 'BADWOLF_BUILD_DIR': self.clone_path, 'BADWOLF_REPO_SLUG': repository, 'BADWOLF_COMMIT': source['commit']['hash'], } if type == 'tag': self.environment['BADWOLF_TAG'] = source['branch']['name'] else: self.environment['BADWOLF_BRANCH'] = source['branch']['name'] if pr_id: self.environment['BADWOLF_PULL_REQUEST'] = str(pr_id)
def get_docker_image(self): docker_image_name = self.repo_full_name.replace('/', '-') output = [] with self.lock: docker_image = self.docker.images(docker_image_name) if not docker_image or self.context.rebuild: dockerfile = os.path.join(self.clone_path, self.spec.dockerfile) build_options = { 'tag': docker_image_name, 'rm': True, } if not os.path.exists(dockerfile): logger.warning( 'No Dockerfile: %s found for repo: %s, using simple runner image', dockerfile, self.repo_full_name) dockerfile_content = 'FROM messense/badwolf-test-runner\n' fileobj = io.BytesIO(dockerfile_content.encode('utf-8')) build_options['fileobj'] = fileobj else: build_options['dockerfile'] = self.spec.dockerfile build_success = False logger.info('Building Docker image %s', docker_image_name) self.update_build_status('INPROGRESS', 'Building Docker image') res = self.docker.build(self.clone_path, **build_options) for line in res: if b'Successfully built' in line: build_success = True log = to_text(json.loads(to_text(line))['stream']) output.append(log) logger.info('`docker build` : %s', log.strip()) if not build_success: return None, ''.join(output) return docker_image_name, ''.join(output)
def test_unhandled_event(test_client): payload = json.dumps({ 'push': { 'changes': [ ] } }) res = test_client.post( url_for('webhook.webhook_push'), data=payload, headers={ 'User-Agent': 'Bitbucket-Webhooks/2.0', 'X-Event-Key': 'repo:created', } ) assert res.status_code == 200 assert to_text(res.data) == ''
def test_pr_updated_ci_skip_found(test_client): payload = json.dumps({ 'repository': { 'full_name': 'deepanalyzer/badwolf', 'scm': 'git', }, 'pullrequest': { 'title': 'Test PR', 'description': 'ci skip', } }) res = test_client.post(url_for('webhook.webhook_push'), data=payload, headers={ 'X-Event-Key': 'pullrequest:updated', }) assert res.status_code == 200 assert to_text(res.data) == ''
def lint_files(self, files): command = [ self.python_name, '-m', 'pylint', '-r', 'n', '-f', 'parseable' ] command += files _, output = run_command(command, split=True, include_errors=True, cwd=self.working_dir) if not output: raise StopIteration() for line in output: parsed = self._parse_line(to_text(line)) if parsed is None: continue filename, line, message = parsed yield Problem(filename, line, message, self.name)
def test_repo_push_unsupported_push_type(test_client): payload = json.dumps({ 'repository': { 'full_name': 'deepanalyzer/badwolf', 'scm': 'git', }, 'push': { 'changes': [{ 'new': { 'type': 'wrong_push_type', } }] } }) res = test_client.post(url_for('webhook.webhook_push'), data=payload, headers={ 'X-Event-Key': 'repo:push', }) assert res.status_code == 200 assert to_text(res.data) == ''
def shell_script(self): def _trace(command): return 'echo + {}\n{} '.format(shlex.quote(command), command) commands = [] after_success = [_trace(cmd) for cmd in self.after_success] after_failure = [_trace(cmd) for cmd in self.after_failure] for service in self.services: commands.append(_trace('service {} start'.format(service))) for script in self.scripts: commands.append(_trace(script)) command_encoded = shlex.quote( to_text(base64.b64encode(to_binary('\n'.join(commands))))) context = { 'command': command_encoded, 'after_success': ' \n'.join(after_success), 'after_failure': ' \n'.join(after_failure), } script = render_template('script.sh', **context) logger.debug('Build script: \n%s', script) return script
def run(self): start_time = time.time() self.branch = self.context.source['branch']['name'] try: self.clone_repository() except git.GitCommandError as e: logger.exception('Git command error') self.update_build_status('FAILED', 'Git clone repository failed') content = ':broken_heart: **Git error**: {}'.format(to_text(e)) if self.context.pr_id: pr = PullRequest(bitbucket, self.repo_full_name) pr.comment(self.context.pr_id, content) else: cs = Changesets(bitbucket, self.repo_full_name) cs.comment(self.commit_hash, content) self.cleanup() return if not self.validate_settings(): self.cleanup() return context = { 'context': self.context, 'task_id': self.task_id, 'build_log_url': url_for('log.build_log', sha=self.commit_hash, _external=True), 'branch': self.branch, 'scripts': self.spec.scripts, } if self.spec.scripts: self.update_build_status('INPROGRESS', 'Test in progress') docker_image_name, build_output = self.get_docker_image() context['build_logs'] = to_text(build_output) context.update({ 'build_logs': to_text(build_output), 'elapsed_time': int(time.time() - start_time), }) if not docker_image_name: self.update_build_status('FAILED', 'Build or get Docker image failed') context['exit_code'] = -1 self.send_notifications(context) self.cleanup() return exit_code, output = self.run_tests_in_container(docker_image_name) if exit_code == 0: # Success logger.info('Test succeed for repo: %s', self.repo_full_name) self.update_build_status('SUCCESSFUL', '1 of 1 test succeed') else: # Failed logger.info('Test failed for repo: %s, exit code: %s', self.repo_full_name, exit_code) self.update_build_status('FAILED', '1 of 1 test failed') context.update({ 'logs': to_text(output), 'exit_code': exit_code, 'elapsed_time': int(time.time() - start_time), }) self.send_notifications(context) # Code linting if self.context.pr_id and self.spec.linters: lint = LintProcessor(self.context, self.spec, self.clone_path) lint.process() self.cleanup()
def decrypt(encrypted): fernet = Fernet(to_binary(current_app.config['SECURE_TOKEN_KEY'])) text = fernet.decrypt(to_binary(encrypted)) return to_text(text)
def run_in_container(self, docker_image_name): environment = {} if self.spec.environments: # TODO: Support run in multiple environments environment = self.spec.environments[0] # TODO: Add more test context related env vars script = shlex.quote( to_text(base64.b64encode(to_binary(self.spec.shell_script)))) environment.update({ 'DEBIAN_FRONTEND': 'noninteractive', 'HOME': '/root', 'SHELL': '/bin/sh', 'CI': 'true', 'CI_NAME': 'badwolf', 'BADWOLF_COMMIT': self.commit_hash, 'BADWOLF_BUILD_DIR': self.context.clone_path, 'BADWOLF_REPO_SLUG': self.context.repository, 'BADWOLF_SCRIPT': script, }) environment.setdefault('TERM', 'xterm-256color') branch = self.context.source['branch'] labels = { 'repo': self.context.repository, 'commit': self.commit_hash, 'task_id': self.context.task_id, } if self.context.type == 'tag': environment['BADWOLF_TAG'] = branch['name'] labels['tag'] = branch['name'] else: environment['BADWOLF_BRANCH'] = branch['name'] labels['branch'] = branch['name'] if self.context.pr_id: environment['BADWOLF_PULL_REQUEST'] = str(self.context.pr_id) labels['pull_request'] = str(self.context.pr_id) volumes = { self.context.clone_path: { 'bind': self.context.clone_path, 'mode': 'rw', }, } if self.spec.docker: volumes['/var/run/docker.sock'] = { 'bind': '/var/run/docker.sock', 'mode': 'ro', } environment.setdefault('DOCKER_HOST', 'unix:///var/run/docker.sock') self._populate_more_envvars(environment) logger.debug('Docker container environment: \n %r', environment) container = self.docker.containers.create( docker_image_name, entrypoint=['/bin/sh', '-c'], command=['echo $BADWOLF_SCRIPT | base64 --decode | /bin/sh'], environment=environment, working_dir=self.context.clone_path, volumes=volumes, privileged=self.spec.privileged, stdin_open=False, tty=True, labels=labels, ) container_id = container.id logger.info('Created container %s from image %s', container_id, docker_image_name) output = [] try: container.start() self.update_build_status('INPROGRESS', 'Running tests in Docker container') exit_code = container.wait( timeout=current_app.config['DOCKER_RUN_TIMEOUT']) except (APIError, DockerException, ReadTimeout) as e: exit_code = -1 output.append(str(e) + '\n') logger.exception('Docker error') finally: try: output.append(to_text(container.logs())) container.remove(force=True) except NotFound: pass except APIError as api_e: if 'can not get logs from container which is dead or marked for removal' in str( api_e): output.append('Build cancelled') else: logger.exception('Error removing docker container') except (DockerException, ReadTimeout): logger.exception('Error removing docker container') return exit_code, ''.join(output)