def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force env_cls = LocalEnvironment self.setup_env = env_cls(project=self.project, version=self.version, build=self.build, record=record) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout, e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) self.config = load_yaml_config(version=self.version)
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, intersphinx=True, localmedia=True, basic=False, **kwargs): env_cls = LocalEnvironment if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record) with self.build_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout, e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423) if self.project.documentation_type == 'auto': self.update_documentation_type() self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], )
def run_setup(self, record=True): """ Run setup in the local environment. Return True if successful. """ self.setup_env = LocalBuildEnvironment( project=self.project, version=self.version, build=self.build, record=record, update_on_success=False, ) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout as e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) try: self.config = load_yaml_config(version=self.version) except ConfigError as e: raise BuildEnvironmentError( 'Problem parsing YAML configuration. {0}'.format(str(e)) ) if self.setup_env.failure or self.config is None: self._log('Failing build because of setup failure: %s' % self.setup_env.failure) # Send notification to users only if the build didn't fail because of # LockTimeout: this exception occurs when a build is triggered before the previous # one has finished (e.g. two webhooks, one after the other) if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout): self.send_notifications() return False if self.setup_env.successful and not self.project.has_valid_clone: self.set_valid_clone() return True
def test_api_failure_returns_previous_error_on_error_in_exit(self): """ Treat previously raised errors with more priority. Don't report a connection problem to Docker on cleanup if we have a more usable error to show the user. """ response = Mock(status_code=500, reason='Internal Server Error') self.mocks.configure_mock( 'docker_client', {'kill.side_effect': BuildEnvironmentError('Outer failed')}) build_env = DockerBuildEnvironment( version=self.version, project=self.project, build={'id': DUMMY_BUILD_ID}, ) with build_env: raise BuildEnvironmentError('Inner failed') # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) self.mocks.mocks['api_v2.build']().put.assert_called_with({ 'id': DUMMY_BUILD_ID, 'version': self.version.pk, 'success': False, 'project': self.project.pk, 'setup_error': u'', 'exit_code': 1, 'length': 0, 'error': 'Inner failed', 'setup': u'', 'output': u'', 'state': u'finished', 'builder': mock.ANY, })
def setup_vcs(self): """ Update the checkout of the repo to make sure it's the latest. This also syncs versions in the DB. :param build_env: Build environment """ self.setup_env.update_build(state=BUILD_STATE_CLONING) self._log(msg='Updating docs from VCS') try: update_imported_docs(self.version.pk) commit = self.project.vcs_repo(self.version.slug).commit if commit: self.build['commit'] = commit except ProjectImportError as e: log.error( LOG_TEMPLATE.format(project=self.project.slug, version=self.version.slug, msg=str(e)), exc_info=True, ) raise BuildEnvironmentError('Failed to import project: %s' % e, status_code=404)
def test_environment_failed_build_without_update_but_with_error(self): """A failed build exits cleanly and doesn't update build.""" build_env = DockerBuildEnvironment( version=self.version, project=self.project, build={'id': DUMMY_BUILD_ID}, update_on_success=False, ) with build_env: raise BuildEnvironmentError('Test') self.assertFalse(build_env.successful) # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) self.mocks.mocks['api_v2.build']().put.assert_called_with({ 'id': DUMMY_BUILD_ID, 'version': self.version.pk, 'success': False, 'project': self.project.pk, 'setup_error': u'', 'exit_code': 1, 'length': 0, 'error': 'Test', 'setup': u'', 'output': u'', 'state': u'finished', 'builder': mock.ANY, })
def test_api_failure_on_error_in_exit(self): response = Mock(status_code=500, reason='Internal Server Error') self.mocks.configure_mock('docker_client', { 'kill.side_effect': BuildEnvironmentError('Failed') }) build_env = DockerBuildEnvironment( version=self.version, project=self.project, build={'id': DUMMY_BUILD_ID}, ) with build_env: pass # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) self.mocks.mocks['api_v2.build']().put.assert_called_with({ 'id': DUMMY_BUILD_ID, 'version': self.version.pk, 'success': False, 'project': self.project.pk, 'setup_error': u'', 'exit_code': 1, 'length': 0, 'error': 'Failed', 'setup': u'', 'output': u'', 'state': u'finished', 'builder': mock.ANY, })
def run_build(self, docker=False, record=True): """ Build the docs in an environment. If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE setting, then build in a Docker environment. Otherwise build locally. """ env_vars = self.get_env_vars() if docker or settings.DOCKER_ENABLE: env_cls = DockerBuildEnvironment else: env_cls = LocalBuildEnvironment self.build_env = env_cls(project=self.project, version=self.version, config=self.config, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker with self.build_env: if self.project.documentation_type == 'auto': self.update_documentation_type() python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) try: self.setup_python_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') except SoftTimeLimitExceeded: raise BuildEnvironmentError(_('Build exited due to time out')) # Finalize build and update web servers if build_id: self.update_app_instances( html=bool(outcomes['html']), search=bool(outcomes['search']), localmedia=bool(outcomes['localmedia']), pdf=bool(outcomes['pdf']), epub=bool(outcomes['epub']), ) else: log.warning('No build ID, not syncing files') if self.build_env.failed: self.send_notifications() build_complete.send(sender=Build, build=self.build_env.build)
def test_failing_execution_with_caught_exception(self): """Build in failing state with BuildEnvironmentError exception.""" build_env = LocalBuildEnvironment( version=self.version, project=self.project, build={'id': DUMMY_BUILD_ID}, ) with build_env: raise BuildEnvironmentError('Foobar') self.assertFalse(self.mocks.process.communicate.called) self.assertEqual(len(build_env.commands), 0) self.assertTrue(build_env.done) self.assertTrue(build_env.failed) # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) self.mocks.mocks['api_v2.build']().put.assert_called_with({ 'id': DUMMY_BUILD_ID, 'version': self.version.pk, 'success': False, 'project': self.project.pk, 'setup_error': u'', 'length': mock.ANY, 'error': 'Foobar', 'setup': u'', 'output': u'', 'state': u'finished', 'builder': mock.ANY, 'exit_code': 1, })
def run_build(self, docker=False, record=True): """Build the docs in an environment. If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE setting, then build in a Docker environment. Otherwise build locally. """ env_vars = self.get_env_vars() if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment else: env_cls = LocalEnvironment self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker with self.build_env: if self.project.documentation_type == 'auto': self.update_documentation_type() python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) try: self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') except SoftTimeLimitExceeded: raise BuildEnvironmentError(_('Build exited due to time out')) # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], ) if self.build_env.failed: self.send_notifications() build_complete.send(sender=Build, build=self.build_env.build) self.build_env.update_build(state=BUILD_STATE_FINISHED)
def test_failing_execution_with_caught_exception(self): '''Build in failing state with BuildEnvironmentError exception''' build_env = LocalEnvironment(version=self.version, project=self.project, build={}) with build_env: raise BuildEnvironmentError('Foobar') self.assertFalse(self.mocks.process.communicate.called) self.assertEqual(len(build_env.commands), 0) self.assertTrue(build_env.done) self.assertTrue(build_env.failed)
def setup_vcs(self): """ Update the checkout of the repo to make sure it's the latest. This also syncs versions in the DB. :param build_env: Build environment """ self.build_env.update_build(state=BUILD_STATE_CLONING) self._log(msg='Updating docs from VCS') try: update_imported_docs(self.version.pk) commit = self.project.vcs_repo(self.version.slug).commit if commit: self.build['commit'] = commit except ProjectImportError: raise BuildEnvironmentError('Failed to import project', status_code=404)
def load_yaml_config(self): """ Load a YAML config. Raise BuildEnvironmentError if failed due to syntax errors. """ try: return yaml.safe_load(open(self.yaml_file, 'r')) except IOError: return { 'site_name': self.version.project.name, } except yaml.YAMLError as exc: note = '' if hasattr(exc, 'problem_mark'): mark = exc.problem_mark note = ' (line %d, column %d)' % (mark.line + 1, mark.column + 1) raise BuildEnvironmentError( 'Your mkdocs.yml could not be loaded, ' 'possibly due to a syntax error{note}'.format(note=note))
def load_yaml_config(self): """ Load a YAML config. Raise BuildEnvironmentError if failed due to syntax errors. """ try: return yaml.safe_load( open(os.path.join(self.root_path, 'mkdocs.yml'), 'r')) except IOError: return { 'site_name': self.version.project.name, } except yaml.YAMLError as exc: note = '' if hasattr(exc, 'problem_mark'): mark = exc.problem_mark note = ' (line %d, column %d)' % (mark.line + 1, mark.column + 1) raise BuildEnvironmentError("Your mkdocs.yml could not be loaded, " "possibly due to a syntax error%s" % (note, ))
def append_conf(self, **kwargs): """ Set mkdocs config values """ # Pull mkdocs config data try: user_config = yaml.safe_load( open(os.path.join(self.root_path, 'mkdocs.yml'), 'r')) except IOError: user_config = { 'site_name': self.version.project.name, } except yaml.YAMLError as exc: note = '' if hasattr(exc, 'problem_mark'): mark = exc.problem_mark note = ' (line %d, column %d)' % (mark.line + 1, mark.column + 1) raise BuildEnvironmentError("Your mkdocs.yml could not be loaded, " "possibly due to a syntax error%s" % (note, )) # Handle custom docs dirs user_docs_dir = user_config.get('docs_dir') if user_docs_dir: user_docs_dir = os.path.join(self.root_path, user_docs_dir) docs_dir = self.docs_dir(docs_dir=user_docs_dir) self.create_index(extension='md') user_config['docs_dir'] = docs_dir # Set mkdocs config values media_url = settings.MEDIA_URL # Mkdocs needs a full domain here because it tries to link to local media files if not media_url.startswith('http'): domain = getattr(settings, 'PRODUCTION_DOMAIN') media_url = 'http://{}{}'.format(domain, media_url) if 'extra_javascript' in user_config: user_config['extra_javascript'].append('readthedocs-data.js') user_config['extra_javascript'].append( 'readthedocs-dynamic-include.js') user_config['extra_javascript'].append( '%sstatic/core/js/readthedocs-doc-embed.js' % media_url) else: user_config['extra_javascript'] = [ 'readthedocs-data.js', 'readthedocs-dynamic-include.js', '%sstatic/core/js/readthedocs-doc-embed.js' % media_url, ] if 'extra_css' in user_config: user_config['extra_css'].append('%s/css/badge_only.css' % media_url) user_config['extra_css'].append( '%s/css/readthedocs-doc-embed.css' % media_url) else: user_config['extra_css'] = [ '%scss/badge_only.css' % media_url, '%scss/readthedocs-doc-embed.css' % media_url, ] # Set our custom theme dir for mkdocs if 'theme_dir' not in user_config and self.use_theme: user_config['theme_dir'] = TEMPLATE_DIR yaml.dump(user_config, open(os.path.join(self.root_path, 'mkdocs.yml'), 'w')) # RTD javascript writing # Will be available in the JavaScript as READTHEDOCS_DATA. readthedocs_data = { 'project': self.version.project.slug, 'version': self.version.slug, 'language': self.version.project.language, 'page': None, 'theme': "readthedocs", 'builder': "mkdocs", 'docroot': docs_dir, 'source_suffix': ".md", 'api_host': getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'), 'commit': self.version.project.vcs_repo(self.version.slug).commit, } data_json = json.dumps(readthedocs_data, indent=4) data_ctx = { 'data_json': data_json, 'current_version': readthedocs_data['version'], 'slug': readthedocs_data['project'], 'html_theme': readthedocs_data['theme'], 'pagename': None, } data_string = template_loader.get_template( 'doc_builder/data.js.tmpl').render(data_ctx) data_file = open( os.path.join(self.root_path, docs_dir, 'readthedocs-data.js'), 'w+') data_file.write(data_string) data_file.write(''' READTHEDOCS_DATA["page"] = mkdocs_page_input_path.substr( 0, mkdocs_page_input_path.lastIndexOf(READTHEDOCS_DATA.source_suffix)); ''') data_file.close() include_ctx = { 'global_analytics_code': getattr(settings, 'GLOBAL_ANALYTICS_CODE', 'UA-17997319-1'), 'user_analytics_code': self.version.project.analytics_code, } include_string = template_loader.get_template( 'doc_builder/include.js.tmpl').render(include_ctx) include_file = open( os.path.join(self.root_path, docs_dir, 'readthedocs-dynamic-include.js'), 'w+') include_file.write(include_string) include_file.close()
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=None, search=True, force=False, localmedia=True, **__): """ Run a documentation sync n' build. This is fully wrapped in exception handling to account for a number of failure cases. We first run a few commands in a local build environment, but do not report on environment success. This avoids a flicker on the build output page where the build is marked as finished in between the local environment steps and the docker build steps. If a failure is raised, or the build is not successful, return ``False``, otherwise, ``True``. Unhandled exceptions raise a generic user facing error, which directs the user to bug us. It is therefore a benefit to have as few unhandled errors as possible. :param pk int: Project id :param version_pk int: Project Version id (latest if None) :param build_pk int: Build id (if None, commands are not recorded) :param record bool: record a build object in the database :param docker bool: use docker to build the project (if ``None``, ``settings.DOCKER_ENABLE`` is used) :param search bool: update search :param force bool: force Sphinx build :param localmedia bool: update localmedia :returns: whether build was successful or not :rtype: bool """ try: if docker is None: docker = settings.DOCKER_ENABLE self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force self.config = None setup_successful = self.run_setup(record=record) if not setup_successful: return False # Catch unhandled errors in the setup step except Exception as e: # noqa log.exception( 'An unhandled exception was raised during build setup', extra={'tags': {'build': build_pk}} ) self.setup_env.failure = BuildEnvironmentError( BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format( build_id=build_pk, ) ) self.setup_env.update_build(BUILD_STATE_FINISHED) return False else: # No exceptions in the setup step, catch unhandled errors in the # build steps try: self.run_build(docker=docker, record=record) except Exception as e: # noqa log.exception( 'An unhandled exception was raised during project build', extra={'tags': {'build': build_pk}} ) self.build_env.failure = BuildEnvironmentError( BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format( build_id=build_pk, ) ) self.build_env.update_build(BUILD_STATE_FINISHED) return False return True
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force self.config = None env_cls = LocalEnvironment self.setup_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, report_build_success=False) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout as e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423) self.config = load_yaml_config(version=self.version) if self.setup_env.failed or self.config is None: self.send_notifications() self.setup_env.update_build(state=BUILD_STATE_FINISHED) return None if self.setup_env.successful and not self.project.has_valid_clone: self.set_valid_clone() env_vars = self.get_env_vars() if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker with self.build_env: if self.project.documentation_type == 'auto': self.update_documentation_type() python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], ) if self.build_env.failed: self.send_notifications() build_complete.send(sender=Build, build=self.build_env.build)
def run_build(self, docker, record): """ Build the docs in an environment. :param docker: if ``True``, the build uses a ``DockerBuildEnvironment``, otherwise it uses a ``LocalBuildEnvironment`` to run all the commands to build the docs :param record: whether or not record all the commands in the ``Build`` instance """ env_vars = self.get_env_vars() if docker: env_cls = DockerBuildEnvironment else: env_cls = LocalBuildEnvironment self.build_env = env_cls(project=self.project, version=self.version, config=self.config, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker with self.build_env: if self.project.documentation_type == 'auto': self.update_documentation_type() python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) try: self.setup_python_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') except vcs_support_utils.LockTimeout as e: self.task.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423) except SoftTimeLimitExceeded: raise BuildEnvironmentError(_('Build exited due to time out')) # Finalize build and update web servers if build_id: self.update_app_instances( html=bool(outcomes['html']), search=bool(outcomes['search']), localmedia=bool(outcomes['localmedia']), pdf=bool(outcomes['pdf']), epub=bool(outcomes['epub']), ) else: log.warning('No build ID, not syncing files') if self.build_env.failed: self.send_notifications() build_complete.send(sender=Build, build=self.build_env.build)