def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False,
search=True, force=False, localmedia=True, **kwargs):
self.project = self.get_project(pk)
self.version = self.get_version(self.project, version_pk)
self.build = self.get_build(build_pk)
self.build_search = search
self.build_localmedia = localmedia
self.build_force = force
env_cls = LocalEnvironment
self.setup_env = env_cls(project=self.project, version=self.version,
build=self.build, record=record)
# Environment used for code checkout & initial configuration reading
with self.setup_env:
if self.project.skip:
raise BuildEnvironmentError(
_('Builds for this project are temporarily disabled'))
try:
self.setup_vcs()
except vcs_support_utils.LockTimeout, e:
self.retry(exc=e, throw=False)
raise BuildEnvironmentError(
'Version locked, retrying in 5 minutes.',
status_code=423
)
self.config = load_yaml_config(version=self.version)
def run(self,
pk,
version_pk=None,
build_pk=None,
record=True,
docker=False,
search=True,
force=False,
intersphinx=True,
localmedia=True,
basic=False,
**kwargs):
env_cls = LocalEnvironment
if docker or settings.DOCKER_ENABLE:
env_cls = DockerEnvironment
self.project = self.get_project(pk)
self.version = self.get_version(self.project, version_pk)
self.build = self.get_build(build_pk)
self.build_search = search
self.build_localmedia = localmedia
self.build_force = force
self.build_env = env_cls(project=self.project,
version=self.version,
build=self.build,
record=record)
with self.build_env:
if self.project.skip:
raise BuildEnvironmentError(
_('Builds for this project are temporarily disabled'))
try:
self.setup_vcs()
except vcs_support_utils.LockTimeout, e:
self.retry(exc=e, throw=False)
raise BuildEnvironmentError(
'Version locked, retrying in 5 minutes.', status_code=423)
if self.project.documentation_type == 'auto':
self.update_documentation_type()
self.setup_environment()
# TODO the build object should have an idea of these states, extend
# the model to include an idea of these outcomes
outcomes = self.build_docs()
build_id = self.build.get('id')
# Web Server Tasks
if build_id:
finish_build.delay(
version_pk=self.version.pk,
build_pk=build_id,
hostname=socket.gethostname(),
html=outcomes['html'],
search=outcomes['search'],
localmedia=outcomes['localmedia'],
pdf=outcomes['pdf'],
epub=outcomes['epub'],
)
def run_setup(self, record=True):
"""
Run setup in the local environment.
Return True if successful.
"""
self.setup_env = LocalBuildEnvironment(
project=self.project,
version=self.version,
build=self.build,
record=record,
update_on_success=False,
)
# Environment used for code checkout & initial configuration reading
with self.setup_env:
if self.project.skip:
raise BuildEnvironmentError(
_('Builds for this project are temporarily disabled'))
try:
self.setup_vcs()
except vcs_support_utils.LockTimeout as e:
self.retry(exc=e, throw=False)
raise BuildEnvironmentError(
'Version locked, retrying in 5 minutes.',
status_code=423
)
try:
self.config = load_yaml_config(version=self.version)
except ConfigError as e:
raise BuildEnvironmentError(
'Problem parsing YAML configuration. {0}'.format(str(e))
)
if self.setup_env.failure or self.config is None:
self._log('Failing build because of setup failure: %s' % self.setup_env.failure)
# Send notification to users only if the build didn't fail because of
# LockTimeout: this exception occurs when a build is triggered before the previous
# one has finished (e.g. two webhooks, one after the other)
if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout):
self.send_notifications()
return False
if self.setup_env.successful and not self.project.has_valid_clone:
self.set_valid_clone()
return True
def test_api_failure_returns_previous_error_on_error_in_exit(self):
"""
Treat previously raised errors with more priority.
Don't report a connection problem to Docker on cleanup if we have a more
usable error to show the user.
"""
response = Mock(status_code=500, reason='Internal Server Error')
self.mocks.configure_mock(
'docker_client',
{'kill.side_effect': BuildEnvironmentError('Outer failed')})
build_env = DockerBuildEnvironment(
version=self.version,
project=self.project,
build={'id': DUMMY_BUILD_ID},
)
with build_env:
raise BuildEnvironmentError('Inner failed')
# api() is not called anymore, we use api_v2 instead
self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
self.mocks.mocks['api_v2.build']().put.assert_called_with({
'id':
DUMMY_BUILD_ID,
'version':
self.version.pk,
'success':
False,
'project':
self.project.pk,
'setup_error':
u'',
'exit_code':
1,
'length':
0,
'error':
'Inner failed',
'setup':
u'',
'output':
u'',
'state':
u'finished',
'builder':
mock.ANY,
})
def setup_vcs(self):
"""
Update the checkout of the repo to make sure it's the latest.
This also syncs versions in the DB.
:param build_env: Build environment
"""
self.setup_env.update_build(state=BUILD_STATE_CLONING)
self._log(msg='Updating docs from VCS')
try:
update_imported_docs(self.version.pk)
commit = self.project.vcs_repo(self.version.slug).commit
if commit:
self.build['commit'] = commit
except ProjectImportError as e:
log.error(
LOG_TEMPLATE.format(project=self.project.slug,
version=self.version.slug,
msg=str(e)),
exc_info=True,
)
raise BuildEnvironmentError('Failed to import project: %s' % e,
status_code=404)
def test_environment_failed_build_without_update_but_with_error(self):
"""A failed build exits cleanly and doesn't update build."""
build_env = DockerBuildEnvironment(
version=self.version,
project=self.project,
build={'id': DUMMY_BUILD_ID},
update_on_success=False,
)
with build_env:
raise BuildEnvironmentError('Test')
self.assertFalse(build_env.successful)
# api() is not called anymore, we use api_v2 instead
self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
self.mocks.mocks['api_v2.build']().put.assert_called_with({
'id': DUMMY_BUILD_ID,
'version': self.version.pk,
'success': False,
'project': self.project.pk,
'setup_error': u'',
'exit_code': 1,
'length': 0,
'error': 'Test',
'setup': u'',
'output': u'',
'state': u'finished',
'builder': mock.ANY,
})
def test_api_failure_on_error_in_exit(self):
response = Mock(status_code=500, reason='Internal Server Error')
self.mocks.configure_mock('docker_client', {
'kill.side_effect': BuildEnvironmentError('Failed')
})
build_env = DockerBuildEnvironment(
version=self.version,
project=self.project,
build={'id': DUMMY_BUILD_ID},
)
with build_env:
pass
# api() is not called anymore, we use api_v2 instead
self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
self.mocks.mocks['api_v2.build']().put.assert_called_with({
'id': DUMMY_BUILD_ID,
'version': self.version.pk,
'success': False,
'project': self.project.pk,
'setup_error': u'',
'exit_code': 1,
'length': 0,
'error': 'Failed',
'setup': u'',
'output': u'',
'state': u'finished',
'builder': mock.ANY,
})
def run_build(self, docker=False, record=True):
"""
Build the docs in an environment.
If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
setting, then build in a Docker environment. Otherwise build locally.
"""
env_vars = self.get_env_vars()
if docker or settings.DOCKER_ENABLE:
env_cls = DockerBuildEnvironment
else:
env_cls = LocalBuildEnvironment
self.build_env = env_cls(project=self.project,
version=self.version,
config=self.config,
build=self.build,
record=record,
environment=env_vars)
# Environment used for building code, usually with Docker
with self.build_env:
if self.project.documentation_type == 'auto':
self.update_documentation_type()
python_env_cls = Virtualenv
if self.config.use_conda:
self._log('Using conda')
python_env_cls = Conda
self.python_env = python_env_cls(version=self.version,
build_env=self.build_env,
config=self.config)
try:
self.setup_python_environment()
# TODO the build object should have an idea of these states, extend
# the model to include an idea of these outcomes
outcomes = self.build_docs()
build_id = self.build.get('id')
except SoftTimeLimitExceeded:
raise BuildEnvironmentError(_('Build exited due to time out'))
# Finalize build and update web servers
if build_id:
self.update_app_instances(
html=bool(outcomes['html']),
search=bool(outcomes['search']),
localmedia=bool(outcomes['localmedia']),
pdf=bool(outcomes['pdf']),
epub=bool(outcomes['epub']),
)
else:
log.warning('No build ID, not syncing files')
if self.build_env.failed:
self.send_notifications()
build_complete.send(sender=Build, build=self.build_env.build)
def test_failing_execution_with_caught_exception(self):
"""Build in failing state with BuildEnvironmentError exception."""
build_env = LocalBuildEnvironment(
version=self.version,
project=self.project,
build={'id': DUMMY_BUILD_ID},
)
with build_env:
raise BuildEnvironmentError('Foobar')
self.assertFalse(self.mocks.process.communicate.called)
self.assertEqual(len(build_env.commands), 0)
self.assertTrue(build_env.done)
self.assertTrue(build_env.failed)
# api() is not called anymore, we use api_v2 instead
self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
self.mocks.mocks['api_v2.build']().put.assert_called_with({
'id': DUMMY_BUILD_ID,
'version': self.version.pk,
'success': False,
'project': self.project.pk,
'setup_error': u'',
'length': mock.ANY,
'error': 'Foobar',
'setup': u'',
'output': u'',
'state': u'finished',
'builder': mock.ANY,
'exit_code': 1,
})
def run_build(self, docker=False, record=True):
"""Build the docs in an environment.
If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
setting, then build in a Docker environment. Otherwise build locally.
"""
env_vars = self.get_env_vars()
if docker or settings.DOCKER_ENABLE:
env_cls = DockerEnvironment
else:
env_cls = LocalEnvironment
self.build_env = env_cls(project=self.project, version=self.version,
build=self.build, record=record, environment=env_vars)
# Environment used for building code, usually with Docker
with self.build_env:
if self.project.documentation_type == 'auto':
self.update_documentation_type()
python_env_cls = Virtualenv
if self.config.use_conda:
self._log('Using conda')
python_env_cls = Conda
self.python_env = python_env_cls(version=self.version,
build_env=self.build_env,
config=self.config)
try:
self.setup_environment()
# TODO the build object should have an idea of these states, extend
# the model to include an idea of these outcomes
outcomes = self.build_docs()
build_id = self.build.get('id')
except SoftTimeLimitExceeded:
raise BuildEnvironmentError(_('Build exited due to time out'))
# Web Server Tasks
if build_id:
finish_build.delay(
version_pk=self.version.pk,
build_pk=build_id,
hostname=socket.gethostname(),
html=outcomes['html'],
search=outcomes['search'],
localmedia=outcomes['localmedia'],
pdf=outcomes['pdf'],
epub=outcomes['epub'],
)
if self.build_env.failed:
self.send_notifications()
build_complete.send(sender=Build, build=self.build_env.build)
self.build_env.update_build(state=BUILD_STATE_FINISHED)
def test_failing_execution_with_caught_exception(self):
'''Build in failing state with BuildEnvironmentError exception'''
build_env = LocalEnvironment(version=self.version, project=self.project,
build={})
with build_env:
raise BuildEnvironmentError('Foobar')
self.assertFalse(self.mocks.process.communicate.called)
self.assertEqual(len(build_env.commands), 0)
self.assertTrue(build_env.done)
self.assertTrue(build_env.failed)
def setup_vcs(self):
"""
Update the checkout of the repo to make sure it's the latest.
This also syncs versions in the DB.
:param build_env: Build environment
"""
self.build_env.update_build(state=BUILD_STATE_CLONING)
self._log(msg='Updating docs from VCS')
try:
update_imported_docs(self.version.pk)
commit = self.project.vcs_repo(self.version.slug).commit
if commit:
self.build['commit'] = commit
except ProjectImportError:
raise BuildEnvironmentError('Failed to import project',
status_code=404)
def load_yaml_config(self):
"""
Load a YAML config.
Raise BuildEnvironmentError if failed due to syntax errors.
"""
try:
return yaml.safe_load(open(self.yaml_file, 'r'))
except IOError:
return {
'site_name': self.version.project.name,
}
except yaml.YAMLError as exc:
note = ''
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark
note = ' (line %d, column %d)' % (mark.line + 1,
mark.column + 1)
raise BuildEnvironmentError(
'Your mkdocs.yml could not be loaded, '
'possibly due to a syntax error{note}'.format(note=note))
def load_yaml_config(self):
"""
Load a YAML config.
Raise BuildEnvironmentError if failed due to syntax errors.
"""
try:
return yaml.safe_load(
open(os.path.join(self.root_path, 'mkdocs.yml'), 'r'))
except IOError:
return {
'site_name': self.version.project.name,
}
except yaml.YAMLError as exc:
note = ''
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark
note = ' (line %d, column %d)' % (mark.line + 1,
mark.column + 1)
raise BuildEnvironmentError("Your mkdocs.yml could not be loaded, "
"possibly due to a syntax error%s" %
(note, ))
def append_conf(self, **kwargs):
"""
Set mkdocs config values
"""
# Pull mkdocs config data
try:
user_config = yaml.safe_load(
open(os.path.join(self.root_path, 'mkdocs.yml'), 'r'))
except IOError:
user_config = {
'site_name': self.version.project.name,
}
except yaml.YAMLError as exc:
note = ''
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark
note = ' (line %d, column %d)' % (mark.line + 1,
mark.column + 1)
raise BuildEnvironmentError("Your mkdocs.yml could not be loaded, "
"possibly due to a syntax error%s" %
(note, ))
# Handle custom docs dirs
user_docs_dir = user_config.get('docs_dir')
if user_docs_dir:
user_docs_dir = os.path.join(self.root_path, user_docs_dir)
docs_dir = self.docs_dir(docs_dir=user_docs_dir)
self.create_index(extension='md')
user_config['docs_dir'] = docs_dir
# Set mkdocs config values
media_url = settings.MEDIA_URL
# Mkdocs needs a full domain here because it tries to link to local media files
if not media_url.startswith('http'):
domain = getattr(settings, 'PRODUCTION_DOMAIN')
media_url = 'http://{}{}'.format(domain, media_url)
if 'extra_javascript' in user_config:
user_config['extra_javascript'].append('readthedocs-data.js')
user_config['extra_javascript'].append(
'readthedocs-dynamic-include.js')
user_config['extra_javascript'].append(
'%sstatic/core/js/readthedocs-doc-embed.js' % media_url)
else:
user_config['extra_javascript'] = [
'readthedocs-data.js',
'readthedocs-dynamic-include.js',
'%sstatic/core/js/readthedocs-doc-embed.js' % media_url,
]
if 'extra_css' in user_config:
user_config['extra_css'].append('%s/css/badge_only.css' %
media_url)
user_config['extra_css'].append(
'%s/css/readthedocs-doc-embed.css' % media_url)
else:
user_config['extra_css'] = [
'%scss/badge_only.css' % media_url,
'%scss/readthedocs-doc-embed.css' % media_url,
]
# Set our custom theme dir for mkdocs
if 'theme_dir' not in user_config and self.use_theme:
user_config['theme_dir'] = TEMPLATE_DIR
yaml.dump(user_config,
open(os.path.join(self.root_path, 'mkdocs.yml'), 'w'))
# RTD javascript writing
# Will be available in the JavaScript as READTHEDOCS_DATA.
readthedocs_data = {
'project':
self.version.project.slug,
'version':
self.version.slug,
'language':
self.version.project.language,
'page':
None,
'theme':
"readthedocs",
'builder':
"mkdocs",
'docroot':
docs_dir,
'source_suffix':
".md",
'api_host':
getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'),
'commit':
self.version.project.vcs_repo(self.version.slug).commit,
}
data_json = json.dumps(readthedocs_data, indent=4)
data_ctx = {
'data_json': data_json,
'current_version': readthedocs_data['version'],
'slug': readthedocs_data['project'],
'html_theme': readthedocs_data['theme'],
'pagename': None,
}
data_string = template_loader.get_template(
'doc_builder/data.js.tmpl').render(data_ctx)
data_file = open(
os.path.join(self.root_path, docs_dir, 'readthedocs-data.js'),
'w+')
data_file.write(data_string)
data_file.write('''
READTHEDOCS_DATA["page"] = mkdocs_page_input_path.substr(
0, mkdocs_page_input_path.lastIndexOf(READTHEDOCS_DATA.source_suffix));
''')
data_file.close()
include_ctx = {
'global_analytics_code':
getattr(settings, 'GLOBAL_ANALYTICS_CODE', 'UA-17997319-1'),
'user_analytics_code':
self.version.project.analytics_code,
}
include_string = template_loader.get_template(
'doc_builder/include.js.tmpl').render(include_ctx)
include_file = open(
os.path.join(self.root_path, docs_dir,
'readthedocs-dynamic-include.js'), 'w+')
include_file.write(include_string)
include_file.close()
def run(self, pk, version_pk=None, build_pk=None, record=True,
docker=None, search=True, force=False, localmedia=True, **__):
"""
Run a documentation sync n' build.
This is fully wrapped in exception handling to account for a number of
failure cases. We first run a few commands in a local build environment,
but do not report on environment success. This avoids a flicker on the
build output page where the build is marked as finished in between the
local environment steps and the docker build steps.
If a failure is raised, or the build is not successful, return
``False``, otherwise, ``True``.
Unhandled exceptions raise a generic user facing error, which directs
the user to bug us. It is therefore a benefit to have as few unhandled
errors as possible.
:param pk int: Project id
:param version_pk int: Project Version id (latest if None)
:param build_pk int: Build id (if None, commands are not recorded)
:param record bool: record a build object in the database
:param docker bool: use docker to build the project (if ``None``,
``settings.DOCKER_ENABLE`` is used)
:param search bool: update search
:param force bool: force Sphinx build
:param localmedia bool: update localmedia
:returns: whether build was successful or not
:rtype: bool
"""
try:
if docker is None:
docker = settings.DOCKER_ENABLE
self.project = self.get_project(pk)
self.version = self.get_version(self.project, version_pk)
self.build = self.get_build(build_pk)
self.build_search = search
self.build_localmedia = localmedia
self.build_force = force
self.config = None
setup_successful = self.run_setup(record=record)
if not setup_successful:
return False
# Catch unhandled errors in the setup step
except Exception as e: # noqa
log.exception(
'An unhandled exception was raised during build setup',
extra={'tags': {'build': build_pk}}
)
self.setup_env.failure = BuildEnvironmentError(
BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
build_id=build_pk,
)
)
self.setup_env.update_build(BUILD_STATE_FINISHED)
return False
else:
# No exceptions in the setup step, catch unhandled errors in the
# build steps
try:
self.run_build(docker=docker, record=record)
except Exception as e: # noqa
log.exception(
'An unhandled exception was raised during project build',
extra={'tags': {'build': build_pk}}
)
self.build_env.failure = BuildEnvironmentError(
BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
build_id=build_pk,
)
)
self.build_env.update_build(BUILD_STATE_FINISHED)
return False
return True
def run(self,
pk,
version_pk=None,
build_pk=None,
record=True,
docker=False,
search=True,
force=False,
localmedia=True,
**kwargs):
self.project = self.get_project(pk)
self.version = self.get_version(self.project, version_pk)
self.build = self.get_build(build_pk)
self.build_search = search
self.build_localmedia = localmedia
self.build_force = force
self.config = None
env_cls = LocalEnvironment
self.setup_env = env_cls(project=self.project,
version=self.version,
build=self.build,
record=record,
report_build_success=False)
# Environment used for code checkout & initial configuration reading
with self.setup_env:
if self.project.skip:
raise BuildEnvironmentError(
_('Builds for this project are temporarily disabled'))
try:
self.setup_vcs()
except vcs_support_utils.LockTimeout as e:
self.retry(exc=e, throw=False)
raise BuildEnvironmentError(
'Version locked, retrying in 5 minutes.', status_code=423)
self.config = load_yaml_config(version=self.version)
if self.setup_env.failed or self.config is None:
self.send_notifications()
self.setup_env.update_build(state=BUILD_STATE_FINISHED)
return None
if self.setup_env.successful and not self.project.has_valid_clone:
self.set_valid_clone()
env_vars = self.get_env_vars()
if docker or settings.DOCKER_ENABLE:
env_cls = DockerEnvironment
self.build_env = env_cls(project=self.project,
version=self.version,
build=self.build,
record=record,
environment=env_vars)
# Environment used for building code, usually with Docker
with self.build_env:
if self.project.documentation_type == 'auto':
self.update_documentation_type()
python_env_cls = Virtualenv
if self.config.use_conda:
self._log('Using conda')
python_env_cls = Conda
self.python_env = python_env_cls(version=self.version,
build_env=self.build_env,
config=self.config)
self.setup_environment()
# TODO the build object should have an idea of these states, extend
# the model to include an idea of these outcomes
outcomes = self.build_docs()
build_id = self.build.get('id')
# Web Server Tasks
if build_id:
finish_build.delay(
version_pk=self.version.pk,
build_pk=build_id,
hostname=socket.gethostname(),
html=outcomes['html'],
search=outcomes['search'],
localmedia=outcomes['localmedia'],
pdf=outcomes['pdf'],
epub=outcomes['epub'],
)
if self.build_env.failed:
self.send_notifications()
build_complete.send(sender=Build, build=self.build_env.build)
def run_build(self, docker, record):
"""
Build the docs in an environment.
:param docker: if ``True``, the build uses a ``DockerBuildEnvironment``,
otherwise it uses a ``LocalBuildEnvironment`` to run all the
commands to build the docs
:param record: whether or not record all the commands in the ``Build``
instance
"""
env_vars = self.get_env_vars()
if docker:
env_cls = DockerBuildEnvironment
else:
env_cls = LocalBuildEnvironment
self.build_env = env_cls(project=self.project,
version=self.version,
config=self.config,
build=self.build,
record=record,
environment=env_vars)
# Environment used for building code, usually with Docker
with self.build_env:
if self.project.documentation_type == 'auto':
self.update_documentation_type()
python_env_cls = Virtualenv
if self.config.use_conda:
self._log('Using conda')
python_env_cls = Conda
self.python_env = python_env_cls(version=self.version,
build_env=self.build_env,
config=self.config)
try:
self.setup_python_environment()
# TODO the build object should have an idea of these states, extend
# the model to include an idea of these outcomes
outcomes = self.build_docs()
build_id = self.build.get('id')
except vcs_support_utils.LockTimeout as e:
self.task.retry(exc=e, throw=False)
raise BuildEnvironmentError(
'Version locked, retrying in 5 minutes.', status_code=423)
except SoftTimeLimitExceeded:
raise BuildEnvironmentError(_('Build exited due to time out'))
# Finalize build and update web servers
if build_id:
self.update_app_instances(
html=bool(outcomes['html']),
search=bool(outcomes['search']),
localmedia=bool(outcomes['localmedia']),
pdf=bool(outcomes['pdf']),
epub=bool(outcomes['epub']),
)
else:
log.warning('No build ID, not syncing files')
if self.build_env.failed:
self.send_notifications()
build_complete.send(sender=Build, build=self.build_env.build)
