def test_build(self):
'''Test full build'''
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
versions=[fixture()])
version = project.versions.all()[0]
self.mocks.configure_mock('api_versions', {'return_value': [version]})
self.mocks.configure_mock(
'api', {'get.return_value': {
'downloads': "no_url_here"
}})
self.mocks.patches['html_build'].stop()
build_env = LocalEnvironment(project=project,
version=version,
build={})
python_env = Virtualenv(version=version, build_env=build_env)
yaml_config = get_build_config({})
config = ConfigWrapper(version=version, yaml_config=yaml_config)
task = UpdateDocsTask(build_env=build_env,
project=project,
python_env=python_env,
version=version,
search=False,
localmedia=False,
config=config)
task.build_docs()
# Get command and check first part of command list is a call to sphinx
self.assertEqual(self.mocks.popen.call_count, 1)
cmd = self.mocks.popen.call_args_list[0][0]
self.assertRegexpMatches(cmd[0][0], r'python')
self.assertRegexpMatches(cmd[0][1], r'sphinx-build')
def test_build_respects_epub_flag(self):
'''Test build with epub enabled'''
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=False,
enable_epub_build=True,
versions=[fixture()])
version = project.versions.all()[0]
build_env = LocalEnvironment(project=project,
version=version,
build={})
python_env = Virtualenv(version=version, build_env=build_env)
yaml_config = get_build_config({})
config = ConfigWrapper(version=version, yaml_config=yaml_config)
task = UpdateDocsTask(build_env=build_env,
project=project,
python_env=python_env,
version=version,
search=False,
localmedia=False,
config=config)
task.build_docs()
# The HTML and the Epub format were built.
self.mocks.html_build.assert_called_once_with()
self.mocks.epub_build.assert_called_once_with()
# PDF however was disabled and therefore not built.
self.assertFalse(self.mocks.pdf_build.called)
def test_build(self):
'''Test full build'''
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
versions=[fixture()])
version = project.versions.all()[0]
self.mocks.configure_mock('api_versions', {'return_value': [version]})
self.mocks.configure_mock('api', {
'get.return_value': {'downloads': "no_url_here"}
})
self.mocks.patches['html_build'].stop()
build_env = LocalEnvironment(project=project, version=version, build={})
python_env = Virtualenv(version=version, build_env=build_env)
config = ConfigWrapper(version=version, yaml_config=create_load()()[0])
task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env,
version=version, search=False, localmedia=False, config=config)
task.build_docs()
# Get command and check first part of command list is a call to sphinx
self.assertEqual(self.mocks.popen.call_count, 3)
cmd = self.mocks.popen.call_args_list[2][0]
self.assertRegexpMatches(cmd[0][0], r'python')
self.assertRegexpMatches(cmd[0][1], r'sphinx-build')
def test_build_respects_pdf_flag(self):
'''Build output format control'''
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=True,
enable_epub_build=False,
versions=[fixture()])
version = project.versions.all()[0]
build_env = LocalEnvironment(project=project,
version=version,
build={})
task = UpdateDocsTask(build_env=build_env,
version=version,
project=project,
search=False,
localmedia=False)
built_docs = task.build_docs()
# The HTML and the Epub format were built.
self.mocks.html_build.assert_called_once_with()
self.mocks.pdf_build.assert_called_once_with()
# PDF however was disabled and therefore not built.
self.assertFalse(self.mocks.epub_build.called)
def test_build_respects_yaml(self):
'''Test YAML build options'''
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=False,
enable_epub_build=False,
versions=[fixture()])
version = project.versions.all()[0]
build_env = LocalEnvironment(project=project, version=version, build={})
python_env = Virtualenv(version=version, build_env=build_env)
config = ConfigWrapper(version=version, yaml_config=create_load({
'formats': ['epub']
})()[0])
task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env,
version=version, search=False, localmedia=False, config=config)
task.build_docs()
# The HTML and the Epub format were built.
self.mocks.html_build.assert_called_once_with()
self.mocks.epub_build.assert_called_once_with()
# PDF however was disabled and therefore not built.
self.assertFalse(self.mocks.pdf_build.called)
def trigger_build(project,
version=None,
record=True,
force=False,
basic=False):
"""Trigger build for project and version
If project has a ``build_queue``, execute task on this build queue. Queue
will be prefixed with ``build-`` to unify build queue names.
"""
# Avoid circular import
from readthedocs.projects.tasks import UpdateDocsTask
from readthedocs.builds.models import Build
if project.skip:
return None
if not version:
version = project.versions.get(slug=LATEST)
kwargs = dict(
pk=project.pk,
version_pk=version.pk,
record=record,
force=force,
basic=basic,
)
build = None
if record:
build = Build.objects.create(
project=project,
version=version,
type='html',
state='triggered',
success=True,
)
kwargs['build_pk'] = build.pk
options = {}
if project.build_queue:
options['queue'] = project.build_queue
# Set per-task time limit
time_limit = DOCKER_LIMITS['time']
try:
if project.container_time_limit:
time_limit = int(project.container_time_limit)
except ValueError:
pass
# Add 20% overhead to task, to ensure the build can timeout and the task
# will cleanly finish.
options['soft_time_limit'] = time_limit
options['time_limit'] = int(time_limit * 1.2)
update_docs = UpdateDocsTask()
update_docs.apply_async(kwargs=kwargs, **options)
return build
def trigger_build(project, version=None, record=True, force=False, basic=False):
"""
Trigger build for project and version.
If project has a ``build_queue``, execute task on this build queue. Queue
will be prefixed with ``build-`` to unify build queue names.
"""
# Avoid circular import
from readthedocs.projects.tasks import UpdateDocsTask
from readthedocs.builds.models import Build
if project.skip:
return None
if not version:
version = project.versions.get(slug=LATEST)
kwargs = dict(
pk=project.pk,
version_pk=version.pk,
record=record,
force=force,
basic=basic,
)
build = None
if record:
build = Build.objects.create(
project=project,
version=version,
type='html',
state='triggered',
success=True,
)
kwargs['build_pk'] = build.pk
options = {}
if project.build_queue:
options['queue'] = project.build_queue
# Set per-task time limit
time_limit = DOCKER_LIMITS['time']
try:
if project.container_time_limit:
time_limit = int(project.container_time_limit)
except ValueError:
pass
# Add 20% overhead to task, to ensure the build can timeout and the task
# will cleanly finish.
options['soft_time_limit'] = time_limit
options['time_limit'] = int(time_limit * 1.2)
update_docs = UpdateDocsTask()
update_docs.apply_async(kwargs=kwargs, **options)
return build
def test_build_pdf_latex_not_failure(self):
'''Test pass during PDF builds and bad latex failure status code'''
self.mocks.patches['html_build'].stop()
self.mocks.patches['pdf_build'].stop()
project = get(Project,
slug='project-2',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=True,
enable_epub_build=False,
versions=[fixture()])
version = project.versions.all()[0]
assert project.conf_dir() == '/tmp/rtd'
build_env = LocalEnvironment(project=project,
version=version,
build={})
python_env = Virtualenv(version=version, build_env=build_env)
yaml_config = get_build_config({})
config = ConfigWrapper(version=version, yaml_config=yaml_config)
task = UpdateDocsTask(build_env=build_env,
project=project,
python_env=python_env,
version=version,
search=False,
localmedia=False,
config=config)
# Mock out the separate calls to Popen using an iterable side_effect
returns = [
(('', ''), 0), # sphinx-build html
(('', ''), 0), # sphinx-build pdf
(('Output written on foo.pdf', ''), 1), # latex
(('', ''), 0), # makeindex
(('', ''), 0), # latex
]
mock_obj = mock.Mock()
mock_obj.communicate.side_effect = [
output for (output, status) in returns
]
type(mock_obj).returncode = mock.PropertyMock(
side_effect=[status for (output, status) in returns])
self.mocks.popen.return_value = mock_obj
with build_env:
task.build_docs()
self.assertEqual(self.mocks.popen.call_count, 5)
self.assertTrue(build_env.successful)
def test_build_pdf_latex_not_failure(self):
'''Test pass during PDF builds and bad latex failure status code'''
if six.PY3:
import pytest
pytest.xfail(
"test_build_pdf_latex_not_failure is known to fail on 3.6")
self.mocks.patches['html_build'].stop()
self.mocks.patches['pdf_build'].stop()
project = get(Project,
slug='project-2',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=True,
enable_epub_build=False,
versions=[fixture()])
version = project.versions.all()[0]
assert project.conf_dir() == '/tmp/rtd'
build_env = LocalEnvironment(project=project, version=version, build={})
python_env = Virtualenv(version=version, build_env=build_env)
config = ConfigWrapper(version=version, yaml_config=create_load()()[0])
task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env,
version=version, search=False, localmedia=False, config=config)
# Mock out the separate calls to Popen using an iterable side_effect
returns = [
(('', ''), 0), # sphinx-build html
(('', ''), 0), # sphinx-build pdf
(('Output written on foo.pdf', ''), 1), # latex
(('', ''), 0), # makeindex
(('', ''), 0), # latex
]
mock_obj = mock.Mock()
mock_obj.communicate.side_effect = [output for (output, status)
in returns]
type(mock_obj).returncode = mock.PropertyMock(
side_effect=[status for (output, status) in returns])
self.mocks.popen.return_value = mock_obj
with build_env:
task.build_docs()
self.assertEqual(self.mocks.popen.call_count, 7)
self.assertTrue(build_env.successful)
def test_build_pdf_latex_failures(self):
'''Build failure if latex fails'''
if six.PY3:
import pytest
pytest.xfail(
"test_build_pdf_latex_failures is known to fail on 3.6")
self.mocks.patches['html_build'].stop()
self.mocks.patches['pdf_build'].stop()
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=True,
enable_epub_build=False,
versions=[fixture()])
version = project.versions.all()[0]
assert project.conf_dir() == '/tmp/rtd'
build_env = LocalEnvironment(project=project, version=version, build={})
python_env = Virtualenv(version=version, build_env=build_env)
config = ConfigWrapper(version=version, yaml_config=create_load()()[0])
task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env,
version=version, search=False, localmedia=False, config=config)
# Mock out the separate calls to Popen using an iterable side_effect
returns = [
(('', ''), 0), # sphinx-build html
(('', ''), 0), # sphinx-build pdf
(('', ''), 1), # latex
(('', ''), 0), # makeindex
(('', ''), 0), # latex
]
mock_obj = mock.Mock()
mock_obj.communicate.side_effect = [output for (output, status)
in returns]
type(mock_obj).returncode = mock.PropertyMock(
side_effect=[status for (output, status) in returns])
self.mocks.popen.return_value = mock_obj
with build_env:
task.build_docs()
self.assertEqual(self.mocks.popen.call_count, 7)
self.assertTrue(build_env.failed)
def test_build_pdf_latex_failures(self):
'''Build failure if latex fails'''
self.mocks.patches['html_build'].stop()
self.mocks.patches['pdf_build'].stop()
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=True,
enable_epub_build=False,
versions=[fixture()])
version = project.versions.all()[0]
assert project.conf_dir() == '/tmp/rtd'
build_env = LocalEnvironment(project=project, version=version, build={})
python_env = Virtualenv(version=version, build_env=build_env)
yaml_config = get_build_config({})
config = ConfigWrapper(version=version, yaml_config=yaml_config)
task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env,
version=version, search=False, localmedia=False, config=config)
# Mock out the separate calls to Popen using an iterable side_effect
returns = [
(('', ''), 0), # sphinx-build html
(('', ''), 0), # sphinx-build pdf
(('', ''), 1), # latex
(('', ''), 0), # makeindex
(('', ''), 0), # latex
]
mock_obj = mock.Mock()
mock_obj.communicate.side_effect = [output for (output, status)
in returns]
type(mock_obj).returncode = mock.PropertyMock(
side_effect=[status for (output, status) in returns])
self.mocks.popen.return_value = mock_obj
with build_env:
task.build_docs()
self.assertEqual(self.mocks.popen.call_count, 7)
self.assertTrue(build_env.failed)
def test_build_respects_epub_flag(self):
'''Test build with epub enabled'''
project = get(Project,
slug='project-1',
documentation_type='sphinx',
conf_py_file='test_conf.py',
enable_pdf_build=False,
enable_epub_build=True,
versions=[fixture()])
version = project.versions.all()[0]
build_env = LocalEnvironment(project=project, version=version, build={})
task = UpdateDocsTask(build_env=build_env, project=project,
version=version, search=False, localmedia=False)
built_docs = task.build_docs()
# The HTML and the Epub format were built.
self.mocks.html_build.assert_called_once_with()
self.mocks.epub_build.assert_called_once_with()
# PDF however was disabled and therefore not built.
self.assertFalse(self.mocks.pdf_build.called)
def handle(self, *args, **options):
for version in Version.objects.filter(active=True, built=False):
update_docs = UpdateDocsTask()
update_docs.run(version.project_id,
record=False,
version_pk=version.pk)
def handle(self, *args, **options):
for version in Version.objects.filter(active=True, built=False):
update_docs = UpdateDocsTask()
update_docs.run(version.project_id, record=False,
version_pk=version.pk)
def prepare_build(
project, version=None, record=True, force=False, immutable=True):
"""
Prepare a build in a Celery task for project and version.
If project has a ``build_queue``, execute the task on this build queue. If
project has ``skip=True``, the build is not triggered.
:param project: project's documentation to be built
:param version: version of the project to be built. Default: ``latest``
:param record: whether or not record the build in a new Build object
:param force: build the HTML documentation even if the files haven't changed
:param immutable: whether or not create an immutable Celery signature
:returns: Celery signature of UpdateDocsTask to be executed
"""
# Avoid circular import
from readthedocs.projects.tasks import UpdateDocsTask
from readthedocs.builds.models import Build
if project.skip:
log.info(
'Build not triggered because Project.skip=True: project=%s',
project.slug,
)
return None
if not version:
version = project.versions.get(slug=LATEST)
kwargs = {
'version_pk': version.pk,
'record': record,
'force': force,
}
if record:
build = Build.objects.create(
project=project,
version=version,
type='html',
state=BUILD_STATE_TRIGGERED,
success=True,
)
kwargs['build_pk'] = build.pk
options = {}
if project.build_queue:
options['queue'] = project.build_queue
# Set per-task time limit
time_limit = DOCKER_LIMITS['time']
try:
if project.container_time_limit:
time_limit = int(project.container_time_limit)
except ValueError:
log.warning('Invalid time_limit for project: %s', project.slug)
# Add 20% overhead to task, to ensure the build can timeout and the task
# will cleanly finish.
options['soft_time_limit'] = time_limit
options['time_limit'] = int(time_limit * 1.2)
update_docs_task = UpdateDocsTask()
# Py 2.7 doesn't support ``**`` expand syntax twice. We create just one big
# kwargs (including the options) for this and expand it just once.
# return update_docs_task.si(project.pk, **kwargs, **options)
kwargs.update(options)
return update_docs_task.si(project.pk, **kwargs)
def prepare_build(
project, version=None, record=True, force=False, immutable=True):
"""
Prepare a build in a Celery task for project and version.
If project has a ``build_queue``, execute the task on this build queue. If
project has ``skip=True``, the build is not triggered.
:param project: project's documentation to be built
:param version: version of the project to be built. Default: ``latest``
:param record: whether or not record the build in a new Build object
:param force: build the HTML documentation even if the files haven't changed
:param immutable: whether or not create an immutable Celery signature
:returns: Celery signature of UpdateDocsTask to be executed
"""
# Avoid circular import
from readthedocs.projects.tasks import UpdateDocsTask
from readthedocs.builds.models import Build
if project.skip:
log.info(
'Build not triggered because Project.skip=True: project=%s',
project.slug,
)
return None
if not version:
version = project.versions.get(slug=LATEST)
kwargs = {
'version_pk': version.pk,
'record': record,
'force': force,
}
if record:
build = Build.objects.create(
project=project,
version=version,
type='html',
state=BUILD_STATE_TRIGGERED,
success=True,
)
kwargs['build_pk'] = build.pk
options = {}
if project.build_queue:
options['queue'] = project.build_queue
# Set per-task time limit
time_limit = DOCKER_LIMITS['time']
try:
if project.container_time_limit:
time_limit = int(project.container_time_limit)
except ValueError:
log.warning('Invalid time_limit for project: %s', project.slug)
# Add 20% overhead to task, to ensure the build can timeout and the task
# will cleanly finish.
options['soft_time_limit'] = time_limit
options['time_limit'] = int(time_limit * 1.2)
update_docs_task = UpdateDocsTask()
# Py 2.7 doesn't support ``**`` expand syntax twice. We create just one big
# kwargs (including the options) for this and expand it just once.
# return update_docs_task.si(project.pk, **kwargs, **options)
kwargs.update(options)
return update_docs_task.si(project.pk, **kwargs)