def test_extra_requirements(self, load_config): load_config.side_effect = create_load({ 'python': { 'pip_install': True, 'extra_requirements': ['tests', 'docs'] } }) config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, ['tests', 'docs']) load_config.side_effect = create_load({ 'python': { 'extra_requirements': ['tests', 'docs'] } }) config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, []) load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, []) load_config.side_effect = create_load({ 'python': { 'setup_py_install': True, 'extra_requirements': ['tests', 'docs'] } }) config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, [])
def test_python_invalid_version_in_config(self, load_config): load_config.side_effect = create_load({ 'python': {'version': 2.6}, }) self.project.container_image = 'readthedocs/build:2.0' self.project.save() with self.assertRaises(InvalidConfig): load_yaml_config(self.version)
def test_install_project(self, load_config): load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.install_project, False) load_config.side_effect = create_load({ 'python': {'setup_py_install': True} }) config = load_yaml_config(self.version) self.assertEqual(config.install_project, True)
def test_requirements_file(self, load_config): requirements_file = 'wsgi.py' load_config.side_effect = create_load({ 'requirements_file': requirements_file }) config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, requirements_file) load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, 'urls.py')
def test_extra_requirements(self, load_config): load_config.side_effect = create_load({ 'python': { 'pip_install': True, 'extra_requirements': ['tests', 'docs'], }, }) config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 2) self.assertTrue( isinstance(config.python.install[0], PythonInstallRequirements) ) self.assertEqual( config.python.install[1].extra_requirements, ['tests', 'docs'] ) load_config.side_effect = create_load({ 'python': { 'extra_requirements': ['tests', 'docs'], }, }) config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 1) self.assertTrue( isinstance(config.python.install[0], PythonInstallRequirements) ) load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 1) self.assertTrue( isinstance(config.python.install[0], PythonInstallRequirements) ) load_config.side_effect = create_load({ 'python': { 'setup_py_install': True, 'extra_requirements': ['tests', 'docs'], }, }) config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 2) self.assertTrue( isinstance(config.python.install[0], PythonInstallRequirements) ) self.assertEqual( config.python.install[1].extra_requirements, [] )
def test_conda(self, load_config): to_find = 'urls.py' load_config.side_effect = create_load({ 'conda': { 'file': to_find } }) config = load_yaml_config(self.version) self.assertEqual(config.use_conda, True) self.assertTrue(config.conda_file[-len(to_find):] == to_find) load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.use_conda, False) self.assertEqual(config.conda_file, None)
def test_requirements_file(self, load_config): if six.PY3: import pytest pytest.xfail("test_requirements_file is known to fail on 3.6") requirements_file = 'wsgi.py' if six.PY2 else 'readthedocs/wsgi.py' load_config.side_effect = create_load({ 'requirements_file': requirements_file }) config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, requirements_file) load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, 'urls.py')
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force env_cls = LocalEnvironment self.setup_env = env_cls(project=self.project, version=self.version, build=self.build, record=record) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout, e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) self.config = load_yaml_config(version=self.version)
def test_python_supported_versions_image_latest(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:latest' self.project.save() config = load_yaml_config(self.version) self.assertEqual(config._yaml_config.get_valid_python_versions(), [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6])
def test_build_respects_pdf_flag(self, load_config): '''Build output format control''' load_config.side_effect = create_load() project = get(Project, slug='project-1', documentation_type='sphinx', conf_py_file='test_conf.py', enable_pdf_build=True, enable_epub_build=False, versions=[fixture()]) version = project.versions.all()[0] build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() # The HTML and the Epub format were built. self.mocks.html_build.assert_called_once_with() self.mocks.pdf_build.assert_called_once_with() # PDF however was disabled and therefore not built. self.assertFalse(self.mocks.epub_build.called)
def test_build(self, load_config): '''Test full build''' load_config.side_effect = create_load() project = get(Project, slug='project-1', documentation_type='sphinx', conf_py_file='test_conf.py', versions=[fixture()]) version = project.versions.all()[0] self.mocks.configure_mock('api_versions', {'return_value': [version]}) self.mocks.configure_mock('api', { 'get.return_value': {'downloads': "no_url_here"} }) self.mocks.patches['html_build'].stop() build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() # Get command and check first part of command list is a call to sphinx self.assertEqual(self.mocks.popen.call_count, 3) cmd = self.mocks.popen.call_args_list[2][0] self.assertRegexpMatches(cmd[0][0], r'python') self.assertRegexpMatches(cmd[0][1], r'sphinx-build')
def test_dont_localmedia_build_pdf_epub_search_in_mkdocs(self, load_config): load_config.side_effect = create_load() project = get( Project, slug='project-1', documentation_type='mkdocs', enable_pdf_build=True, enable_epub_build=True, versions=[fixture()], ) version = project.versions.all().first() build_env = LocalBuildEnvironment( project=project, version=version, build={}, ) python_env = Virtualenv(version=version, build_env=build_env) config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, version=version, config=config, ) task.build_docs() # Only html for mkdocs was built self.mocks.html_build_mkdocs.assert_called_once() self.mocks.html_build.assert_not_called() self.mocks.localmedia_build.assert_not_called() self.mocks.pdf_build.assert_not_called() self.mocks.epub_build.assert_not_called()
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force self.config = load_yaml_config(version=self.version) env_cls = LocalEnvironment if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record) python_env_cls = Virtualenv if self.config.use_conda: python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) with self.build_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout, e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) if self.project.documentation_type == 'auto': self.update_documentation_type() self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], )
def test_python_supported_versions_default_image_1_0(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:1.0' self.project.enable_epub_build = True self.project.enable_pdf_build = True self.project.save() config = load_yaml_config(self.version) expected_env_config = { 'build': {'image': 'readthedocs/build:1.0'}, 'defaults': { 'install_project': self.project.install_project, 'formats': [ 'htmlzip', 'epub', 'pdf' ], 'use_system_packages': self.project.use_system_packages, 'requirements_file': self.project.requirements_file, 'python_version': 3, 'sphinx_configuration': mock.ANY, 'build_image': 'readthedocs/build:1.0', 'doctype': self.project.documentation_type, }, } img_settings = DOCKER_IMAGE_SETTINGS.get(self.project.container_image, None) if img_settings: expected_env_config.update(img_settings) load_config.assert_called_once_with( path=mock.ANY, env_config=expected_env_config, ) self.assertEqual(config.python.version, 3)
def test_python_set_python_version_on_project(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:2.0' self.project.python_interpreter = 'python3' self.project.save() config = load_yaml_config(self.version) self.assertEqual(config.python_version, 3) self.assertEqual(config.python_interpreter, 'python3.5')
def __init__(self, version, build_env, config=None): self.version = version self.project = version.project self.build_env = build_env if config: self.config = config else: self.config = load_yaml_config(version) # Compute here, since it's used a lot self.checkout_path = self.project.checkout_path(self.version.slug)
def test_install_project(self, load_config): load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 1) self.assertTrue( isinstance(config.python.install[0], PythonInstallRequirements) ) load_config.side_effect = create_load({ 'python': {'setup_py_install': True}, }) config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 2) self.assertTrue( isinstance(config.python.install[0], PythonInstallRequirements) ) self.assertEqual( config.python.install[1].method, SETUPTOOLS )
def get_update_docs_task(self): build_env = LocalBuildEnvironment( self.project, self.version, record=False, ) update_docs = tasks.UpdateDocsTaskStep( build_env=build_env, config=load_yaml_config(self.version), project=self.project, version=self.version, ) return update_docs
def run_setup(self, record=True): """ Run setup in the local environment. Return True if successful. """ self.setup_env = LocalEnvironment( project=self.project, version=self.version, build=self.build, record=record, update_on_success=False, ) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout as e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) try: self.config = load_yaml_config(version=self.version) except ConfigError as e: raise BuildEnvironmentError( 'Problem parsing YAML configuration. {0}'.format(str(e)) ) if self.setup_env.failure or self.config is None: self._log('Failing build because of setup failure: %s' % self.setup_env.failure) # Send notification to users only if the build didn't fail because of # LockTimeout: this exception occurs when a build is triggered before the previous # one has finished (e.g. two webhooks, one after the other) if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout): self.send_notifications() return False if self.setup_env.successful and not self.project.has_valid_clone: self.set_valid_clone() return True
def test_build_pdf_latex_not_failure(self, load_config): """Test pass during PDF builds and bad latex failure status code.""" load_config.side_effect = create_load() self.mocks.patches['html_build'].stop() self.mocks.patches['pdf_build'].stop() project = get( Project, slug='project-2', documentation_type='sphinx', conf_py_file='test_conf.py', enable_pdf_build=True, enable_epub_build=False, versions=[fixture()], ) version = project.versions.all()[0] assert project.conf_dir() == '/tmp/rtd' build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, version=version, config=config, ) # Mock out the separate calls to Popen using an iterable side_effect returns = [ ((b'', b''), 0), # sphinx-build html ((b'', b''), 0), # sphinx-build pdf ((b'Output written on foo.pdf', b''), 1), # latex ((b'', b''), 0), # makeindex ((b'', b''), 0), # latex ] mock_obj = mock.Mock() mock_obj.communicate.side_effect = [ output for (output, status) in returns ] type(mock_obj).returncode = mock.PropertyMock( side_effect=[status for (output, status) in returns], ) self.mocks.popen.return_value = mock_obj with build_env: task.build_docs() self.assertEqual(self.mocks.popen.call_count, 7) self.assertTrue(build_env.successful)
def test_python_supported_versions_default_image_1_0(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:1.0' self.project.save() config = load_yaml_config(self.version) self.assertEqual(load_config.call_count, 1) load_config.assert_has_calls([ mock.call(path=mock.ANY, env_config={ 'build': {'image': 'readthedocs/build:1.0'}, 'type': 'sphinx', 'output_base': '', 'name': mock.ANY }), ]) self.assertEqual(config.python_version, 2)
def test_python_supported_versions_image_latest(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:latest' self.project.save() config = load_yaml_config(self.version) self.assertEqual(load_config.call_count, 1) load_config.assert_has_calls([ mock.call(path=mock.ANY, env_config={ 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, 'type': 'sphinx', 'output_base': '', 'name': mock.ANY }), ]) self.assertEqual(config.python_version, 2)
def test_requirements_file_from_project_setting(self, checkout_path): base_path = tempfile.mkdtemp() checkout_path.return_value = base_path requirements_file = 'requirements.txt' self.project.requirements_file = requirements_file self.project.save() full_requirements_file = path.join(base_path, requirements_file) with open(full_requirements_file, 'w') as f: f.write('pip') config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 1) self.assertEqual( config.python.install[0].requirements, full_requirements_file )
def test_conda_with_cofig(self, checkout_path): base_path = tempfile.mkdtemp() checkout_path.return_value = base_path conda_file = 'environmemt.yml' full_conda_file = path.join(base_path, conda_file) with open(full_conda_file, 'w') as f: f.write('conda') create_config_file( { 'conda': { 'file': conda_file, }, }, base_path=base_path, ) config = load_yaml_config(self.version) self.assertTrue(config.conda is not None) self.assertEqual(config.conda.environment, full_conda_file)
def test_requirements_file_from_yml(self, checkout_path): base_path = tempfile.mkdtemp() checkout_path.return_value = base_path self.project.requirements_file = 'no-existent-file.txt' self.project.save() requirements_file = 'requirements.txt' full_requirements_file = path.join(base_path, requirements_file) with open(full_requirements_file, 'w') as f: f.write('pip') create_config_file( { 'requirements_file': requirements_file, }, base_path=base_path, ) config = load_yaml_config(self.version) self.assertEqual(len(config.python.install), 1) self.assertEqual( config.python.install[0].requirements, full_requirements_file )
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force env_cls = LocalEnvironment if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record) with self.build_env: # Setup config inside of build environment so we can handle exceptions properly self.config = load_yaml_config(version=self.version) python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout, e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423) if self.project.documentation_type == 'auto': self.update_documentation_type() self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], )
def test_python_supported_versions_image_2_0(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:2.0' self.project.save() config = load_yaml_config(self.version) self.assertEqual(config.get_valid_python_versions(), [2, 2.7, 3, 3.5])
def test_python_invalid_version_in_config(self, load_config): load_config.side_effect = create_load({'python': {'version': 2.6}}) self.project.container_image = 'readthedocs/build:2.0' self.project.save() with self.assertRaises(InvalidConfig): load_yaml_config(self.version)
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force self.config = None env_cls = LocalEnvironment self.setup_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, report_build_success=False) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout as e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) self.config = load_yaml_config(version=self.version) if self.setup_env.failed or self.config is None: self.send_notifications() self.setup_env.update_build(state=BUILD_STATE_FINISHED) return None if self.setup_env.successful and not self.project.has_valid_clone: self.set_valid_clone() env_vars = self.get_env_vars() if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker with self.build_env: if self.project.documentation_type == 'auto': self.update_documentation_type() python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], ) if self.build_env.failed: self.send_notifications()
def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, search=True, force=False, localmedia=True, **kwargs): self.project = self.get_project(pk) self.version = self.get_version(self.project, version_pk) self.build = self.get_build(build_pk) self.build_search = search self.build_localmedia = localmedia self.build_force = force self.config = None env_cls = LocalEnvironment self.setup_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, report_build_success=False) # Environment used for code checkout & initial configuration reading with self.setup_env: if self.project.skip: raise BuildEnvironmentError( _('Builds for this project are temporarily disabled')) try: self.setup_vcs() except vcs_support_utils.LockTimeout as e: self.retry(exc=e, throw=False) raise BuildEnvironmentError( 'Version locked, retrying in 5 minutes.', status_code=423 ) self.config = load_yaml_config(version=self.version) if self.setup_env.failed or self.config is None: self.send_notifications() return None if self.setup_env.successful and not self.project.has_valid_clone: self.set_valid_clone() env_vars = self.get_env_vars() if docker or settings.DOCKER_ENABLE: env_cls = DockerEnvironment self.build_env = env_cls(project=self.project, version=self.version, build=self.build, record=record, environment=env_vars) # Environment used for building code, usually with Docker with self.build_env: if self.project.documentation_type == 'auto': self.update_documentation_type() python_env_cls = Virtualenv if self.config.use_conda: self._log('Using conda') python_env_cls = Conda self.python_env = python_env_cls(version=self.version, build_env=self.build_env, config=self.config) self.setup_environment() # TODO the build object should have an idea of these states, extend # the model to include an idea of these outcomes outcomes = self.build_docs() build_id = self.build.get('id') # Web Server Tasks if build_id: finish_build.delay( version_pk=self.version.pk, build_pk=build_id, hostname=socket.gethostname(), html=outcomes['html'], search=outcomes['search'], localmedia=outcomes['localmedia'], pdf=outcomes['pdf'], epub=outcomes['epub'], ) if self.build_env.failed: self.send_notifications()
def test_conda_without_cofig(self, checkout_path): base_path = tempfile.mkdtemp() checkout_path.return_value = base_path config = load_yaml_config(self.version) self.assertIsNone(config.conda)
def test_conda_without_cofig(self, checkout_path): base_path = tempfile.mkdtemp() checkout_path.return_value = base_path config = load_yaml_config(self.version) self.assertIsNone(config.conda)
def test_python_default_version(self, load_config): load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.python.version, 3) self.assertEqual(config.python_interpreter, 'python3.7')
def test_python_default_version(self, load_config): load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.python_version, 2) self.assertEqual(config.python_interpreter, 'python2.7')