Exemple #1
0
    def test_incremental_state_update_with_no_update(self):
        """Build updates to a non-finished state when update_on_success=True."""
        build_envs = [
            LocalEnvironment(
                version=self.version,
                project=self.project,
                build={'id': DUMMY_BUILD_ID},
            ),
            LocalEnvironment(
                version=self.version,
                project=self.project,
                build={'id': DUMMY_BUILD_ID},
                update_on_success=False,
            ),
        ]

        for build_env in build_envs:
            with build_env:
                build_env.update_build(BUILD_STATE_CLONING)
                self.mocks.mocks['api_v2.build']().put.assert_called_with({
                    'id': DUMMY_BUILD_ID,
                    'version': self.version.pk,
                    'success': True,
                    'project': self.project.pk,
                    'setup_error': u'',
                    'length': mock.ANY,
                    'error': '',
                    'setup': u'',
                    'output': u'',
                    'state': BUILD_STATE_CLONING,
                    'builder': mock.ANY,
                    'exit_code': 0,
                })
Exemple #2
0
 def __init__(self, project):
     self.project = project
     self.project_root = os.path.join(
         self.WEB_ROOT,
         project.slug,
     )
     self.subproject_root = os.path.join(
         self.project_root,
         'projects',
     )
     self.environment = LocalEnvironment(project)
     self.sanity_check()
    def test_normal_execution(self):
        """Normal build in passing state"""
        self.mocks.configure_mock("process", {"communicate.return_value": ("This is okay", "")})
        type(self.mocks.process).returncode = PropertyMock(return_value=0)

        build_env = LocalEnvironment(version=self.version, project=self.project, build={})
        with build_env:
            build_env.run("echo", "test")
        self.assertTrue(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.successful)
        self.assertEqual(len(build_env.commands), 1)
        self.assertEqual(build_env.commands[0].output, u"This is okay")
    def test_failing_execution(self):
        """Build in failing state"""
        self.mocks.configure_mock("process", {"communicate.return_value": ("This is not okay", "")})
        type(self.mocks.process).returncode = PropertyMock(return_value=1)

        build_env = LocalEnvironment(version=self.version, project=self.project, build={})
        with build_env:
            build_env.run("echo", "test")
            self.fail("This should be unreachable")
        self.assertTrue(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
        self.assertEqual(len(build_env.commands), 1)
        self.assertEqual(build_env.commands[0].output, u"This is not okay")
Exemple #5
0
    def run_setup(self, record=True):
        """
        Run setup in the local environment.

        Return True if successful.
        """
        self.setup_env = LocalEnvironment(
            project=self.project,
            version=self.version,
            build=self.build,
            record=record,
            update_on_success=False,
        )

        # Environment used for code checkout & initial configuration reading
        with self.setup_env:
            if self.project.skip:
                raise BuildEnvironmentError(
                    _('Builds for this project are temporarily disabled'))
            try:
                self.setup_vcs()
            except vcs_support_utils.LockTimeout as e:
                self.retry(exc=e, throw=False)
                raise BuildEnvironmentError(
                    'Version locked, retrying in 5 minutes.',
                    status_code=423
                )

            try:
                self.config = load_yaml_config(version=self.version)
            except ConfigError as e:
                raise BuildEnvironmentError(
                    'Problem parsing YAML configuration. {0}'.format(str(e))
                )

        if self.setup_env.failure or self.config is None:
            self._log('Failing build because of setup failure: %s' % self.setup_env.failure)

            # Send notification to users only if the build didn't fail because of
            # LockTimeout: this exception occurs when a build is triggered before the previous
            # one has finished (e.g. two webhooks, one after the other)
            if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout):
                self.send_notifications()

            return False

        if self.setup_env.successful and not self.project.has_valid_clone:
            self.set_valid_clone()

        return True
Exemple #6
0
    def test_failing_execution(self):
        """Build in failing state."""
        self.mocks.configure_mock(
            'process', {'communicate.return_value': (b'This is not okay', '')})
        type(self.mocks.process).returncode = PropertyMock(return_value=1)

        build_env = LocalEnvironment(
            version=self.version,
            project=self.project,
            build={'id': DUMMY_BUILD_ID},
        )

        with build_env:
            build_env.run('echo', 'test')
            self.fail('This should be unreachable')
        self.assertTrue(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
        self.assertEqual(len(build_env.commands), 1)
        self.assertEqual(build_env.commands[0].output, u'This is not okay')

        # api() is not called anymore, we use api_v2 instead
        self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
        self.mocks.mocks['api_v2.build']().put.assert_called_with({
            'id':
            DUMMY_BUILD_ID,
            'version':
            self.version.pk,
            'success':
            False,
            'project':
            self.project.pk,
            'setup_error':
            u'',
            'length':
            mock.ANY,
            'error':
            '',
            'setup':
            u'',
            'output':
            u'',
            'state':
            u'finished',
            'builder':
            mock.ANY,
            'exit_code':
            1,
        })
    def test_normal_execution(self):
        '''Normal build in passing state'''
        self.mocks.configure_mock('process', {
            'communicate.return_value': ('This is okay', '')})
        type(self.mocks.process).returncode = PropertyMock(return_value=0)

        build_env = LocalEnvironment(version=self.version, project=self.project,
                                     build={})
        with build_env:
            build_env.run('echo', 'test')
        self.assertTrue(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.successful)
        self.assertEqual(len(build_env.commands), 1)
        self.assertEqual(build_env.commands[0].output, u'This is okay')
Exemple #8
0
    def test_failing_execution_with_unexpected_exception(self):
        """Build in failing state with exception from code."""
        build_env = LocalEnvironment(
            version=self.version,
            project=self.project,
            build={'id': DUMMY_BUILD_ID},
        )

        with build_env:
            raise ValueError('uncaught')

        self.assertFalse(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)

        # api() is not called anymore, we use api_v2 instead
        self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
        self.mocks.mocks['api_v2.build']().put.assert_called_with({
            'id': DUMMY_BUILD_ID,
            'version': self.version.pk,
            'success': False,
            'project': self.project.pk,
            'setup_error': u'',
            'length': mock.ANY,
            'error': (
                'There was a problem with Read the Docs while building your '
                'documentation. Please report this to us with your build id (123).'
            ),
            'setup': u'',
            'output': u'',
            'state': u'finished',
            'builder': mock.ANY,
        })
Exemple #9
0
    def test_failing_execution_with_caught_exception(self):
        """Build in failing state with BuildEnvironmentError exception."""
        build_env = LocalEnvironment(
            version=self.version,
            project=self.project,
            build={'id': DUMMY_BUILD_ID},
        )

        with build_env:
            raise BuildEnvironmentError('Foobar')

        self.assertFalse(self.mocks.process.communicate.called)
        self.assertEqual(len(build_env.commands), 0)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)

        # api() is not called anymore, we use api_v2 instead
        self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
        self.mocks.mocks['api_v2.build']().put.assert_called_with({
            'id': DUMMY_BUILD_ID,
            'version': self.version.pk,
            'success': False,
            'project': self.project.pk,
            'setup_error': u'',
            'length': mock.ANY,
            'error': 'Foobar',
            'setup': u'',
            'output': u'',
            'state': u'finished',
            'builder': mock.ANY,
            'exit_code': 1,
        })
Exemple #10
0
    def test_build_respects_pdf_flag(self):
        '''Build output format control'''
        project = get(Project,
                      slug='project-1',
                      documentation_type='sphinx',
                      conf_py_file='test_conf.py',
                      enable_pdf_build=True,
                      enable_epub_build=False,
                      versions=[fixture()])
        version = project.versions.all()[0]

        build_env = LocalEnvironment(project=project,
                                     version=version,
                                     build={})
        task = UpdateDocsTask(build_env=build_env,
                              version=version,
                              project=project,
                              search=False,
                              localmedia=False)
        built_docs = task.build_docs()

        # The HTML and the Epub format were built.
        self.mocks.html_build.assert_called_once_with()
        self.mocks.pdf_build.assert_called_once_with()
        # PDF however was disabled and therefore not built.
        self.assertFalse(self.mocks.epub_build.called)
    def test_failing_execution(self):
        '''Build in failing state'''
        self.mocks.configure_mock('process', {
            'communicate.return_value': ('This is not okay', '')})
        type(self.mocks.process).returncode = PropertyMock(return_value=1)

        build_env = LocalEnvironment(version=self.version, project=self.project,
                                     build={})
        with build_env:
            build_env.run('echo', 'test')
            self.fail('This should be unreachable')
        self.assertTrue(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
        self.assertEqual(len(build_env.commands), 1)
        self.assertEqual(build_env.commands[0].output, u'This is not okay')
Exemple #12
0
    def test_build_respects_epub_flag(self):
        '''Test build with epub enabled'''
        project = get(Project,
                      slug='project-1',
                      documentation_type='sphinx',
                      conf_py_file='test_conf.py',
                      enable_pdf_build=False,
                      enable_epub_build=True,
                      versions=[fixture()])
        version = project.versions.all()[0]

        build_env = LocalEnvironment(project=project,
                                     version=version,
                                     build={})
        python_env = Virtualenv(version=version, build_env=build_env)
        yaml_config = get_build_config({})
        config = ConfigWrapper(version=version, yaml_config=yaml_config)
        task = UpdateDocsTask(build_env=build_env,
                              project=project,
                              python_env=python_env,
                              version=version,
                              search=False,
                              localmedia=False,
                              config=config)
        task.build_docs()

        # The HTML and the Epub format were built.
        self.mocks.html_build.assert_called_once_with()
        self.mocks.epub_build.assert_called_once_with()
        # PDF however was disabled and therefore not built.
        self.assertFalse(self.mocks.pdf_build.called)
Exemple #13
0
    def test_build(self):
        '''Test full build'''
        project = get(Project,
                      slug='project-1',
                      documentation_type='sphinx',
                      conf_py_file='test_conf.py',
                      versions=[fixture()])
        version = project.versions.all()[0]
        self.mocks.configure_mock('api_versions', {'return_value': [version]})
        self.mocks.configure_mock(
            'api', {'get.return_value': {
                'downloads': "no_url_here"
            }})
        self.mocks.patches['html_build'].stop()

        build_env = LocalEnvironment(project=project,
                                     version=version,
                                     build={})
        python_env = Virtualenv(version=version, build_env=build_env)
        yaml_config = get_build_config({})
        config = ConfigWrapper(version=version, yaml_config=yaml_config)
        task = UpdateDocsTask(build_env=build_env,
                              project=project,
                              python_env=python_env,
                              version=version,
                              search=False,
                              localmedia=False,
                              config=config)
        task.build_docs()

        # Get command and check first part of command list is a call to sphinx
        self.assertEqual(self.mocks.popen.call_count, 1)
        cmd = self.mocks.popen.call_args_list[0][0]
        self.assertRegexpMatches(cmd[0][0], r'python')
        self.assertRegexpMatches(cmd[0][1], r'sphinx-build')
Exemple #14
0
    def __init__(self, project, version_slug, environment=None, **kwargs):
        self.default_branch = project.default_branch
        self.name = project.name
        self.repo_url = project.clean_repo
        self.working_dir = project.checkout_path(version_slug)

        from readthedocs.doc_builder.environments import LocalEnvironment
        self.environment = environment or LocalEnvironment(project)

        # Update the env variables with the proper VCS env variables
        self.environment.environment.update(self.env)
Exemple #15
0
 def __init__(self, project):
     self.project = project
     self.project_root = os.path.join(
         self.WEB_ROOT,
         project.slug,
     )
     self.subproject_root = os.path.join(
         self.project_root,
         'projects',
     )
     self.environment = LocalEnvironment(project)
     self.sanity_check()
    def test_failing_execution_with_caught_exception(self):
        '''Build in failing state with BuildEnvironmentError exception'''
        build_env = LocalEnvironment(version=self.version, project=self.project,
                                     build={})

        with build_env:
            raise BuildEnvironmentError('Foobar')

        self.assertFalse(self.mocks.process.communicate.called)
        self.assertEqual(len(build_env.commands), 0)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
Exemple #17
0
 def test_builder_comments(self):
     '''Normal build with comments'''
     project = get(Project,
                   documentation_type='sphinx',
                   allow_comments=True,
                   versions=[fixture()])
     version = project.versions.all()[0]
     build_env = LocalEnvironment(version=version, project=project, build={})
     python_env = Virtualenv(version=version, build_env=build_env)
     builder_class = get_builder_class(project.documentation_type)
     builder = builder_class(build_env, python_env)
     self.assertEqual(builder.sphinx_builder, 'readthedocs-comments')
    def test_failing_execution_with_unexpected_exception(self):
        '''Build in failing state with exception from code'''
        build_env = LocalEnvironment(version=self.version,
                                     project=self.project,
                                     build={'id': DUMMY_BUILD_ID})

        with build_env:
            raise ValueError('uncaught')

        self.assertFalse(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
        self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
    def test_failing_execution_with_uncaught_exception(self):
        '''Build in failing state with exception from code'''
        build_env = LocalEnvironment(version=self.version, project=self.project,
                                     build={})

        def _inner():
            with build_env:
                raise Exception()

        self.assertRaises(Exception, _inner)
        self.assertFalse(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
Exemple #20
0
 def test_builder_no_comments(self):
     '''Test builder without comments'''
     project = get(Project,
                   documentation_type='sphinx',
                   allow_comments=False,
                   versions=[fixture()])
     version = project.versions.all()[0]
     build_env = LocalEnvironment(version=version,
                                  project=project,
                                  build={})
     builder_class = get_builder_class(project.documentation_type)
     builder = builder_class(build_env)
     self.assertEqual(builder.sphinx_builder, 'readthedocs')
    def test_failing_execution(self):
        """Build in failing state."""
        self.mocks.configure_mock('process', {
            'communicate.return_value': (b'This is not okay', '')
        })
        type(self.mocks.process).returncode = PropertyMock(return_value=1)

        build_env = LocalEnvironment(
            version=self.version,
            project=self.project,
            build={'id': DUMMY_BUILD_ID},
        )

        with build_env:
            build_env.run('echo', 'test')
            self.fail('This should be unreachable')
        self.assertTrue(self.mocks.process.communicate.called)
        self.assertTrue(build_env.done)
        self.assertTrue(build_env.failed)
        self.assertEqual(len(build_env.commands), 1)
        self.assertEqual(build_env.commands[0].output, u'This is not okay')

        # api() is not called anymore, we use api_v2 instead
        self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
        self.mocks.mocks['api_v2.build']().put.assert_called_with({
            'id': DUMMY_BUILD_ID,
            'version': self.version.pk,
            'success': False,
            'project': self.project.pk,
            'setup_error': u'',
            'length': mock.ANY,
            'error': '',
            'setup': u'',
            'output': u'',
            'state': u'finished',
            'builder': mock.ANY,
            'exit_code': 1,
        })
Exemple #22
0
    def test_build_pdf_latex_not_failure(self):
        '''Test pass during PDF builds and bad latex failure status code'''
        self.mocks.patches['html_build'].stop()
        self.mocks.patches['pdf_build'].stop()

        project = get(Project,
                      slug='project-2',
                      documentation_type='sphinx',
                      conf_py_file='test_conf.py',
                      enable_pdf_build=True,
                      enable_epub_build=False,
                      versions=[fixture()])
        version = project.versions.all()[0]
        assert project.conf_dir() == '/tmp/rtd'

        build_env = LocalEnvironment(project=project,
                                     version=version,
                                     build={})
        python_env = Virtualenv(version=version, build_env=build_env)
        yaml_config = get_build_config({})
        config = ConfigWrapper(version=version, yaml_config=yaml_config)
        task = UpdateDocsTask(build_env=build_env,
                              project=project,
                              python_env=python_env,
                              version=version,
                              search=False,
                              localmedia=False,
                              config=config)

        # Mock out the separate calls to Popen using an iterable side_effect
        returns = [
            (('', ''), 0),  # sphinx-build html
            (('', ''), 0),  # sphinx-build pdf
            (('Output written on foo.pdf', ''), 1),  # latex
            (('', ''), 0),  # makeindex
            (('', ''), 0),  # latex
        ]
        mock_obj = mock.Mock()
        mock_obj.communicate.side_effect = [
            output for (output, status) in returns
        ]
        type(mock_obj).returncode = mock.PropertyMock(
            side_effect=[status for (output, status) in returns])
        self.mocks.popen.return_value = mock_obj

        with build_env:
            task.build_docs()
        self.assertEqual(self.mocks.popen.call_count, 5)
        self.assertTrue(build_env.successful)
    def test_build_pdf_latex_failures(self):
        '''Build failure if latex fails'''
        if six.PY3:
            import pytest
            pytest.xfail(
                "test_build_pdf_latex_failures is known to fail on 3.6")

        self.mocks.patches['html_build'].stop()
        self.mocks.patches['pdf_build'].stop()

        project = get(Project,
                      slug='project-1',
                      documentation_type='sphinx',
                      conf_py_file='test_conf.py',
                      enable_pdf_build=True,
                      enable_epub_build=False,
                      versions=[fixture()])
        version = project.versions.all()[0]
        assert project.conf_dir() == '/tmp/rtd'

        build_env = LocalEnvironment(project=project, version=version, build={})
        python_env = Virtualenv(version=version, build_env=build_env)
        config = ConfigWrapper(version=version, yaml_config=create_load()()[0])
        task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env,
                              version=version, search=False, localmedia=False, config=config)

        # Mock out the separate calls to Popen using an iterable side_effect
        returns = [
            (('', ''), 0),  # sphinx-build html
            (('', ''), 0),  # sphinx-build pdf
            (('', ''), 1),  # latex
            (('', ''), 0),  # makeindex
            (('', ''), 0),  # latex
        ]
        mock_obj = mock.Mock()
        mock_obj.communicate.side_effect = [output for (output, status)
                                            in returns]
        type(mock_obj).returncode = mock.PropertyMock(
            side_effect=[status for (output, status) in returns])
        self.mocks.popen.return_value = mock_obj

        with build_env:
            task.build_docs()
        self.assertEqual(self.mocks.popen.call_count, 7)
        self.assertTrue(build_env.failed)
Exemple #24
0
class UpdateDocsTask(Task):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.

    """

    max_retries = 5
    default_retry_delay = (7 * 60)
    name = 'update_docs'

    def __init__(self,
                 build_env=None,
                 python_env=None,
                 config=None,
                 force=False,
                 search=True,
                 localmedia=True,
                 build=None,
                 project=None,
                 version=None):
        self.build_env = build_env
        self.python_env = python_env
        self.build_force = force
        self.build_search = search
        self.build_localmedia = localmedia
        self.build = {}
        if build is not None:
            self.build = build
        self.version = {}
        if version is not None:
            self.version = version
        self.project = {}
        if project is not None:
            self.project = project
        if config is not None:
            self.config = config

    def _log(self, msg):
        log.info(
            LOG_TEMPLATE.format(project=self.project.slug,
                                version=self.version.slug,
                                msg=msg))

    def run(self,
            pk,
            version_pk=None,
            build_pk=None,
            record=True,
            docker=False,
            search=True,
            force=False,
            localmedia=True,
            **__):
        # pylint: disable=arguments-differ
        self.project = self.get_project(pk)
        self.version = self.get_version(self.project, version_pk)
        self.build = self.get_build(build_pk)
        self.build_search = search
        self.build_localmedia = localmedia
        self.build_force = force
        self.config = None

        setup_successful = self.run_setup(record=record)
        if setup_successful:
            self.run_build(record=record, docker=docker)

    def run_setup(self, record=True):
        """Run setup in the local environment.

        Return True if successful.

        """
        self.setup_env = LocalEnvironment(project=self.project,
                                          version=self.version,
                                          build=self.build,
                                          record=record)

        # Environment used for code checkout & initial configuration reading
        with self.setup_env:
            if self.project.skip:
                raise BuildEnvironmentError(
                    _('Builds for this project are temporarily disabled'))
            try:
                self.setup_vcs()
            except vcs_support_utils.LockTimeout as e:
                self.retry(exc=e, throw=False)
                raise BuildEnvironmentError(
                    'Version locked, retrying in 5 minutes.', status_code=423)

            try:
                self.config = load_yaml_config(version=self.version)
            except ConfigError as e:
                raise BuildEnvironmentError(
                    'Problem parsing YAML configuration. {0}'.format(str(e)))

        if self.setup_env.failure or self.config is None:
            self._log('Failing build because of setup failure: %s' %
                      self.setup_env.failure)

            # Send notification to users only if the build didn't fail because of
            # LockTimeout: this exception occurs when a build is triggered before the previous
            # one has finished (e.g. two webhooks, one after the other)
            if not isinstance(self.setup_env.failure,
                              vcs_support_utils.LockTimeout):
                self.send_notifications()

            self.setup_env.update_build(state=BUILD_STATE_FINISHED)
            return False

        if self.setup_env.successful and not self.project.has_valid_clone:
            self.set_valid_clone()

        return True

    def run_build(self, docker=False, record=True):
        """Build the docs in an environment.

        If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
        setting, then build in a Docker environment. Otherwise build locally.

        """
        env_vars = self.get_env_vars()

        if docker or settings.DOCKER_ENABLE:
            env_cls = DockerEnvironment
        else:
            env_cls = LocalEnvironment
        self.build_env = env_cls(project=self.project,
                                 version=self.version,
                                 build=self.build,
                                 record=record,
                                 environment=env_vars)

        # Environment used for building code, usually with Docker
        with self.build_env:

            if self.project.documentation_type == 'auto':
                self.update_documentation_type()

            python_env_cls = Virtualenv
            if self.config.use_conda:
                self._log('Using conda')
                python_env_cls = Conda
            self.python_env = python_env_cls(version=self.version,
                                             build_env=self.build_env,
                                             config=self.config)

            try:
                self.setup_environment()

                # TODO the build object should have an idea of these states, extend
                # the model to include an idea of these outcomes
                outcomes = self.build_docs()
                build_id = self.build.get('id')
            except SoftTimeLimitExceeded:
                raise BuildEnvironmentError(_('Build exited due to time out'))

            # Finalize build and update web servers
            if build_id:
                self.update_app_instances(
                    html=bool(outcomes['html']),
                    search=bool(outcomes['search']),
                    localmedia=bool(outcomes['localmedia']),
                    pdf=bool(outcomes['pdf']),
                    epub=bool(outcomes['epub']),
                )

        if self.build_env.failed:
            self.send_notifications()
        build_complete.send(sender=Build, build=self.build_env.build)

        self.build_env.update_build(state=BUILD_STATE_FINISHED)

    @staticmethod
    def get_project(project_pk):
        """Get project from API"""
        project_data = api_v2.project(project_pk).get()
        project = make_api_project(project_data)
        return project

    @staticmethod
    def get_version(project, version_pk):
        """Ensure we're using a sane version"""
        if version_pk:
            version_data = api_v2.version(version_pk).get()
        else:
            version_data = (api_v2.version(
                project.slug).get(slug=LATEST)['objects'][0])
        return make_api_version(version_data)

    @staticmethod
    def get_build(build_pk):
        """
        Retrieve build object from API

        :param build_pk: Build primary key
        """
        build = {}
        if build_pk:
            build = api_v2.build(build_pk).get()
        return dict((key, val) for (key, val) in list(build.items())
                    if key not in
                    ['project', 'version', 'resource_uri', 'absolute_uri'])

    def setup_vcs(self):
        """
        Update the checkout of the repo to make sure it's the latest.

        This also syncs versions in the DB.

        :param build_env: Build environment
        """
        self.setup_env.update_build(state=BUILD_STATE_CLONING)

        self._log(msg='Updating docs from VCS')
        try:
            update_imported_docs(self.version.pk)
            commit = self.project.vcs_repo(self.version.slug).commit
            if commit:
                self.build['commit'] = commit
        except ProjectImportError as e:
            log.error(
                LOG_TEMPLATE.format(project=self.project.slug,
                                    version=self.version.slug,
                                    msg=str(e)),
                exc_info=True,
            )
            raise BuildEnvironmentError('Failed to import project: %s' % e,
                                        status_code=404)

    def get_env_vars(self):
        """Get bash environment variables used for all builder commands."""
        env = {
            'READTHEDOCS': True,
            'READTHEDOCS_VERSION': self.version.slug,
            'READTHEDOCS_PROJECT': self.project.slug
        }

        if self.config.use_conda:
            env.update({
                'CONDA_ENVS_PATH':
                os.path.join(self.project.doc_path, 'conda'),
                'CONDA_DEFAULT_ENV':
                self.version.slug,
                'BIN_PATH':
                os.path.join(self.project.doc_path, 'conda', self.version.slug,
                             'bin')
            })
        else:
            env.update({
                'BIN_PATH':
                os.path.join(self.project.doc_path, 'envs', self.version.slug,
                             'bin')
            })

        return env

    def set_valid_clone(self):
        """Mark on the project that it has been cloned properly."""
        project_data = api_v2.project(self.project.pk).get()
        project_data['has_valid_clone'] = True
        api_v2.project(self.project.pk).put(project_data)
        self.project.has_valid_clone = True

    def update_documentation_type(self):
        """
        Force Sphinx for 'auto' documentation type

        This used to determine the type and automatically set the documentation
        type to Sphinx for rST and Mkdocs for markdown. It now just forces
        Sphinx, due to markdown support.
        """
        ret = 'sphinx'
        project_data = api_v2.project(self.project.pk).get()
        project_data['documentation_type'] = ret
        api_v2.project(self.project.pk).put(project_data)
        self.project.documentation_type = ret

    def update_app_instances(self,
                             html=False,
                             localmedia=False,
                             search=False,
                             pdf=False,
                             epub=False):
        """Update application instances with build artifacts

        This triggers updates across application instances for html, pdf, epub,
        downloads, and search. Tasks are broadcast to all web servers from here.
        """
        # Update version if we have successfully built HTML output
        try:
            if html:
                version = api_v2.version(self.version.pk)
                version.patch({
                    'active': True,
                    'built': True,
                })
        except HttpClientError as e:
            log.error(
                'Updating version failed, skipping file sync: version=%s',
                self.version.pk,
                exc_info=True)
        else:
            # Broadcast finalization steps to web application instances
            broadcast(type='app',
                      task=sync_files,
                      args=[
                          self.project.pk,
                          self.version.pk,
                      ],
                      kwargs=dict(
                          hostname=socket.gethostname(),
                          html=html,
                          localmedia=localmedia,
                          search=search,
                          pdf=pdf,
                          epub=epub,
                      ))

            # Delayed tasks
            # TODO these should be chained on to the broadcast calls. The
            # broadcast calls could be lumped together into a promise, and on
            # task result, these next few tasks can be updated, also in a
            # chained fashion
            fileify.delay(self.version.pk, commit=self.build.get('commit'))
            update_search.delay(self.version.pk,
                                commit=self.build.get('commit'))

    def setup_environment(self):
        """
        Build the virtualenv and install the project into it.

        Always build projects with a virtualenv.

        :param build_env: Build environment to pass commands and execution through.
        """
        self.build_env.update_build(state=BUILD_STATE_INSTALLING)

        with self.project.repo_nonblockinglock(version=self.version,
                                               max_lock_age=getattr(
                                                   settings,
                                                   'REPO_LOCK_SECONDS', 30)):

            self.python_env.delete_existing_build_dir()
            self.python_env.setup_base()
            self.python_env.install_core_requirements()
            self.python_env.install_user_requirements()
            self.python_env.install_package()

    def build_docs(self):
        """Wrapper to all build functions

        Executes the necessary builds for this task and returns whether the
        build was successful or not.

        :returns: Build outcomes with keys for html, search, localmedia, pdf,
                  and epub
        :rtype: dict
        """
        self.build_env.update_build(state=BUILD_STATE_BUILDING)
        before_build.send(sender=self.version)

        outcomes = defaultdict(lambda: False)
        with self.project.repo_nonblockinglock(version=self.version,
                                               max_lock_age=getattr(
                                                   settings,
                                                   'REPO_LOCK_SECONDS', 30)):
            outcomes['html'] = self.build_docs_html()
            outcomes['search'] = self.build_docs_search()
            outcomes['localmedia'] = self.build_docs_localmedia()
            outcomes['pdf'] = self.build_docs_pdf()
            outcomes['epub'] = self.build_docs_epub()

        after_build.send(sender=self.version)
        return outcomes

    def build_docs_html(self):
        """Build HTML docs"""
        html_builder = get_builder_class(self.project.documentation_type)(
            build_env=self.build_env,
            python_env=self.python_env,
        )
        if self.build_force:
            html_builder.force()
        html_builder.append_conf()
        success = html_builder.build()
        if success:
            html_builder.move()

        # Gracefully attempt to move files via task on web workers.
        try:
            broadcast(type='app',
                      task=move_files,
                      args=[self.version.pk,
                            socket.gethostname()],
                      kwargs=dict(html=True))
        except socket.error:
            # TODO do something here
            pass

        return success

    def build_docs_search(self):
        """Build search data with separate build"""
        if self.build_search:
            if self.project.is_type_mkdocs:
                return self.build_docs_class('mkdocs_json')
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_search')
        return False

    def build_docs_localmedia(self):
        """Get local media files with separate build"""
        if 'htmlzip' not in self.config.formats:
            return False

        if self.build_localmedia:
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_singlehtmllocalmedia')
        return False

    def build_docs_pdf(self):
        """Build PDF docs"""
        if ('pdf' not in self.config.formats or self.project.slug in HTML_ONLY
                or not self.project.is_type_sphinx):
            return False
        return self.build_docs_class('sphinx_pdf')

    def build_docs_epub(self):
        """Build ePub docs"""
        if ('epub' not in self.config.formats or self.project.slug in HTML_ONLY
                or not self.project.is_type_sphinx):
            return False
        return self.build_docs_class('sphinx_epub')

    def build_docs_class(self, builder_class):
        """Build docs with additional doc backends

        These steps are not necessarily required for the build to halt, so we
        only raise a warning exception here. A hard error will halt the build
        process.
        """
        builder = get_builder_class(builder_class)(self.build_env,
                                                   python_env=self.python_env)
        success = builder.build()
        builder.move()
        return success

    def send_notifications(self):
        """Send notifications on build failure"""
        send_notifications.delay(self.version.pk, build_pk=self.build['id'])
Exemple #25
0
class UpdateDocsTask(Task):

    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.

    """

    max_retries = 5
    default_retry_delay = (7 * 60)
    name = 'update_docs'

    def __init__(self, build_env=None, python_env=None, config=None,
                 force=False, search=True, localmedia=True,
                 build=None, project=None, version=None):
        self.build_env = build_env
        self.python_env = python_env
        self.build_force = force
        self.build_search = search
        self.build_localmedia = localmedia
        self.build = {}
        if build is not None:
            self.build = build
        self.version = {}
        if version is not None:
            self.version = version
        self.project = {}
        if project is not None:
            self.project = project
        if config is not None:
            self.config = config

    def _log(self, msg):
        log.info(LOG_TEMPLATE
                 .format(project=self.project.slug,
                         version=self.version.slug,
                         msg=msg))

    def run(self, pk, version_pk=None, build_pk=None, record=True,
            docker=False, search=True, force=False, localmedia=True, **__):
        # pylint: disable=arguments-differ
        self.project = self.get_project(pk)
        self.version = self.get_version(self.project, version_pk)
        self.build = self.get_build(build_pk)
        self.build_search = search
        self.build_localmedia = localmedia
        self.build_force = force
        self.config = None

        setup_successful = self.run_setup(record=record)
        if setup_successful:
            self.run_build(record=record, docker=docker)

    def run_setup(self, record=True):
        """Run setup in the local environment.

        Return True if successful.

        """
        self.setup_env = LocalEnvironment(
            project=self.project,
            version=self.version,
            build=self.build,
            record=record
        )

        # Environment used for code checkout & initial configuration reading
        with self.setup_env:
            if self.project.skip:
                raise BuildEnvironmentError(
                    _('Builds for this project are temporarily disabled'))
            try:
                self.setup_vcs()
            except vcs_support_utils.LockTimeout as e:
                self.retry(exc=e, throw=False)
                raise BuildEnvironmentError(
                    'Version locked, retrying in 5 minutes.',
                    status_code=423
                )

            try:
                self.config = load_yaml_config(version=self.version)
            except ConfigError as e:
                raise BuildEnvironmentError(
                    'Problem parsing YAML configuration. {0}'.format(str(e))
                )

        if self.setup_env.failure or self.config is None:
            self._log('Failing build because of setup failure: %s' % self.setup_env.failure)

            # Send notification to users only if the build didn't fail because of
            # LockTimeout: this exception occurs when a build is triggered before the previous
            # one has finished (e.g. two webhooks, one after the other)
            if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout):
                self.send_notifications()

            self.setup_env.update_build(state=BUILD_STATE_FINISHED)
            return False

        if self.setup_env.successful and not self.project.has_valid_clone:
            self.set_valid_clone()

        return True

    def run_build(self, docker=False, record=True):
        """Build the docs in an environment.

        If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
        setting, then build in a Docker environment. Otherwise build locally.

        """
        env_vars = self.get_env_vars()

        if docker or settings.DOCKER_ENABLE:
            env_cls = DockerEnvironment
        else:
            env_cls = LocalEnvironment
        self.build_env = env_cls(project=self.project, version=self.version,
                                 build=self.build, record=record, environment=env_vars)

        # Environment used for building code, usually with Docker
        with self.build_env:

            if self.project.documentation_type == 'auto':
                self.update_documentation_type()

            python_env_cls = Virtualenv
            if self.config.use_conda:
                self._log('Using conda')
                python_env_cls = Conda
            self.python_env = python_env_cls(version=self.version,
                                             build_env=self.build_env,
                                             config=self.config)

            try:
                self.setup_environment()

                # TODO the build object should have an idea of these states, extend
                # the model to include an idea of these outcomes
                outcomes = self.build_docs()
                build_id = self.build.get('id')
            except SoftTimeLimitExceeded:
                raise BuildEnvironmentError(_('Build exited due to time out'))

            # Web Server Tasks
            if build_id:
                finish_build.delay(
                    version_pk=self.version.pk,
                    build_pk=build_id,
                    hostname=socket.gethostname(),
                    html=outcomes['html'],
                    search=outcomes['search'],
                    localmedia=outcomes['localmedia'],
                    pdf=outcomes['pdf'],
                    epub=outcomes['epub'],
                )

        if self.build_env.failed:
            self.send_notifications()
        build_complete.send(sender=Build, build=self.build_env.build)

        self.build_env.update_build(state=BUILD_STATE_FINISHED)

    @staticmethod
    def get_project(project_pk):
        """Get project from API"""
        project_data = api_v1.project(project_pk).get()
        project = make_api_project(project_data)
        return project

    @staticmethod
    def get_version(project, version_pk):
        """Ensure we're using a sane version"""
        if version_pk:
            version_data = api_v1.version(version_pk).get()
        else:
            version_data = (api_v1
                            .version(project.slug)
                            .get(slug=LATEST)['objects'][0])
        return make_api_version(version_data)

    @staticmethod
    def get_build(build_pk):
        """
        Retrieve build object from API

        :param build_pk: Build primary key
        """
        build = {}
        if build_pk:
            build = api_v2.build(build_pk).get()
        return dict((key, val) for (key, val) in list(build.items())
                    if key not in ['project', 'version', 'resource_uri',
                                   'absolute_uri'])

    def setup_vcs(self):
        """
        Update the checkout of the repo to make sure it's the latest.

        This also syncs versions in the DB.

        :param build_env: Build environment
        """
        self.setup_env.update_build(state=BUILD_STATE_CLONING)

        self._log(msg='Updating docs from VCS')
        try:
            update_imported_docs(self.version.pk)
            commit = self.project.vcs_repo(self.version.slug).commit
            if commit:
                self.build['commit'] = commit
        except ProjectImportError as e:
            log.error(
                LOG_TEMPLATE.format(project=self.project.slug,
                                    version=self.version.slug,
                                    msg=str(e)),
                exc_info=True,
            )
            raise BuildEnvironmentError('Failed to import project: %s' % e,
                                        status_code=404)

    def get_env_vars(self):
        """Get bash environment variables used for all builder commands."""
        env = {
            'READTHEDOCS': True,
            'READTHEDOCS_VERSION': self.version.slug,
            'READTHEDOCS_PROJECT': self.project.slug
        }

        if self.config.use_conda:
            env.update({
                'CONDA_ENVS_PATH': os.path.join(self.project.doc_path, 'conda'),
                'CONDA_DEFAULT_ENV': self.version.slug,
                'BIN_PATH': os.path.join(self.project.doc_path, 'conda', self.version.slug, 'bin')
            })
        else:
            env.update({
                'BIN_PATH': os.path.join(self.project.doc_path, 'envs', self.version.slug, 'bin')
            })

        return env

    def set_valid_clone(self):
        """Mark on the project that it has been cloned properly."""
        project_data = api_v2.project(self.project.pk).get()
        project_data['has_valid_clone'] = True
        api_v2.project(self.project.pk).put(project_data)
        self.project.has_valid_clone = True

    def update_documentation_type(self):
        """
        Force Sphinx for 'auto' documentation type

        This used to determine the type and automatically set the documentation
        type to Sphinx for rST and Mkdocs for markdown. It now just forces
        Sphinx, due to markdown support.
        """
        ret = 'sphinx'
        project_data = api_v2.project(self.project.pk).get()
        project_data['documentation_type'] = ret
        api_v2.project(self.project.pk).put(project_data)
        self.project.documentation_type = ret

    def setup_environment(self):
        """
        Build the virtualenv and install the project into it.

        Always build projects with a virtualenv.

        :param build_env: Build environment to pass commands and execution through.
        """
        self.build_env.update_build(state=BUILD_STATE_INSTALLING)

        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):

            self.python_env.delete_existing_build_dir()
            self.python_env.setup_base()
            self.python_env.install_core_requirements()
            self.python_env.install_user_requirements()
            self.python_env.install_package()

    def build_docs(self):
        """Wrapper to all build functions

        Executes the necessary builds for this task and returns whether the
        build was successful or not.

        :returns: Build outcomes with keys for html, search, localmedia, pdf,
                  and epub
        :rtype: dict
        """
        self.build_env.update_build(state=BUILD_STATE_BUILDING)
        before_build.send(sender=self.version)

        outcomes = defaultdict(lambda: False)
        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):
            outcomes['html'] = self.build_docs_html()
            outcomes['search'] = self.build_docs_search()
            outcomes['localmedia'] = self.build_docs_localmedia()
            outcomes['pdf'] = self.build_docs_pdf()
            outcomes['epub'] = self.build_docs_epub()

        after_build.send(sender=self.version)
        return outcomes

    def build_docs_html(self):
        """Build HTML docs"""
        html_builder = get_builder_class(self.project.documentation_type)(
            build_env=self.build_env,
            python_env=self.python_env,
        )
        if self.build_force:
            html_builder.force()
        html_builder.append_conf()
        success = html_builder.build()
        if success:
            html_builder.move()

        # Gracefully attempt to move files via task on web workers.
        try:
            broadcast(type='app', task=move_files,
                      args=[self.version.pk, socket.gethostname()],
                      kwargs=dict(html=True)
                      )
        except socket.error:
            # TODO do something here
            pass

        return success

    def build_docs_search(self):
        """Build search data with separate build"""
        if self.build_search:
            if self.project.is_type_mkdocs:
                return self.build_docs_class('mkdocs_json')
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_search')
        return False

    def build_docs_localmedia(self):
        """Get local media files with separate build"""
        if 'htmlzip' not in self.config.formats:
            return False

        if self.build_localmedia:
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_singlehtmllocalmedia')
        return False

    def build_docs_pdf(self):
        """Build PDF docs"""
        if ('pdf' not in self.config.formats or
            self.project.slug in HTML_ONLY or
                not self.project.is_type_sphinx):
            return False
        return self.build_docs_class('sphinx_pdf')

    def build_docs_epub(self):
        """Build ePub docs"""
        if ('epub' not in self.config.formats or
            self.project.slug in HTML_ONLY or
                not self.project.is_type_sphinx):
            return False
        return self.build_docs_class('sphinx_epub')

    def build_docs_class(self, builder_class):
        """Build docs with additional doc backends

        These steps are not necessarily required for the build to halt, so we
        only raise a warning exception here. A hard error will halt the build
        process.
        """
        builder = get_builder_class(builder_class)(self.build_env, python_env=self.python_env)
        success = builder.build()
        builder.move()
        return success

    def send_notifications(self):
        """Send notifications on build failure"""
        send_notifications.delay(self.version.pk, build_pk=self.build['id'])
Exemple #26
0
class Symlink:
    """Base class for symlinking of projects."""
    def __init__(self, project):
        self.project = project
        self.project_root = os.path.join(
            self.WEB_ROOT,
            project.slug,
        )
        self.subproject_root = os.path.join(
            self.project_root,
            'projects',
        )
        self.environment = LocalEnvironment(project)
        self.sanity_check()

    def sanity_check(self):
        """
        Make sure the project_root is the proper structure before continuing.

        This will leave it in the proper state for the single_project setting.
        """
        if os.path.islink(
                self.project_root) and not self.project.single_version:
            log.info(
                constants.LOG_TEMPLATE, {
                    'project': self.project.slug,
                    'version': '',
                    'msg': 'Removing single version symlink',
                })
            safe_unlink(self.project_root)
            safe_makedirs(self.project_root)
        elif (self.project.single_version
              and not os.path.islink(self.project_root)
              and os.path.exists(self.project_root)):
            shutil.rmtree(self.project_root)
        elif not os.path.lexists(self.project_root):
            safe_makedirs(self.project_root)

        # CNAME root directories
        if not os.path.lexists(self.CNAME_ROOT):
            safe_makedirs(self.CNAME_ROOT)
        if not os.path.lexists(self.PROJECT_CNAME_ROOT):
            safe_makedirs(self.PROJECT_CNAME_ROOT)

    def run(self):
        """
        Create proper symlinks in the right order.

        Since we have a small nest of directories and symlinks, the ordering of
        these calls matter, so we provide this helper to make life easier.
        """
        # Outside of the web root
        self.symlink_cnames()

        # Build structure inside symlink zone
        if self.project.single_version:
            self.symlink_single_version()
            self.symlink_subprojects()
        else:
            self.symlink_translations()
            self.symlink_subprojects()
            self.symlink_versions()

    def symlink_cnames(self, domain=None):
        """
        Symlink project CNAME domains.

        Link from HOME/$CNAME_ROOT/<cname> ->
                  HOME/$WEB_ROOT/<project>

        Also give cname -> project link

        Link from HOME/public_cname_project/<cname> ->
                  HOME/<project>/
        """
        if domain:
            domains = [domain]
        else:
            domains = Domain.objects.filter(project=self.project).values_list(
                'domain', flat=True)
        for dom in domains:
            log_msg = 'Symlinking CNAME: {} -> {}'.format(
                dom,
                self.project.slug,
            )
            log.debug(constants.LOG_TEMPLATE, {
                'project': self.project.slug,
                'version': '',
                'msg': log_msg,
            })
            # CNAME to doc root
            symlink = os.path.join(self.CNAME_ROOT, dom)
            self.environment.run('ln', '-nsf', self.project_root, symlink)

            # Project symlink
            project_cname_symlink = os.path.join(
                self.PROJECT_CNAME_ROOT,
                dom,
            )
            self.environment.run(
                'ln',
                '-nsf',
                self.project.doc_path,
                project_cname_symlink,
            )

    def remove_symlink_cname(self, domain):
        """
        Remove CNAME symlink.

        :param domain: domain for which symlink is to be removed
        :type domain: str
        """
        log_msg = 'Removing symlink for CNAME {}'.format(domain)
        log.debug(constants.LOG_TEMPLATE, {
            'project': self.project.slug,
            'version': '',
            'msg': log_msg,
        })
        symlink = os.path.join(self.CNAME_ROOT, domain)
        safe_unlink(symlink)

    def symlink_subprojects(self):
        """
        Symlink project subprojects.

        Link from $WEB_ROOT/projects/<project> ->           $WEB_ROOT/<project>
        """
        subprojects = set()
        rels = self.get_subprojects()
        if rels.count():
            # Don't create the `projects/` directory unless subprojects exist.
            if not os.path.exists(self.subproject_root):
                safe_makedirs(self.subproject_root)
        for rel in rels:
            # A mapping of slugs for the subproject URL to the actual built
            # documentation
            from_to = OrderedDict({rel.child.slug: rel.child.slug})
            subprojects.add(rel.child.slug)
            if rel.alias:
                from_to[rel.alias] = rel.child.slug
                subprojects.add(rel.alias)
            for from_slug, to_slug in list(from_to.items()):
                log_msg = 'Symlinking subproject: {} -> {}'.format(
                    from_slug,
                    to_slug,
                )
                log.debug(constants.LOG_TEMPLATE, {
                    'project': self.project.slug,
                    'version': '',
                    'msg': log_msg,
                })
                symlink = os.path.join(self.subproject_root, from_slug)
                docs_dir = os.path.join(
                    self.WEB_ROOT,
                    to_slug,
                )
                symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
                if not os.path.lexists(symlink_dir):
                    safe_makedirs(symlink_dir)
                # TODO this should use os.symlink, not a call to shell. For now,
                # this passes command as a list to be explicit about escaping
                # characters like spaces.
                result = self.environment.run('ln', '-nsf', docs_dir, symlink)
                if result.exit_code > 0:
                    log.error(
                        'Could not symlink path: status=%d error=%s',
                        result.exit_code,
                        result.error,
                    )

        # Remove old symlinks
        if os.path.exists(self.subproject_root):
            for subproj in os.listdir(self.subproject_root):
                if subproj not in subprojects:
                    safe_unlink(os.path.join(self.subproject_root, subproj))

    def symlink_translations(self):
        """
        Symlink project translations.

        Link from $WEB_ROOT/<project>/<language>/ ->
        $WEB_ROOT/<translation>/<language>/
        """
        translations = {}

        for trans in self.get_translations():
            translations[trans.language] = trans.slug

        # Make sure the language directory is a directory
        language_dir = os.path.join(self.project_root, self.project.language)
        if os.path.islink(language_dir):
            safe_unlink(language_dir)
        if not os.path.lexists(language_dir):
            safe_makedirs(language_dir)

        for (language, slug) in list(translations.items()):

            log_msg = 'Symlinking translation: {}->{}'.format(language, slug)
            log.debug(constants.LOG_TEMPLATE, {
                'project': self.project.slug,
                'version': '',
                'msg': log_msg,
            })
            symlink = os.path.join(self.project_root, language)
            docs_dir = os.path.join(self.WEB_ROOT, slug, language)
            self.environment.run('ln', '-nsf', docs_dir, symlink)

        # Remove old symlinks
        for lang in os.listdir(self.project_root):
            if (lang not in translations
                    and lang not in ['projects', self.project.language]):
                to_delete = os.path.join(self.project_root, lang)
                if os.path.islink(to_delete):
                    safe_unlink(to_delete)
                else:
                    shutil.rmtree(to_delete)

    def symlink_single_version(self):
        """
        Symlink project single version.

        Link from:

        $WEB_ROOT/<project> -> HOME/user_builds/<project>/rtd-builds/latest/
        """
        version = self.get_default_version()

        # Clean up symlinks
        symlink = self.project_root
        if os.path.islink(symlink):
            safe_unlink(symlink)
        elif os.path.exists(symlink):
            shutil.rmtree(symlink)

        # Create symlink
        if version is not None:
            docs_dir = os.path.join(
                settings.DOCROOT,
                self.project.slug,
                'rtd-builds',
                version.slug,
            )
            self.environment.run('ln', '-nsf', docs_dir, symlink)

    def symlink_versions(self):
        """
        Symlink project's versions.

        Link from $WEB_ROOT/<project>/<language>/<version>/ ->
        HOME/user_builds/<project>/rtd-builds/<version>
        """
        versions = set()
        version_dir = os.path.join(
            self.WEB_ROOT,
            self.project.slug,
            self.project.language,
        )
        # Include active public versions,
        # as well as public versions that are built but not active, for archived versions
        version_queryset = self.get_version_queryset()
        if version_queryset.count():
            if not os.path.exists(version_dir):
                safe_makedirs(version_dir)
        for version in version_queryset:
            log_msg = 'Symlinking Version: {}'.format(version)
            log.debug(constants.LOG_TEMPLATE, {
                'project': self.project.slug,
                'version': '',
                'msg': log_msg,
            })
            symlink = os.path.join(version_dir, version.slug)
            docs_dir = os.path.join(
                settings.DOCROOT,
                self.project.slug,
                'rtd-builds',
                version.slug,
            )
            self.environment.run('ln', '-nsf', docs_dir, symlink)
            versions.add(version.slug)

        # Remove old symlinks
        if os.path.exists(version_dir):
            for old_ver in os.listdir(version_dir):
                if old_ver not in versions:
                    safe_unlink(os.path.join(version_dir, old_ver))

    def get_default_version(self):
        """Look up project default version, return None if not found."""
        default_version = self.project.get_default_version()
        try:
            return self.get_version_queryset().get(slug=default_version)
        except Version.DoesNotExist:
            return None
Exemple #27
0
class Symlink:

    """Base class for symlinking of projects."""

    def __init__(self, project):
        self.project = project
        self.project_root = os.path.join(
            self.WEB_ROOT,
            project.slug,
        )
        self.subproject_root = os.path.join(
            self.project_root,
            'projects',
        )
        self.environment = LocalEnvironment(project)
        self.sanity_check()

    def sanity_check(self):
        """
        Make sure the project_root is the proper structure before continuing.

        This will leave it in the proper state for the single_project setting.
        """
        if os.path.islink(self.project_root) and not self.project.single_version:
            log.info(
                constants.LOG_TEMPLATE,
                {
                    'project': self.project.slug,
                    'version': '',
                    'msg': 'Removing single version symlink',
                }
            )
            safe_unlink(self.project_root)
            safe_makedirs(self.project_root)
        elif (self.project.single_version and
              not os.path.islink(self.project_root) and
              os.path.exists(self.project_root)):
            shutil.rmtree(self.project_root)
        elif not os.path.lexists(self.project_root):
            safe_makedirs(self.project_root)

        # CNAME root directories
        if not os.path.lexists(self.CNAME_ROOT):
            safe_makedirs(self.CNAME_ROOT)
        if not os.path.lexists(self.PROJECT_CNAME_ROOT):
            safe_makedirs(self.PROJECT_CNAME_ROOT)

    def run(self):
        """
        Create proper symlinks in the right order.

        Since we have a small nest of directories and symlinks, the ordering of
        these calls matter, so we provide this helper to make life easier.
        """
        # Outside of the web root
        self.symlink_cnames()

        # Build structure inside symlink zone
        if self.project.single_version:
            self.symlink_single_version()
            self.symlink_subprojects()
        else:
            self.symlink_translations()
            self.symlink_subprojects()
            self.symlink_versions()

    def symlink_cnames(self, domain=None):
        """
        Symlink project CNAME domains.

        Link from HOME/$CNAME_ROOT/<cname> ->
                  HOME/$WEB_ROOT/<project>

        Also give cname -> project link

        Link from HOME/public_cname_project/<cname> ->
                  HOME/<project>/
        """
        if domain:
            domains = [domain]
        else:
            domains = Domain.objects.filter(project=self.project).values_list('domain', flat=True)
        for dom in domains:
            log_msg = 'Symlinking CNAME: {} -> {}'.format(
                dom,
                self.project.slug,
            )
            log.debug(
                constants.LOG_TEMPLATE,
                {
                    'project': self.project.slug,
                    'version': '',
                    'msg': log_msg,
                }
            )
            # CNAME to doc root
            symlink = os.path.join(self.CNAME_ROOT, dom)
            self.environment.run('ln', '-nsf', self.project_root, symlink)

            # Project symlink
            project_cname_symlink = os.path.join(
                self.PROJECT_CNAME_ROOT,
                dom,
            )
            self.environment.run(
                'ln',
                '-nsf',
                self.project.doc_path,
                project_cname_symlink,
            )

    def remove_symlink_cname(self, domain):
        """
        Remove CNAME symlink.

        :param domain: domain for which symlink is to be removed
        :type domain: str
        """
        log_msg = 'Removing symlink for CNAME {}'.format(domain)
        log.debug(
            constants.LOG_TEMPLATE,
            {
                'project': self.project.slug,
                'version': '',
                'msg': log_msg,
            }
        )
        symlink = os.path.join(self.CNAME_ROOT, domain)
        safe_unlink(symlink)

    def symlink_subprojects(self):
        """
        Symlink project subprojects.

        Link from $WEB_ROOT/projects/<project> ->           $WEB_ROOT/<project>
        """
        subprojects = set()
        rels = self.get_subprojects()
        if rels.count():
            # Don't create the `projects/` directory unless subprojects exist.
            if not os.path.exists(self.subproject_root):
                safe_makedirs(self.subproject_root)
        for rel in rels:
            # A mapping of slugs for the subproject URL to the actual built
            # documentation
            from_to = OrderedDict({rel.child.slug: rel.child.slug})
            subprojects.add(rel.child.slug)
            if rel.alias:
                from_to[rel.alias] = rel.child.slug
                subprojects.add(rel.alias)
            for from_slug, to_slug in list(from_to.items()):
                log_msg = 'Symlinking subproject: {} -> {}'.format(
                    from_slug,
                    to_slug,
                )
                log.debug(
                    constants.LOG_TEMPLATE,
                    {
                        'project': self.project.slug,
                        'version': '',
                        'msg': log_msg,
                    }
                )
                symlink = os.path.join(self.subproject_root, from_slug)
                docs_dir = os.path.join(
                    self.WEB_ROOT,
                    to_slug,
                )
                symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
                if not os.path.lexists(symlink_dir):
                    safe_makedirs(symlink_dir)
                # TODO this should use os.symlink, not a call to shell. For now,
                # this passes command as a list to be explicit about escaping
                # characters like spaces.
                result = self.environment.run('ln', '-nsf', docs_dir, symlink)
                if result.exit_code > 0:
                    log.error(
                        'Could not symlink path: status=%d error=%s',
                        result.exit_code,
                        result.error,
                    )

        # Remove old symlinks
        if os.path.exists(self.subproject_root):
            for subproj in os.listdir(self.subproject_root):
                if subproj not in subprojects:
                    safe_unlink(os.path.join(self.subproject_root, subproj))

    def symlink_translations(self):
        """
        Symlink project translations.

        Link from $WEB_ROOT/<project>/<language>/ ->
        $WEB_ROOT/<translation>/<language>/
        """
        translations = {}

        for trans in self.get_translations():
            translations[trans.language] = trans.slug

        # Make sure the language directory is a directory
        language_dir = os.path.join(self.project_root, self.project.language)
        if os.path.islink(language_dir):
            safe_unlink(language_dir)
        if not os.path.lexists(language_dir):
            safe_makedirs(language_dir)

        for (language, slug) in list(translations.items()):

            log_msg = 'Symlinking translation: {}->{}'.format(language, slug)
            log.debug(
                constants.LOG_TEMPLATE,
                {
                    'project': self.project.slug,
                    'version': '',
                    'msg': log_msg,
                }
            )
            symlink = os.path.join(self.project_root, language)
            docs_dir = os.path.join(self.WEB_ROOT, slug, language)
            self.environment.run('ln', '-nsf', docs_dir, symlink)

        # Remove old symlinks
        for lang in os.listdir(self.project_root):
            if (lang not in translations and
                    lang not in ['projects', self.project.language]):
                to_delete = os.path.join(self.project_root, lang)
                if os.path.islink(to_delete):
                    safe_unlink(to_delete)
                else:
                    shutil.rmtree(to_delete)

    def symlink_single_version(self):
        """
        Symlink project single version.

        Link from:

        $WEB_ROOT/<project> -> HOME/user_builds/<project>/rtd-builds/latest/
        """
        version = self.get_default_version()

        # Clean up symlinks
        symlink = self.project_root
        if os.path.islink(symlink):
            safe_unlink(symlink)
        elif os.path.exists(symlink):
            shutil.rmtree(symlink)

        # Create symlink
        if version is not None:
            docs_dir = os.path.join(
                settings.DOCROOT,
                self.project.slug,
                'rtd-builds',
                version.slug,
            )
            self.environment.run('ln', '-nsf', docs_dir, symlink)

    def symlink_versions(self):
        """
        Symlink project's versions.

        Link from $WEB_ROOT/<project>/<language>/<version>/ ->
        HOME/user_builds/<project>/rtd-builds/<version>
        """
        versions = set()
        version_dir = os.path.join(
            self.WEB_ROOT,
            self.project.slug,
            self.project.language,
        )
        # Include active public versions,
        # as well as public versions that are built but not active, for archived versions
        version_queryset = self.get_version_queryset()
        if version_queryset.count():
            if not os.path.exists(version_dir):
                safe_makedirs(version_dir)
        for version in version_queryset:
            log_msg = 'Symlinking Version: {}'.format(version)
            log.debug(
                constants.LOG_TEMPLATE,
                {
                    'project': self.project.slug,
                    'version': '',
                    'msg': log_msg,
                }
            )
            symlink = os.path.join(version_dir, version.slug)
            docs_dir = os.path.join(
                settings.DOCROOT,
                self.project.slug,
                'rtd-builds',
                version.slug,
            )
            self.environment.run('ln', '-nsf', docs_dir, symlink)
            versions.add(version.slug)

        # Remove old symlinks
        if os.path.exists(version_dir):
            for old_ver in os.listdir(version_dir):
                if old_ver not in versions:
                    safe_unlink(os.path.join(version_dir, old_ver))

    def get_default_version(self):
        """Look up project default version, return None if not found."""
        default_version = self.project.get_default_version()
        try:
            return self.get_version_queryset().get(slug=default_version)
        except Version.DoesNotExist:
            return None
Exemple #28
0
class UpdateDocsTask(Task):

    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.
    """

    max_retries = 5
    default_retry_delay = (7 * 60)
    name = __name__ + '.update_docs'

    def __init__(self, build_env=None, python_env=None, config=None,
                 force=False, search=True, localmedia=True,
                 build=None, project=None, version=None):
        self.build_env = build_env
        self.python_env = python_env
        self.build_force = force
        self.build_search = search
        self.build_localmedia = localmedia
        self.build = {}
        if build is not None:
            self.build = build
        self.version = {}
        if version is not None:
            self.version = version
        self.project = {}
        if project is not None:
            self.project = project
        if config is not None:
            self.config = config

    def _log(self, msg):
        log.info(LOG_TEMPLATE
                 .format(project=self.project.slug,
                         version=self.version.slug,
                         msg=msg))

    # pylint: disable=arguments-differ
    def run(self, pk, version_pk=None, build_pk=None, record=True,
            docker=False, search=True, force=False, localmedia=True, **__):
        """
        Run a documentation build.

        This is fully wrapped in exception handling to account for a number of
        failure cases. We first run a few commands in a local build environment,
        but do not report on environment success. This avoids a flicker on the
        build output page where the build is marked as finished in between the
        local environment steps and the docker build steps.

        If a failure is raised, or the build is not successful, return ``False``,
        otherwise, ``True``.

        Unhandled exceptions raise a generic user facing error, which directs
        the user to bug us. It is therefore a benefit to have as few unhandled
        errors as possible.

        :param pk int: Project id
        :param version_pk int: Project Version id
        :param build_pk int: Build id
        :param record bool: record a build object in the database
        :param docker bool: use docker to build the project
        :param search bool: update search
        :param force bool: force Sphinx build
        :param localmedia: update localmedia
        :returns: if build was successful or not
        :rtype: bool
        """
        try:
            self.project = self.get_project(pk)
            self.version = self.get_version(self.project, version_pk)
            self.build = self.get_build(build_pk)
            self.build_search = search
            self.build_localmedia = localmedia
            self.build_force = force
            self.config = None

            setup_successful = self.run_setup(record=record)
            if not setup_successful:
                return False
        # Catch unhandled errors in the setup step
        except Exception as e:  # noqa
            log.exception(
                'An unhandled exception was raised during build setup',
                extra={'tags': {'build': build_pk}}
            )
            self.setup_env.failure = BuildEnvironmentError(
                BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                    build_id=build_pk,
                )
            )
            self.setup_env.update_build(BUILD_STATE_FINISHED)
            return False
        else:
            # No exceptions in the setup step, catch unhandled errors in the
            # build steps
            try:
                self.run_build(record=record, docker=docker)
            except Exception as e:  # noqa
                log.exception(
                    'An unhandled exception was raised during project build',
                    extra={'tags': {'build': build_pk}}
                )
                self.build_env.failure = BuildEnvironmentError(
                    BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
                        build_id=build_pk,
                    )
                )
                self.build_env.update_build(BUILD_STATE_FINISHED)
                return False

        return True

    def run_setup(self, record=True):
        """
        Run setup in the local environment.

        Return True if successful.
        """
        self.setup_env = LocalEnvironment(
            project=self.project,
            version=self.version,
            build=self.build,
            record=record,
            update_on_success=False,
        )

        # Environment used for code checkout & initial configuration reading
        with self.setup_env:
            if self.project.skip:
                raise BuildEnvironmentError(
                    _('Builds for this project are temporarily disabled'))
            try:
                self.setup_vcs()
            except vcs_support_utils.LockTimeout as e:
                self.retry(exc=e, throw=False)
                raise BuildEnvironmentError(
                    'Version locked, retrying in 5 minutes.',
                    status_code=423
                )

            try:
                self.config = load_yaml_config(version=self.version)
            except ConfigError as e:
                raise BuildEnvironmentError(
                    'Problem parsing YAML configuration. {0}'.format(str(e))
                )

        if self.setup_env.failure or self.config is None:
            self._log('Failing build because of setup failure: %s' % self.setup_env.failure)

            # Send notification to users only if the build didn't fail because of
            # LockTimeout: this exception occurs when a build is triggered before the previous
            # one has finished (e.g. two webhooks, one after the other)
            if not isinstance(self.setup_env.failure, vcs_support_utils.LockTimeout):
                self.send_notifications()

            return False

        if self.setup_env.successful and not self.project.has_valid_clone:
            self.set_valid_clone()

        return True

    def run_build(self, docker=False, record=True):
        """
        Build the docs in an environment.

        If `docker` is True, or Docker is enabled by the settings.DOCKER_ENABLE
        setting, then build in a Docker environment. Otherwise build locally.
        """
        env_vars = self.get_env_vars()

        if docker or settings.DOCKER_ENABLE:
            env_cls = DockerEnvironment
        else:
            env_cls = LocalEnvironment
        self.build_env = env_cls(project=self.project, version=self.version,
                                 build=self.build, record=record, environment=env_vars)

        # Environment used for building code, usually with Docker
        with self.build_env:

            if self.project.documentation_type == 'auto':
                self.update_documentation_type()

            python_env_cls = Virtualenv
            if self.config.use_conda:
                self._log('Using conda')
                python_env_cls = Conda
            self.python_env = python_env_cls(version=self.version,
                                             build_env=self.build_env,
                                             config=self.config)

            try:
                self.setup_environment()

                # TODO the build object should have an idea of these states, extend
                # the model to include an idea of these outcomes
                outcomes = self.build_docs()
                build_id = self.build.get('id')
            except SoftTimeLimitExceeded:
                raise BuildEnvironmentError(_('Build exited due to time out'))

            # Finalize build and update web servers
            if build_id:
                self.update_app_instances(
                    html=bool(outcomes['html']),
                    search=bool(outcomes['search']),
                    localmedia=bool(outcomes['localmedia']),
                    pdf=bool(outcomes['pdf']),
                    epub=bool(outcomes['epub']),
                )
            else:
                log.warning('No build ID, not syncing files')

        if self.build_env.failed:
            self.send_notifications()

        build_complete.send(sender=Build, build=self.build_env.build)

    @staticmethod
    def get_project(project_pk):
        """Get project from API."""
        project_data = api_v2.project(project_pk).get()
        return APIProject(**project_data)

    @staticmethod
    def get_version(project, version_pk):
        """Ensure we're using a sane version."""
        if version_pk:
            version_data = api_v2.version(version_pk).get()
        else:
            version_data = (api_v2
                            .version(project.slug)
                            .get(slug=LATEST)['objects'][0])
        return APIVersion(**version_data)

    @staticmethod
    def get_build(build_pk):
        """
        Retrieve build object from API.

        :param build_pk: Build primary key
        """
        build = {}
        if build_pk:
            build = api_v2.build(build_pk).get()
        return dict((key, val) for (key, val) in list(build.items())
                    if key not in ['project', 'version', 'resource_uri',
                                   'absolute_uri'])

    def setup_vcs(self):
        """
        Update the checkout of the repo to make sure it's the latest.

        This also syncs versions in the DB.

        :param build_env: Build environment
        """
        self.setup_env.update_build(state=BUILD_STATE_CLONING)

        self._log(msg='Updating docs from VCS')
        update_imported_docs(self.version.pk)
        commit = self.project.vcs_repo(self.version.slug).commit
        if commit:
            self.build['commit'] = commit

    def get_env_vars(self):
        """Get bash environment variables used for all builder commands."""
        env = {
            'READTHEDOCS': True,
            'READTHEDOCS_VERSION': self.version.slug,
            'READTHEDOCS_PROJECT': self.project.slug
        }

        if self.config.use_conda:
            env.update({
                'CONDA_ENVS_PATH': os.path.join(self.project.doc_path, 'conda'),
                'CONDA_DEFAULT_ENV': self.version.slug,
                'BIN_PATH': os.path.join(self.project.doc_path, 'conda', self.version.slug, 'bin')
            })
        else:
            env.update({
                'BIN_PATH': os.path.join(self.project.doc_path, 'envs', self.version.slug, 'bin')
            })

        return env

    def set_valid_clone(self):
        """Mark on the project that it has been cloned properly."""
        project_data = api_v2.project(self.project.pk).get()
        project_data['has_valid_clone'] = True
        api_v2.project(self.project.pk).put(project_data)
        self.project.has_valid_clone = True

    def update_documentation_type(self):
        """
        Force Sphinx for 'auto' documentation type.

        This used to determine the type and automatically set the documentation
        type to Sphinx for rST and Mkdocs for markdown. It now just forces
        Sphinx, due to markdown support.
        """
        ret = 'sphinx'
        project_data = api_v2.project(self.project.pk).get()
        project_data['documentation_type'] = ret
        api_v2.project(self.project.pk).put(project_data)
        self.project.documentation_type = ret

    def update_app_instances(self, html=False, localmedia=False, search=False,
                             pdf=False, epub=False):
        """
        Update application instances with build artifacts.

        This triggers updates across application instances for html, pdf, epub,
        downloads, and search. Tasks are broadcast to all web servers from here.
        """
        # Update version if we have successfully built HTML output
        try:
            if html:
                version = api_v2.version(self.version.pk)
                version.patch({
                    'active': True,
                    'built': True,
                })
        except HttpClientError:
            log.exception('Updating version failed, skipping file sync: version=%s' % self.version)

        # Broadcast finalization steps to web application instances
        broadcast(
            type='app',
            task=sync_files,
            args=[
                self.project.pk,
                self.version.pk,
            ],
            kwargs=dict(
                hostname=socket.gethostname(),
                html=html,
                localmedia=localmedia,
                search=search,
                pdf=pdf,
                epub=epub,
            ),
            callback=sync_callback.s(version_pk=self.version.pk, commit=self.build['commit']),
        )

    def setup_environment(self):
        """
        Build the virtualenv and install the project into it.

        Always build projects with a virtualenv.

        :param build_env: Build environment to pass commands and execution through.
        """
        self.build_env.update_build(state=BUILD_STATE_INSTALLING)

        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):

            self.python_env.delete_existing_build_dir()
            self.python_env.setup_base()
            self.python_env.install_core_requirements()
            self.python_env.install_user_requirements()
            self.python_env.install_package()

    def build_docs(self):
        """
        Wrapper to all build functions.

        Executes the necessary builds for this task and returns whether the
        build was successful or not.

        :returns: Build outcomes with keys for html, search, localmedia, pdf,
                  and epub
        :rtype: dict
        """
        self.build_env.update_build(state=BUILD_STATE_BUILDING)
        before_build.send(sender=self.version)

        outcomes = defaultdict(lambda: False)
        with self.project.repo_nonblockinglock(
                version=self.version,
                max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):
            outcomes['html'] = self.build_docs_html()
            outcomes['search'] = self.build_docs_search()
            outcomes['localmedia'] = self.build_docs_localmedia()
            outcomes['pdf'] = self.build_docs_pdf()
            outcomes['epub'] = self.build_docs_epub()

        after_build.send(sender=self.version)
        return outcomes

    def build_docs_html(self):
        """Build HTML docs."""
        html_builder = get_builder_class(self.project.documentation_type)(
            build_env=self.build_env,
            python_env=self.python_env,
        )
        if self.build_force:
            html_builder.force()
        html_builder.append_conf()
        success = html_builder.build()
        if success:
            html_builder.move()

        # Gracefully attempt to move files via task on web workers.
        try:
            broadcast(type='app', task=move_files,
                      args=[self.version.pk, socket.gethostname()],
                      kwargs=dict(html=True)
                      )
        except socket.error:
            log.exception('move_files task has failed on socket error.')

        return success

    def build_docs_search(self):
        """Build search data with separate build."""
        if self.build_search:
            if self.project.is_type_mkdocs:
                return self.build_docs_class('mkdocs_json')
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_search')
        return False

    def build_docs_localmedia(self):
        """Get local media files with separate build."""
        if 'htmlzip' not in self.config.formats:
            return False

        if self.build_localmedia:
            if self.project.is_type_sphinx:
                return self.build_docs_class('sphinx_singlehtmllocalmedia')
        return False

    def build_docs_pdf(self):
        """Build PDF docs."""
        if ('pdf' not in self.config.formats or
            self.project.slug in HTML_ONLY or
                not self.project.is_type_sphinx):
            return False
        return self.build_docs_class('sphinx_pdf')

    def build_docs_epub(self):
        """Build ePub docs."""
        if ('epub' not in self.config.formats or
            self.project.slug in HTML_ONLY or
                not self.project.is_type_sphinx):
            return False
        return self.build_docs_class('sphinx_epub')

    def build_docs_class(self, builder_class):
        """
        Build docs with additional doc backends.

        These steps are not necessarily required for the build to halt, so we
        only raise a warning exception here. A hard error will halt the build
        process.
        """
        builder = get_builder_class(builder_class)(self.build_env, python_env=self.python_env)
        success = builder.build()
        builder.move()
        return success

    def send_notifications(self):
        """Send notifications on build failure."""
        send_notifications.delay(self.version.pk, build_pk=self.build['id'])