Exemplo n.º 1
0
def clear_artifacts(version_pk):
    """ Remove artifacts from the build server. """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    run('rm -rf %s' % version.project.full_epub_path(version.slug))
    run('rm -rf %s' % version.project.full_man_path(version.slug))
    run('rm -rf %s' % version.project.full_build_path(version.slug))
    run('rm -rf %s' % version.project.full_latex_path(version.slug))
Exemplo n.º 2
0
def clear_artifacts(version_pk):
    """ Remove artifacts from the build server. """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    run('rm -rf %s' % version.project.full_epub_path(version.slug))
    run('rm -rf %s' % version.project.full_man_path(version.slug))
    run('rm -rf %s' % version.project.full_build_path(version.slug))
    run('rm -rf %s' % version.project.full_latex_path(version.slug))
Exemplo n.º 3
0
def update_intersphinx(version_pk):
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    try:
        object_file = version.project.find('objects.inv', version.slug)[0]
    except IndexError, e:
        print "Failed to find objects file"
        return None
Exemplo n.º 4
0
def update_intersphinx(version_pk):
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    try:
        object_file = version.project.find('objects.inv', version.slug)[0]
    except IndexError, e:
        print "Failed to find objects file"
        return None
Exemplo n.º 5
0
def build_docs(version_pk, pdf, man, epub, record, force):
    """
    This handles the actual building of the documentation and DB records
    """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    if not project.conf_file(version.slug):
        return ('', 'Conf file not found.', -1)

    with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)):

        html_builder = builder_loading.get(project.documentation_type)(version)
        if force:
            html_builder.force()
        html_builder.clean()
        html_results = html_builder.build()
        if html_results[0] == 0:
            html_builder.move()

        fake_results = (999, "Project Skipped, Didn't build",
                        "Project Skipped, Didn't build")
        # Only build everything else if the html build changed.
        if html_builder.changed and not project.skip:
            if pdf:
                pdf_builder = builder_loading.get('sphinx_pdf')(version)
                latex_results, pdf_results = pdf_builder.build()
                # Always move pdf results even when there's an error.
                #if pdf_results[0] == 0:
                pdf_builder.move()
            else:
                pdf_results = latex_results = fake_results
            if man:
                man_builder = builder_loading.get('sphinx_man')(version)
                man_results = man_builder.build()
                if man_results[0] == 0:
                    man_builder.move()
            else:
                man_results = fake_results
            if epub:
                epub_builder = builder_loading.get('sphinx_epub')(version)
                epub_results = epub_builder.build()
                if epub_results[0] == 0:
                    epub_builder.move()
            else:
                epub_results = fake_results

    return (html_results, latex_results, pdf_results, man_results,
            epub_results)
Exemplo n.º 6
0
def build_docs(version_pk, pdf, man, epub, record, force):
    """
    This handles the actual building of the documentation and DB records
    """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    if not project.conf_file(version.slug):
        return ('', 'Conf file not found.', -1)

    with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)):

        html_builder = builder_loading.get(project.documentation_type)(version)
        if force:
            html_builder.force()
        html_builder.clean()
        html_results = html_builder.build()
        if html_results[0] == 0:
            html_builder.move()

        fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build")
        # Only build everything else if the html build changed.
        if html_builder.changed and not project.skip:
            if pdf:
                pdf_builder = builder_loading.get('sphinx_pdf')(version)
                latex_results, pdf_results = pdf_builder.build()
                # Always move pdf results even when there's an error.
                #if pdf_results[0] == 0:
                pdf_builder.move()
            else:
                pdf_results = latex_results = fake_results
            if man:
                man_builder = builder_loading.get('sphinx_man')(version)
                man_results = man_builder.build()
                if man_results[0] == 0:
                    man_builder.move()
            else:
                man_results = fake_results
            if epub:
                epub_builder = builder_loading.get('sphinx_epub')(version)
                epub_results = epub_builder.build()
                if epub_results[0] == 0:
                    epub_builder.move()
            else:
                epub_results = fake_results

    return (html_results, latex_results, pdf_results, man_results, epub_results)
Exemplo n.º 7
0
def update_docs(pk,
                record=True,
                pdf=True,
                man=True,
                epub=True,
                version_pk=None,
                force=False,
                **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    project_data = api.project(pk).get()
    project = make_api_project(project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0

    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = api.version(
                project.slug).get(slug='latest')['objects'][0]
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                raise e

    version = make_api_version(version_data)
    version.save = new_save

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data[
                'project'] = "/api/v1/version/%s/" % version_data['project'].pk
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(
            dict(
                project='/api/v1/project/%s/' % project.pk,
                version='/api/v1/version/%s/' % version.pk,
                type='html',
                state='triggered',
            ))
    else:
        build = {}

    try:
        log.info("Updating docs from VCS")
        update_output = update_imported_docs(version.pk)
        #update_output = update_result.get()
    except ProjectImportError, err:
        log.error("Failed to import project; skipping build.", exc_info=True)
        build['state'] = 'finished'
        build[
            'setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
        api.build(build['id']).put(build)
        return False
Exemplo n.º 8
0
def update_imported_docs(version_pk):
    """
    Check out or update the given project's repository.
    """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    # Make Dirs
    if not os.path.exists(project.doc_path):
        os.makedirs(project.doc_path)

    with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)):
        update_docs_output = {}
        if not project.vcs_repo():
            raise ProjectImportError("Repo type '{repo_type}' unknown".format(
                repo_type=project.repo_type))

        # Get the actual code on disk
        if version:
            log.info('Checking out version {slug}: {identifier}'.format(
                slug=version.slug, identifier=version.identifier))
            version_slug = version.slug
            version_repo = project.vcs_repo(version_slug)
            update_docs_output['checkout'] = version_repo.checkout(
                version.identifier)
        else:
            # Does this ever get called?
            log.info('Updating to latest revision')
            version_slug = 'latest'
            version_repo = project.vcs_repo(version_slug)
            update_docs_output['checkout'] = version_repo.update()

        # Ensure we have a conf file (an exception is raised if not)
        project.conf_file(version.slug)

        # Do Virtualenv bits:
        if project.use_virtualenv:
            if project.use_system_packages:
                site_packages = '--system-site-packages'
            else:
                site_packages = '--no-site-packages'
            update_docs_output['venv'] = run(
                '{cmd} --distribute {site_packages} {path}'.format(
                    cmd='virtualenv',
                    site_packages=site_packages,
                    path=project.venv_path(version=version_slug)))
            # Other code expects sphinx-build to be installed inside the virtualenv.
            # Using the -I option makes sure it gets installed even if it is
            # already installed system-wide (and --system-site-packages is used)
            if project.use_system_packages:
                ignore_option = '-I'
            else:
                ignore_option = ''
            update_docs_output['sphinx'] = run(
                '{cmd} install -U {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1'
                .format(cmd=project.venv_bin(version=version_slug, bin='pip'),
                        ignore_option=ignore_option))

            if project.requirements_file:
                os.chdir(project.checkout_path(version_slug))
                update_docs_output['requirements'] = run(
                    '{cmd} install -r {requirements}'.format(
                        cmd=project.venv_bin(version=version_slug, bin='pip'),
                        requirements=project.requirements_file))
            os.chdir(project.checkout_path(version_slug))
            update_docs_output['install'] = run(
                '{cmd} setup.py install --force'.format(
                    cmd=project.venv_bin(version=version_slug, bin='python')))

        # check tags/version
        #XXX:dc: what in this block raises the values error?
        try:
            old_versions = [
                obj['identifier']
                for obj in api.version.get(project__slug=project.slug,
                                           limit=5000)['objects']
            ]
            if version_repo.supports_tags:
                transaction.enter_transaction_management(True)
                tags = version_repo.tags
                for tag in tags:
                    if tag.identifier in old_versions:
                        continue
                    log.debug('NEW TAG: (%s not in %s)' %
                              (tag.identifier, old_versions))
                    slug = slugify_uniquely(Version,
                                            tag.verbose_name,
                                            'slug',
                                            255,
                                            project=project)
                    try:

                        version_data = api.version.post(
                            dict(project="/api/v1/project/%s/" % project.pk,
                                 slug=slug,
                                 identifier=tag.identifier,
                                 verbose_name=tag.verbose_name))
                        ver = make_api_version(version_data)
                        log.info("New tag found: {0}".format(tag.identifier))
                        ver, highest = project.highest_version[1]
                        ver_obj = mkversion(ver)
                        #TODO: Handle updating higher versions automatically.
                        #This never worked very well, anyways.
                        if highest and ver_obj and ver_obj > highest:
                            log.info("Highest version known, building docs")
                            update_docs.delay(ver.project.pk,
                                              version_pk=ver.pk)
                    except Exception, e:
                        log.error("Failed to create version (tag)",
                                  exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
            if version_repo.supports_branches:
                transaction.enter_transaction_management(True)
                branches = version_repo.branches
                for branch in branches:
                    if branch.identifier in old_versions:
                        continue
                    log.debug('NEW BRANCH: (%s not in %s)' %
                              (branch, old_versions))
                    slug = slugify_uniquely(Version,
                                            branch.verbose_name,
                                            'slug',
                                            255,
                                            project=project)
                    try:
                        api.version.post(
                            dict(project="/api/v1/project/%s/" % project.pk,
                                 slug=slug,
                                 identifier=branch.identifier,
                                 verbose_name=branch.verbose_name))
                        log.info("New branch found: {0}".format(
                            branch.identifier))
                    except Exception, e:
                        log.error("Failed to create version (branch)",
                                  exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
Exemplo n.º 9
0
        api.build.post(
            dict(
                project='/api/v1/project/%s/' % project.pk,
                version='/api/v1/version/%s/' % version.pk,
                success=pdf_results[0] == 0,
                type='pdf',
                setup=latex_results[1],
                setup_error=latex_results[2],
                output=pdf_results[1],
                error=pdf_results[2],
            ))

    if version:
        # Mark version active on the site
        version_data = api.version(version.pk).get()
        version_data['active'] = True
        version_data['built'] = True
        #Need to delete this because a bug in tastypie breaks on the users list.
        del version_data['project']
        try:
            api.version(version.pk).put(version_data)
        except Exception, e:
            log.error("Unable to post a new version", exc_info=True)

    # Build Finished, do house keeping bits

    if 'no targets are out of date.' in out:
        log.info("Build Unchanged")
    else:
        if ret == 0:
Exemplo n.º 10
0
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    project_data = api.project(pk).get()
    project = make_api_project(project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0
    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = api.version(project.slug).get(slug='latest')['objects'][0]
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
                )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                raise e

    version = make_api_version(version_data)
    version.save = new_save

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data['project'] = "/api/v1/version/%s/" % version_data['project'].pk
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(dict(
            project='/api/v1/project/%s/' % project.pk,
            version='/api/v1/version/%s/' % version.pk,
            type='html',
            state='triggered',
        ))
    else:
        build = {}

    try:
        log.info("Updating docs from VCS")
        update_output = update_imported_docs(version.pk)
        #update_output = update_result.get()
    except ProjectImportError, err:
        log.error("Failed to import project; skipping build.", exc_info=True)
        build['state'] = 'finished'
        build['setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
        api.build(build['id']).put(build)
        return False
Exemplo n.º 11
0
def update_imported_docs(version_pk):
    """
    Check out or update the given project's repository.
    """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    # Make Dirs
    if not os.path.exists(project.doc_path):
        os.makedirs(project.doc_path)

    with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)):
        update_docs_output = {}
        if not project.vcs_repo():
            raise ProjectImportError("Repo type '{repo_type}' unknown".format(
                    repo_type=project.repo_type))

        # Get the actual code on disk
        if version:
            log.info('Checking out version {slug}: {identifier}'.format(
                slug=version.slug, identifier=version.identifier))
            version_slug = version.slug
            version_repo = project.vcs_repo(version_slug)
            update_docs_output['checkout'] = version_repo.checkout(version.identifier)
        else:
            # Does this ever get called?
            log.info('Updating to latest revision')
            version_slug = 'latest'
            version_repo = project.vcs_repo(version_slug)
            update_docs_output['checkout'] = version_repo.update()

        # Ensure we have a conf file (an exception is raised if not)
        project.conf_file(version.slug)

        # Do Virtualenv bits:
        if project.use_virtualenv:
            if project.use_system_packages:
                site_packages = '--system-site-packages'
            else:
                site_packages = '--no-site-packages'
            update_docs_output['venv'] = run('{cmd} --distribute {site_packages} {path}'.format(
                    cmd='virtualenv',
                    site_packages=site_packages,
                    path=project.venv_path(version=version_slug)))
            # Other code expects sphinx-build to be installed inside the virtualenv.
            # Using the -I option makes sure it gets installed even if it is
            # already installed system-wide (and --system-site-packages is used)
            if project.use_system_packages:
                ignore_option = '-I'
            else:
                ignore_option = ''
            update_docs_output['sphinx'] = run('{cmd} install -U {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1'.format(
                    cmd=project.venv_bin(version=version_slug, bin='pip'), ignore_option=ignore_option))

            if project.requirements_file:
                os.chdir(project.checkout_path(version_slug))
                update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
                        cmd=project.venv_bin(version=version_slug, bin='pip'),
                        requirements=project.requirements_file))
            os.chdir(project.checkout_path(version_slug))
            update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
                    cmd=project.venv_bin(version=version_slug, bin='python')))

        # check tags/version
        #XXX:dc: what in this block raises the values error?
        try:
            old_versions = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=5000)['objects']]
            if version_repo.supports_tags:
                transaction.enter_transaction_management(True)
                tags = version_repo.tags
                for tag in tags:
                    if tag.identifier in old_versions:
                        continue
                    log.debug('NEW TAG: (%s not in %s)' % (tag.identifier, old_versions))
                    slug = slugify_uniquely(Version, tag.verbose_name,
                                            'slug', 255, project=project)
                    try:

                        version_data = api.version.post(dict(
                            project="/api/v1/project/%s/" % project.pk,
                            slug=slug,
                            identifier=tag.identifier,
                            verbose_name=tag.verbose_name
                        ))
                        ver = make_api_version(version_data)
                        log.info("New tag found: {0}".format(tag.identifier))
                        ver, highest = project.highest_version[1]
                        ver_obj = mkversion(ver)
                        #TODO: Handle updating higher versions automatically.
                        #This never worked very well, anyways.
                        if highest and ver_obj and ver_obj > highest:
                            log.info("Highest version known, building docs")
                            update_docs.delay(ver.project.pk, version_pk=ver.pk)
                    except Exception, e:
                        log.error("Failed to create version (tag)", exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
            if version_repo.supports_branches:
                transaction.enter_transaction_management(True)
                branches = version_repo.branches
                for branch in branches:
                    if branch.identifier in old_versions:
                        continue
                    log.debug('NEW BRANCH: (%s not in %s)' % (branch, old_versions))
                    slug = slugify_uniquely(Version, branch.verbose_name,
                                            'slug', 255, project=project)
                    try:
                        api.version.post(dict(
                            project="/api/v1/project/%s/" % project.pk,
                            slug=slug,
                            identifier=branch.identifier,
                            verbose_name=branch.verbose_name
                        ))
                        log.info("New branch found: {0}".format(branch.identifier))
                    except Exception, e:
                        log.error("Failed to create version (branch)", exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
Exemplo n.º 12
0
        api.build(build['id']).put(build)

        api.build.post(dict(
            project = '/api/v1/project/%s/' % project.pk,
            version = '/api/v1/version/%s/' % version.pk,
            success=pdf_results[0] == 0,
            type='pdf',
            setup=latex_results[1],
            setup_error=latex_results[2],
            output=pdf_results[1],
            error=pdf_results[2],
        ))

    if version:
        # Mark version active on the site
        version_data = api.version(version.pk).get()
        version_data['active'] = True
        version_data['built'] = True
        #Need to delete this because a bug in tastypie breaks on the users list.
        del version_data['project']
        try:
            api.version(version.pk).put(version_data)
        except Exception, e:
            log.error("Unable to post a new version", exc_info=True)

    # Build Finished, do house keeping bits

    if 'no targets are out of date.' in out:
        log.info("Build Unchanged")
    else:
        if ret == 0: