Beispiel #1
0
def setup_vcs(version, build, api):
    """
    Update the checkout of the repo to make sure it's the latest.
    This also syncs versions in the DB.
    """

    log.info(
        LOG_TEMPLATE.format(project=version.project.slug,
                            version=version.slug,
                            msg='Updating docs from VCS'))
    try:
        update_output = update_imported_docs(version.pk, api)
        commit = version.project.vcs_repo(version.slug).commit
        if commit:
            build['commit'] = commit
    except ProjectImportError, err:
        log.error(LOG_TEMPLATE.format(
            project=version.project.slug,
            version=version.slug,
            msg='Failed to import project; skipping build'),
                  exc_info=True)
        build['state'] = 'finished'
        build['setup_error'] = ('Failed to import project; skipping build.\n'
                                '\nError\n-----\n\n%s' % err.message)
        api.build(build['id']).put(build)
        return False
Beispiel #2
0
def record_build(api, record, build, results, state, start_time=None):
    """
    Record a build by hitting the API.

    Returns nothing
    """

    if not record:
        return None

    setup_steps = [
        'checkout', 'venv', 'doc_builder', 'requirements', 'install'
    ]
    output_steps = ['html']
    all_steps = setup_steps + output_steps

    build['state'] = state
    if 'html' in results:
        build['success'] = results['html'][0] == 0
    else:
        build['success'] = False

    # Set global state
    # for step in all_steps:
    #     if results.get(step, False):
    #         if results.get(step)[0] != 0:
    #             results['success'] = False

    build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])

    build['setup'] = build['setup_error'] = ""
    build['output'] = build['error'] = ""

    if start_time:
        build['length'] = (datetime.datetime.utcnow() -
                           start_time).total_seconds()

    for step in setup_steps:
        if step in results:
            build['setup'] += "\n\n%s\n-----\n\n" % step
            build['setup'] += results.get(step)[1]
            build['setup_error'] += "\n\n%s\n-----\n\n" % step
            build['setup_error'] += results.get(step)[2]

    for step in output_steps:
        if step in results:
            build['output'] += "\n\n%s\n-----\n\n" % step
            build['output'] += results.get(step)[1]
            build['error'] += "\n\n%s\n-----\n\n" % step
            build['error'] += results.get(step)[2]

    # Attempt to stop unicode errors on build reporting
    for key, val in build.items():
        if isinstance(val, basestring):
            build[key] = val.decode('utf-8', 'ignore')

    try:
        api.build(build['id']).put(build)
    except Exception:
        log.error("Unable to post a new build", exc_info=True)
def record_build(api, record, build, results, state):
    """
    Record a build by hitting the API.

    Returns nothing
    """

    if not record:
        return None

    setup_steps = ['checkout', 'venv', 'doc_builder', 'requirements', 'install']
    output_steps = ['html']
    all_steps = setup_steps + output_steps

    build['state'] = state
    if 'html' in results:
        build['success'] = results['html'][0] == 0
    else:
        build['success'] = False

    # Set global state
    # for step in all_steps:
    #     if results.get(step, False):
    #         if results.get(step)[0] != 0:
    #             results['success'] = False

    build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])

    build['setup'] = build['setup_error'] = ""
    build['output'] = build['error'] = ""

    for step in setup_steps:
        if step in results:
            build['setup'] += "\n\n%s\n-----\n\n" % step
            build['setup'] += results.get(step)[1]
            build['setup_error'] += "\n\n%s\n-----\n\n" % step
            build['setup_error'] += results.get(step)[2]

    for step in output_steps:
        if step in results:
            build['output'] += "\n\n%s\n-----\n\n" % step
            build['output'] += results.get(step)[1]
            build['error'] += "\n\n%s\n-----\n\n" % step
            build['error'] += results.get(step)[2]

    # Attempt to stop unicode errors on build reporting
    for key, val in build.items():
        if isinstance(val, basestring):
            build[key] = val.decode('utf-8', 'ignore')

    try:
        api.build(build['id']).put(build)
    except Exception:
        log.error("Unable to post a new build", exc_info=True)
Beispiel #4
0
def record_build(api, record, build, results, state):
    """
    Record a build by hitting the API.

    Returns nothing
    """

    if not record:
        return None

    setup_steps = [
        'checkout', 'venv', 'doc_builder', 'requirements', 'install'
    ]
    output_steps = ['html']
    all_steps = setup_steps + output_steps

    build['state'] = state
    if 'html' in results:
        build['success'] = results['html'][0] == 0
    else:
        build['success'] = False

    # Set global state
    # for step in all_steps:
    #     if results.get(step, False):
    #         if results.get(step)[0] != 0:
    #             results['success'] = False

    build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])

    build['setup'] = build.get('setup', '')
    build['setup_error'] = build.get('setup_error', '')

    build['output'] = build.get('output', '')
    build['error'] = build.get('error', '')

    for step in setup_steps:
        if step in results:
            build['setup'] += "\n\n%s\n-----\n\n" % step
            build['setup'] += results.get(step)[1]
            build['setup_error'] += "\n\n%s\n-----\n\n" % step
            build['setup_error'] += results.get(step)[2]

    for step in output_steps:
        if step in results:
            build['output'] += "\n\n%s\n-----\n\n" % step
            build['output'] += results.get(step)[1]
            build['error'] += "\n\n%s\n-----\n\n" % step
            build['error'] += results.get(step)[2]
    try:
        api.build(build['id']).put(build)
    except Exception:
        log.error("Unable to post a new build", exc_info=True)
Beispiel #5
0
def record_build(api, record, build, results, state):
    """
    Record a build by hitting the API.

    Returns nothing
    """

    if not record:
        return None

    setup_steps = ['checkout', 'venv', 'doc_builder', 'requirements', 'install']
    output_steps = ['html']
    all_steps = setup_steps + output_steps

    build['state'] = state
    if 'html' in results:
        build['success'] = results['html'][0] == 0
    else:
        build['success'] = False

    # Set global state
    # for step in all_steps:
    #     if results.get(step, False):
    #         if results.get(step)[0] != 0:
    #             results['success'] = False

    build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])

    build['setup'] = build.get('setup', '')
    build['setup_error'] = build.get('setup_error', '')

    build['output'] = build.get('output', '')
    build['error'] = build.get('error', '')

    for step in setup_steps:
        if step in results:
            build['setup'] += "\n\n%s\n-----\n\n" % step
            build['setup'] += results.get(step)[1]
            build['setup_error'] += "\n\n%s\n-----\n\n" % step
            build['setup_error'] += results.get(step)[2]

    for step in output_steps:
        if step in results:
            build['output'] += "\n\n%s\n-----\n\n" % step
            build['output'] += results.get(step)[1]
            build['error'] += "\n\n%s\n-----\n\n" % step
            build['error'] += results.get(step)[2]
    try:
        api.build(build['id']).put(build)
    except Exception:
        log.error("Unable to post a new build", exc_info=True)
Beispiel #6
0
def setup_vcs(version, build, api):
    """
    Update the checkout of the repo to make sure it's the latest.
    This also syncs versions in the DB.
    """

    log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Updating docs from VCS'))
    try:
        update_output = update_imported_docs(version.pk, api)
    except ProjectImportError, err:
        log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True)
        build['state'] = 'finished'
        build['setup_error'] = (
            'Failed to import project; skipping build.\n'
            '\nError\n-----\n\n%s' % err.message
        )
        api.build(build['id']).put(build)
        return False
Beispiel #7
0
def setup_vcs(version, build, api):
    """
    Update the checkout of the repo to make sure it's the latest.
    This also syncs versions in the DB.
    """

    log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Updating docs from VCS"))
    try:
        update_output = update_imported_docs(version.pk, api)
    except ProjectImportError, err:
        log.error(
            LOG_TEMPLATE.format(
                project=version.project.slug, version=version.slug, msg="Failed to import project; skipping build"
            ),
            exc_info=True,
        )
        build["state"] = "finished"
        build["setup_error"] = "Failed to import project; skipping build.\n" "\nError\n-----\n\n%s" % err.message
        api.build(build["id"]).put(build)
        return False
Beispiel #8
0
def create_build(build_pk):
    """
    Old placeholder for build creation. Now it just gets it from the database.
    """
    if build_pk:
        build = api.build(build_pk).get()
        for key in ['project', 'version', 'resource_uri', 'absolute_uri']:
            if key in build:
                del build[key]
    else:
        build = {}
    return build
Beispiel #9
0
def create_build(build_pk):
    """
    Old placeholder for build creation. Now it just gets it from the database.
    """
    if build_pk:
        build = api.build(build_pk).get()
        for key in ['project', 'version', 'resource_uri', 'absolute_uri']:
            if key in build:
                del build[key]
    else:
        build = {}
    return build
Beispiel #10
0
def record_build(api, record, build, results, state):
    """
    Record a build by hitting the API.

    Returns nothing
    """

    if not record:
        return None

    setup_steps = ["checkout", "venv", "sphinx", "requirements", "install"]
    output_steps = ["html"]
    all_steps = setup_steps + output_steps

    build["state"] = state
    if "html" in results:
        build["success"] = results["html"][0] == 0
    else:
        build["success"] = False

    # # Set global state
    # for step in all_steps:
    #     if results.get(step, False):
    #         if results.get(step)[0] != 0:
    #             results['success'] = False

    build["exit_code"] = max([results.get(step, [0])[0] for step in all_steps])

    build["setup"] = build["setup_error"] = ""
    build["output"] = build["error"] = ""

    for step in setup_steps:
        if step in results:
            build["setup"] += "\n\n%s\n-----\n\n" % step
            build["setup"] += results.get(step)[1]
            build["setup_error"] += "\n\n%s\n-----\n\n" % step
            build["setup_error"] += results.get(step)[2]

    for step in output_steps:
        if step in results:
            build["output"] += "\n\n%s\n-----\n\n" % step
            build["output"] += results.get(step)[1]
            build["error"] += "\n\n%s\n-----\n\n" % step
            build["error"] += results.get(step)[2]
    try:
        ret = api.build(build["id"]).put(build)
    except Exception, e:
        log.error("Unable to post a new build", exc_info=True)
Beispiel #11
0
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    update_output = kwargs.get('update_output', {})
    project_data = api.project(pk).get()
    del project_data['users']
    del project_data['resource_uri']
    del project_data['absolute_url']
    project = Project(**project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0
    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
        del version_data['resource_uri']
    else:
        #Create or use the 'latest' branch, which is the default for a project.
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            version_data = api.version(project.slug).get(slug='latest')['objects'][0]
            del version_data['resource_uri']
        except (slumber.exceptions.HttpClientError, IndexError) as exc:
            #if exc.response.status_code in [404,500]:
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
                )
            try:
                version_data = api.version.post(version_data)
                del version_data['resource_uri']
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                #raise e
    version_data['project'] = project
    version = Version(**version_data)
    version.save = new_save

    if not version_pk:
        #Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data['project'] = "/api/v1/version/%s/" % version_data['project'].pk
            api.version(version.pk).put(version_data)

    if record:
        #Create Build Object.
        build = api.build.post(dict(
            project= '/api/v1/project/%s/' % project.pk,
            version= '/api/v1/version/%s/' % version.pk,
            type='html',
            state='triggered',
        ))
    else:
        build = {}

    #Make Dirs
    path = project.doc_path
    if not os.path.exists(path):
        os.makedirs(path)
    with project.repo_lock(30):
        if project.is_imported:
            try:
                update_output = update_imported_docs(project, version)
            except ProjectImportError, err:
                log.error("Failed to import project; skipping build.", exc_info=True)
                build['state'] = 'finished'
                build['setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
                api.build(build['id']).put(build)
                return False
        else:
Beispiel #12
0
def update_docs(pk,
                record=True,
                pdf=True,
                man=True,
                epub=True,
                dash=True,
                version_pk=None,
                force=False,
                **kwargs):
    """The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts. It
    also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.

    """

    ###
    # Handle passed in arguments
    ###
    project_data = api.project(pk).get()
    project = make_api_project(project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0

    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = (api.version(
                project.slug).get(slug='latest')['objects'][0])
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                raise e

    version = make_api_version(version_data)
    version.save = new_save

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data['project'] = ("/api/v1/version/%s/" %
                                       version_data['project'].pk)
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(
            dict(
                project='/api/v1/project/%s/' % project.pk,
                version='/api/v1/version/%s/' % version.pk,
                type='html',
                state='triggered',
            ))
    else:
        build = {}

    try:
        log.info("Updating docs from VCS")
        update_output = update_imported_docs(version.pk)
        #update_output = update_result.get()
    except ProjectImportError, err:
        log.error("Failed to import project; skipping build.", exc_info=True)
        build['state'] = 'finished'
        build['setup_error'] = ('Failed to import project; skipping build.\n'
                                'Please make sure your repo is correct and '
                                'you have a conf.py')
        api.build(build['id']).put(build)
        return False
Beispiel #13
0
    if record:
        # Update the build with info about the setup
        build['state'] = 'building'
        output_data = error_data = ''
        # Grab all the text from updating the code via VCS.
        for key in ['checkout', 'venv', 'sphinx', 'requirements', 'install']:
            data = update_output.get(key, None)
            if data:
                try:
                    output_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[1])
                    error_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[2])
                except UnicodeDecodeError:
                    log.debug("Unicode Error in setup")
        build['setup'] = output_data
        build['setup_error'] = error_data
        api.build(build['id']).put(build)

    log.info("Building docs")
    # This is only checking the results of the HTML build, as it's a canary
    try:
        results = build_docs(version_pk=version.pk,
                             pdf=pdf,
                             man=man,
                             epub=epub,
                             dash=dash,
                             record=record,
                             force=force)
        (html_results, latex_results, pdf_results, man_results, epub_results,
         dash_results) = results
        (ret, out, err) = html_results
    except Exception as e:
Beispiel #14
0
    if record:
        # Update the build with info about the setup
        build['state'] = 'building'
        output_data = error_data = ''
        # Grab all the text from updating the code via VCS.
        for key in ['checkout', 'venv', 'sphinx', 'requirements', 'install']:
            data = update_output.get(key, None)
            if data:
                try:
                    output_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[1])
                    error_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[2])
                except UnicodeDecodeError:
                    log.debug("Unicode Error in setup")
        build['setup'] = output_data
        build['setup_error'] = error_data
        api.build(build['id']).put(build)

    log.info("Building docs")
    # This is only checking the results of the HTML build, as it's a canary
    try:
        results = build_docs(version_pk=version.pk, pdf=pdf, man=man,
                             epub=epub, dash=dash, record=record, force=force)
        (html_results, latex_results, pdf_results, man_results, epub_results,
         dash_results) = results
        (ret, out, err) = html_results
    except Exception as e:
        log.error("Exception in flailboat build_docs", exc_info=True)
        html_results = (999, "Project build Failed", str(e))
        latex_results = (999, "Project build Failed", str(e))
        pdf_results = (999, "Project build Failed", str(e))
        # These variables aren't currently being used.
Beispiel #15
0
def update_docs(pk,
                record=True,
                pdf=True,
                man=True,
                epub=True,
                version_pk=None,
                force=False,
                **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    update_output = kwargs.get('update_output', {})
    project_data = api.project(pk).get()
    del project_data['users']
    del project_data['resource_uri']
    del project_data['absolute_url']
    project = Project(**project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0

    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
        del version_data['resource_uri']
    else:
        #Create or use the 'latest' branch, which is the default for a project.
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            version_data = api.version(
                project.slug).get(slug='latest')['objects'][0]
            del version_data['resource_uri']
        except (slumber.exceptions.HttpClientError, IndexError) as exc:
            #if exc.response.status_code in [404,500]:
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                active=True,
                verbose_name='latest',
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
                del version_data['resource_uri']
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                #raise e
    version_data['project'] = project
    version = Version(**version_data)
    version.save = new_save

    if not version_pk:
        #Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data[
                'project'] = "/api/v1/version/%s/" % version_data['project'].pk
            api.version(version.pk).put(version_data)

    if record:
        #Create Build Object.
        build = api.build.post(
            dict(
                project='/api/v1/project/%s/' % project.pk,
                version='/api/v1/version/%s/' % version.pk,
                type='html',
                state='triggered',
            ))
    else:
        build = {}

    #Make Dirs
    path = project.doc_path
    if not os.path.exists(path):
        os.makedirs(path)
    with project.repo_lock(30):
        if project.is_imported:
            try:
                update_output = update_imported_docs(project, version)
            except ProjectImportError, err:
                log.error("Failed to import project; skipping build.",
                          exc_info=True)
                build['state'] = 'finished'
                build[
                    'setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
                api.build(build['id']).put(build)
                return False
        else:
Beispiel #16
0
def update_docs(pk, record=True, pdf=True, man=True, epub=True, dash=True,
                search=True, version_pk=None, force=False, intersphinx=True,
                localmedia=True, api=None, **kwargs):
    """The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts. It
    also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.

    """

    if api is None:
        api = tastyapi.api

    project_data = api.project(pk).get()
    project = make_api_project(project_data)
    if 'tryton' in  project.repo:
        # Skip for now
        return

    log.info(LOG_TEMPLATE.format(project=project.slug, version='', msg='Building'))
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = (api.version(project.slug)
                            .get(slug='latest')['objects'][0])
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                type='branch',
                active=True,
                verbose_name='latest',
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info(LOG_TEMPLATE.format(project=project.slug, version='', msg='Exception in creating version: %s' % e))
                raise e

    version = make_api_version(version_data)

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data['project'] = ("/api/v1/version/%s/"
                                       % version_data['project'].pk)
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(dict(
            project='/api/v1/project/%s/' % project.pk,
            version='/api/v1/version/%s/' % version.pk,
            type='html',
            state='triggered',
        ))
    else:
        build = {}

    try:
        log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Updating docs from VCS'))
        update_output = update_imported_docs(version.pk, api)
        #update_output = update_result.get()
    except ProjectImportError, err:
        log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True)
        build['state'] = 'finished'
        build['setup_error'] = (
            'Failed to import project; skipping build.\n'
            '\nError\n-----\n\n%s' % err.message
        )
        api.build(build['id']).put(build)
        return False
Beispiel #17
0
def update_docs(pk,
                record=True,
                pdf=True,
                man=True,
                epub=True,
                dash=True,
                search=True,
                version_pk=None,
                force=False,
                intersphinx=True,
                localmedia=True,
                api=None,
                **kwargs):
    """The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts. It
    also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.

    """

    if api is None:
        api = tastyapi.api

    project_data = api.project(pk).get()
    project = make_api_project(project_data)
    if 'tryton' in project.repo:
        # Skip for now
        return

    log.info(
        LOG_TEMPLATE.format(project=project.slug, version='', msg='Building'))
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = (api.version(
                project.slug).get(slug='latest')['objects'][0])
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project='/api/v1/project/%s/' % project.pk,
                slug='latest',
                type='branch',
                active=True,
                verbose_name='latest',
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info(
                    LOG_TEMPLATE.format(
                        project=project.slug,
                        version='',
                        msg='Exception in creating version: %s' % e))
                raise e

    version = make_api_version(version_data)

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data['verbose_name'] = 'latest'
            to_save = True
        if not version.active:
            version_data['active'] = True
            to_save = True
        if version.identifier != branch:
            version_data['identifier'] = branch
            to_save = True
        if to_save:
            version_data['project'] = ("/api/v1/version/%s/" %
                                       version_data['project'].pk)
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(
            dict(
                project='/api/v1/project/%s/' % project.pk,
                version='/api/v1/version/%s/' % version.pk,
                type='html',
                state='triggered',
            ))
    else:
        build = {}

    try:
        log.info(
            LOG_TEMPLATE.format(project=project.slug,
                                version=version.slug,
                                msg='Updating docs from VCS'))
        update_output = update_imported_docs(version.pk, api)
        #update_output = update_result.get()
    except ProjectImportError, err:
        log.error(LOG_TEMPLATE.format(
            project=project.slug,
            version=version.slug,
            msg='Failed to import project; skipping build'),
                  exc_info=True)
        build['state'] = 'finished'
        build['setup_error'] = ('Failed to import project; skipping build.\n'
                                '\nError\n-----\n\n%s' % err.message)
        api.build(build['id']).put(build)
        return False
Beispiel #18
0
def update_docs(
    pk,
    record=True,
    pdf=True,
    man=True,
    epub=True,
    dash=True,
    search=True,
    version_pk=None,
    force=False,
    intersphinx=True,
    api=None,
    **kwargs
):
    """The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we
    created it.  Then it will build the html docs and other requested parts. It
    also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands
        from the shell, for example.

    """

    if api is None:
        api = tastyapi.api

    project_data = api.project(pk).get()
    project = make_api_project(project_data)
    # if 'edx-platform' in  project.repo:
    # Skip edx for now
    # return

    log.info(LOG_TEMPLATE.format(project=project.slug, version="", msg="Building"))
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = api.version(project.slug).get(slug="latest")["objects"][0]
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project="/api/v1/project/%s/" % project.pk,
                slug="latest",
                type="branch",
                active=True,
                verbose_name="latest",
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info(
                    LOG_TEMPLATE.format(project=project.slug, version="", msg="Exception in creating version: %s" % e)
                )
                raise e

    version = make_api_version(version_data)

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data["verbose_name"] = "latest"
            to_save = True
        if not version.active:
            version_data["active"] = True
            to_save = True
        if version.identifier != branch:
            version_data["identifier"] = branch
            to_save = True
        if to_save:
            version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(
            dict(
                project="/api/v1/project/%s/" % project.pk,
                version="/api/v1/version/%s/" % version.pk,
                type="html",
                state="triggered",
            )
        )
    else:
        build = {}

    try:
        log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="Updating docs from VCS"))
        update_output = update_imported_docs(version.pk, api)
        # update_output = update_result.get()
    except ProjectImportError, err:
        log.error(
            LOG_TEMPLATE.format(
                project=project.slug, version=version.slug, msg="Failed to import project; skipping build"
            ),
            exc_info=True,
        )
        build["state"] = "finished"
        build["setup_error"] = "Failed to import project; skipping build.\n" "\nError\n-----\n\n%s" % err.message
        api.build(build["id"]).put(build)
        return False
Beispiel #19
0
    if record:
        # Update the build with info about the setup
        build["state"] = "building"
        output_data = error_data = ""
        # Grab all the text from updating the code via VCS.
        for key in ["checkout", "venv", "sphinx", "requirements", "install"]:
            data = update_output.get(key, None)
            if data:
                try:
                    output_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[1])
                    error_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[2])
                except UnicodeDecodeError:
                    log.debug("Unicode Error in setup")
        build["setup"] = output_data
        build["setup_error"] = error_data
        api.build(build["id"]).put(build)

    log.info("Building docs")
    # This is only checking the results of the HTML build, as it's a canary
    try:
        (html_results, latex_results, pdf_results, man_results, epub_results) = build_docs(
            version_pk=version.pk, pdf=pdf, man=man, epub=epub, record=record, force=force
        )
        (ret, out, err) = html_results
    except Exception as e:
        log.error("Exception in flailboat build_docs", exc_info=True)
        html_results = (999, "Project build Failed", str(e))
        latex_results = (999, "Project build Failed", str(e))
        pdf_results = (999, "Project build Failed", str(e))
        man_results = (999, "Project build Failed", str(e))
        epub_results = (999, "Project build Failed", str(e))
Beispiel #20
0
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
    """
    The main entry point for updating documentation.

    It handles all of the logic around whether a project is imported or we created it.
    Then it will build the html docs and other requested parts.
    It also handles clearing the varnish cache.

    `pk`
        Primary key of the project to update

    `record`
        Whether or not to keep a record of the update in the database. Useful
        for preventing changes visible to the end-user when running commands from
        the shell, for example.
    """

    ###
    # Handle passed in arguments
    ###
    project_data = api.project(pk).get()
    project = make_api_project(project_data)

    # Prevent saving the temporary Project instance
    def new_save(*args, **kwargs):
        log.warning("Called save on a non-real object.")
        return 0

    project.save = new_save

    log.info("Building %s" % project)
    if version_pk:
        version_data = api.version(version_pk).get()
    else:
        branch = project.default_branch or project.vcs_repo().fallback_branch
        try:
            # Use latest version
            version_data = api.version(project.slug).get(slug="latest")["objects"][0]
        except (slumber.exceptions.HttpClientError, IndexError):
            # Create the latest version since it doesn't exist
            version_data = dict(
                project="/api/v1/project/%s/" % project.pk,
                slug="latest",
                active=True,
                verbose_name="latest",
                identifier=branch,
            )
            try:
                version_data = api.version.post(version_data)
            except Exception as e:
                log.info("Exception in creating version: %s" % e)
                raise e

    version = make_api_version(version_data)
    version.save = new_save

    if not version_pk:
        # Lots of course correction.
        to_save = False
        if not version.verbose_name:
            version_data["verbose_name"] = "latest"
            to_save = True
        if not version.active:
            version_data["active"] = True
            to_save = True
        if version.identifier != branch:
            version_data["identifier"] = branch
            to_save = True
        if to_save:
            version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk
            api.version(version.pk).put(version_data)

    if record:
        # Create Build Object.
        build = api.build.post(
            dict(
                project="/api/v1/project/%s/" % project.pk,
                version="/api/v1/version/%s/" % version.pk,
                type="html",
                state="triggered",
            )
        )
    else:
        build = {}

    try:
        log.info("Updating docs from VCS")
        update_output = update_imported_docs(version.pk)
        # update_output = update_result.get()
    except ProjectImportError, err:
        log.error("Failed to import project; skipping build.", exc_info=True)
        build["state"] = "finished"
        build[
            "setup_error"
        ] = "Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py"
        api.build(build["id"]).put(build)
        return False
Beispiel #21
0
def record_build(api, record, build, results, state, start_time=None):
    """
    Record a build by hitting the API.

    Returns nothing
    """

    if not record:
        return None

    build["builder"] = socket.gethostname()

    setup_steps = ["checkout", "venv", "doc_builder", "requirements", "install"]
    output_steps = ["html"]
    all_steps = setup_steps + output_steps

    build["state"] = state
    if "html" in results:
        build["success"] = results["html"][0] == 0
    else:
        build["success"] = False

    # Set global state
    # for step in all_steps:
    #     if results.get(step, False):
    #         if results.get(step)[0] != 0:
    #             results['success'] = False

    build["exit_code"] = max([results.get(step, [0])[0] for step in all_steps])

    build["setup"] = build["setup_error"] = ""
    build["output"] = build["error"] = ""

    if start_time:
        build["length"] = (datetime.datetime.utcnow() - start_time).total_seconds()

    for step in setup_steps:
        if step in results:
            build["setup"] += "\n\n%s\n-----\n\n" % step
            try:
                build["setup"] += results.get(step)[1]
            except (IndexError, TypeError):
                pass
            build["setup_error"] += "\n\n%s\n-----\n\n" % step
            try:
                build["setup_error"] += results.get(step)[2]
            except (IndexError, TypeError):
                pass

    for step in output_steps:
        if step in results:
            build["output"] += "\n\n%s\n-----\n\n" % step
            try:
                build["output"] += results.get(step)[1]
            except (IndexError, TypeError):
                pass
            build["error"] += "\n\n%s\n-----\n\n" % step
            try:
                build["error"] += results.get(step)[2]
            except (IndexError, TypeError):
                pass

    # Attempt to stop unicode errors on build reporting
    for key, val in build.items():
        if isinstance(val, basestring):
            build[key] = val.decode("utf-8", "ignore")

    try:
        api.build(build["id"]).put(build)
    except Exception:
        log.error("Unable to post a new build", exc_info=True)