Esempio n. 1
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    if not project.vcs_repo:
        raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
    if version:
        print 'Checking out version %s: %s' % (version.slug, version.identifier)
        project.vcs_repo.checkout(version.identifier)
    else:
        print 'Updating to latest revision'
        project.vcs_repo.update()

    # check tags/version
    try:
        if project.vcs_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = project.vcs_repo.get_tags()
            old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True)
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project)
                try:
                    Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=tag.identifier,
                        verbose_name=tag.verbose_name
                    )
                except Exception, e:
                    print "Failed to create version (tag): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
        if project.vcs_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = project.vcs_repo.get_branches()
            old_branches = Version.objects.filter(project=project).values_list('identifier', flat=True)
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project)
                try:
                    Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=branch.identifier,
                        verbose_name=branch.verbose_name
                    )
                except Exception, e:
                    print "Failed to create version (branch): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
            #Kill deleted branches
            Version.objects.filter(project=project).exclude(identifier__in=old_branches).delete()
Esempio n. 2
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    if not project.vcs_repo:
        raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
    if version:
        print 'Checking out version %s: %s' % (version.slug, version.identifier)
        project.vcs_repo.checkout(version.identifier)
    else:
        print 'Updating to latest revision'
        project.vcs_repo.update()

        # check tags/version
        try:
            if project.vcs_repo.supports_tags:
                tags = project.vcs_repo.get_tags()
                old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True)
                for tag in tags:
                    if tag.identifier in old_tags:
                        continue
                    slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project)
                    try:
                        Version.objects.get_or_create(
                            project=project,
                            slug=slug,
                            identifier=tag.identifier,
                            verbose_name=tag.verbose_name
                        )
                    except Exception, e:
                        print "Failed to create version: %s" % e
                        transaction.rollback()
        except ValueError, e:
            print "Error getting tags: %s" % e
Esempio n. 3
0
def sync_versions(project, versions):
    """
    Update the database with the current versions from the repository.
    """
    old_versions = {}
    old_version_values = project.versions.values("identifier", "verbose_name")
    for version in old_version_values:
        old_versions[version["verbose_name"]] = version["identifier"]

    added = set()
    # Add new versions
    for version in versions:
        version_id = version["identifier"]
        version_name = version["verbose_name"]
        if version_name in old_versions.keys():
            if version_id == old_versions[version_name]:
                # Version is correct
                continue
            else:
                # Update slug with new identifier
                Version.objects.filter(project=project, verbose_name=version_name).update(identifier=version_id)
                log.info("(Sync Versions) Updated Version: [%s=%s] " % (version["verbose_name"], version["identifier"]))
        else:
            # New Version
            slug = slugify_uniquely(Version, version["verbose_name"], "slug", 255, project=project)
            Version.objects.create(
                project=project, slug=slug, identifier=version["identifier"], verbose_name=version["verbose_name"]
            )
            added.add(slug)
    if added:
        log.info("(Sync Versions) Added Versions: [%s] " % " ".join(added))
    return added
Esempio n. 4
0
def sync_versions(project, versions, type):
    """
    Update the database with the current versions from the repository.
    """
    # Bookkeeping for keeping tag/branch identifies correct
    verbose_names = [v['verbose_name'] for v in versions]
    project.versions.filter(verbose_name__in=verbose_names).update(type=type)

    old_versions = {}
    old_version_values = project.versions.values('identifier', 'verbose_name')
    for version in old_version_values:
        old_versions[version['verbose_name']] = version['identifier']

    added = set()
    # Add new versions
    for version in versions:
        version_id = version['identifier']
        version_name = version['verbose_name']
        if version_name in old_versions.keys():
            if version_id == old_versions[version_name]:
                # Version is correct
                continue
            else:
                # Update slug with new identifier
                Version.objects.filter(project=project,
                                       verbose_name=version_name).update(
                                           identifier=version_id,
                                           type=type,
                                           machine=False,
                                       )
                log.info("(Sync Versions) Updated Version: [%s=%s] " %
                         (version['verbose_name'], version['identifier']))
        else:
            # New Version
            slug = slugify_uniquely(Version,
                                    version['verbose_name'],
                                    'slug',
                                    255,
                                    project=project)
            Version.objects.create(
                project=project,
                slug=slug,
                type=type,
                identifier=version['identifier'],
                verbose_name=version['verbose_name'],
            )
            added.add(slug)
    if added:
        log.info("(Sync Versions) Added Versions: [%s] " % ' '.join(added))
    return added
Esempio n. 5
0
def sync_versions(project, versions, type):
    """
    Update the database with the current versions from the repository.
    """
    # Bookkeeping for keeping tag/branch identifies correct
    verbose_names = [v['verbose_name'] for v in versions]
    project.versions.filter(verbose_name__in=verbose_names).update(type=type)

    old_versions = {}
    old_version_values = project.versions.values('identifier', 'verbose_name')
    for version in old_version_values:
        old_versions[version['verbose_name']] = version['identifier']

    added = set()
    # Add new versions
    for version in versions:
        version_id = version['identifier']
        version_name = version['verbose_name']
        if version_name in old_versions.keys():
            if version_id == old_versions[version_name]:
                # Version is correct
                continue
            else:
                # Update slug with new identifier
                Version.objects.filter(
                    project=project, verbose_name=version_name
                ).update(
                    identifier=version_id,
                    type=type,
                    machine=False,
                )
                log.info("(Sync Versions) Updated Version: [%s=%s] " % (version['verbose_name'], version['identifier']))
        else:
            # New Version
            slug = slugify_uniquely(Version, version['verbose_name'], 'slug', 255, project=project)
            Version.objects.create(
                project=project,
                slug=slug,
                type=type,
                identifier=version['identifier'],
                verbose_name=version['verbose_name'],
            )
            added.add(slug)
    if added:
        log.info("(Sync Versions) Added Versions: [%s] " % ' '.join(added))
    return added
Esempio n. 6
0
    def _sync_versions(self, project, versions): 
        """
        Update the database with the current versions from the repository.
        """
        old_versions = project.versions.values_list('identifier', flat=True)

        # Add new versions
        for version in versions:
            if version['identifier'] in old_versions:
                continue
            slug = slugify_uniquely(Version, version['verbose_name'], 'slug', 255, project=project)
            Version.objects.create(
                    project=project,
                    slug=slug,
                    identifier=version['identifier'],
                    verbose_name=version['verbose_name'],
                )
Esempio n. 7
0
    def _sync_versions(self, project, versions):
        """
        Update the database with the current versions from the repository.
        """
        old_versions = project.versions.values_list('identifier', flat=True)

        # Add new versions
        for version in versions:
            if version['identifier'] in old_versions:
                continue
            slug = slugify_uniquely(Version,
                                    version['verbose_name'],
                                    'slug',
                                    255,
                                    project=project)
            Version.objects.create(
                project=project,
                slug=slug,
                identifier=version['identifier'],
                verbose_name=version['verbose_name'],
            )
Esempio n. 8
0
def update_imported_docs(version_pk):
    """
    Check out or update the given project's repository.
    """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    # Make Dirs
    if not os.path.exists(project.doc_path):
        os.makedirs(project.doc_path)

    with project.repo_lock(getattr(settings, "REPO_LOCK_SECONDS", 30)):
        update_docs_output = {}
        if not project.vcs_repo():
            raise ProjectImportError("Repo type '{repo_type}' unknown".format(repo_type=project.repo_type))

        # Get the actual code on disk
        if version:
            log.info(
                "Checking out version {slug}: {identifier}".format(slug=version.slug, identifier=version.identifier)
            )
            version_slug = version.slug
            version_repo = project.vcs_repo(version_slug)
            update_docs_output["checkout"] = version_repo.checkout(version.identifier)
        else:
            # Does this ever get called?
            log.info("Updating to latest revision")
            version_slug = "latest"
            version_repo = project.vcs_repo(version_slug)
            update_docs_output["checkout"] = version_repo.update()

        # Ensure we have a conf file (an exception is raised if not)
        project.conf_file(version.slug)

        # Do Virtualenv bits:
        if project.use_virtualenv:
            if project.use_system_packages:
                site_packages = "--system-site-packages"
            else:
                site_packages = "--no-site-packages"
            # Here the command has been modified to support different
            # interpreters.
            update_docs_output["venv"] = run(
                "{cmd} --distribute {site_packages} {path}".format(
                    cmd="virtualenv -p {interpreter}".format(interpreter=project.python_interpreter),
                    site_packages=site_packages,
                    path=project.venv_path(version=version_slug),
                )
            )
            # Other code expects sphinx-build to be installed inside the virtualenv.
            # Using the -I option makes sure it gets installed even if it is
            # already installed system-wide (and --system-site-packages is used)
            if project.use_system_packages:
                ignore_option = "-I"
            else:
                ignore_option = ""
            if project.python_interpreter != "python3":
                update_docs_output["sphinx"] = run(
                    "{cmd} install -U {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1".format(
                        cmd=project.venv_bin(version=version_slug, bin="pip"), ignore_option=ignore_option
                    )
                )
            else:
                # python 3 specific hax
                update_docs_output["sphinx"] = run(
                    "{cmd} install {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 docutils==0.8.1".format(
                        cmd=project.venv_bin(version=version_slug, bin="pip"), ignore_option=ignore_option
                    )
                )

            if project.requirements_file:
                os.chdir(project.checkout_path(version_slug))
                update_docs_output["requirements"] = run(
                    "{cmd} install -r {requirements}".format(
                        cmd=project.venv_bin(version=version_slug, bin="pip"), requirements=project.requirements_file
                    )
                )
            os.chdir(project.checkout_path(version_slug))
            if getattr(settings, "USE_PIP_INSTALL", False):
                update_docs_output["install"] = run(
                    "{cmd} install --ignore-installed .".format(cmd=project.venv_bin(version=version_slug, bin="pip"))
                )
            else:
                update_docs_output["install"] = run(
                    "{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version_slug, bin="python"))
                )

        # check tags/version
        # XXX:dc: what in this block raises the values error?
        try:
            old_versions = [
                obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=5000)["objects"]
            ]
            if version_repo.supports_tags:
                transaction.enter_transaction_management(True)
                tags = version_repo.tags
                for tag in tags:
                    if tag.identifier in old_versions:
                        continue
                    log.debug("NEW TAG: (%s not in %s)" % (tag.identifier, old_versions))
                    slug = slugify_uniquely(Version, tag.verbose_name, "slug", 255, project=project)
                    try:

                        version_data = api.version.post(
                            dict(
                                project="/api/v1/project/%s/" % project.pk,
                                slug=slug,
                                identifier=tag.identifier,
                                verbose_name=tag.verbose_name,
                            )
                        )
                        ver = make_api_version(version_data)
                        log.info("New tag found: {0}".format(tag.identifier))
                        ver, highest = project.highest_version[1]
                        ver_obj = mkversion(ver)
                        # TODO: Handle updating higher versions automatically.
                        # This never worked very well, anyways.
                        if highest and ver_obj and ver_obj > highest:
                            log.info("Highest version known, building docs")
                            update_docs.delay(ver.project.pk, version_pk=ver.pk)
                    except Exception, e:
                        log.error("Failed to create version (tag)", exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
            if version_repo.supports_branches:
                transaction.enter_transaction_management(True)
                branches = version_repo.branches
                for branch in branches:
                    if branch.identifier in old_versions:
                        continue
                    log.debug("NEW BRANCH: (%s not in %s)" % (branch, old_versions))
                    slug = slugify_uniquely(Version, branch.verbose_name, "slug", 255, project=project)
                    try:
                        api.version.post(
                            dict(
                                project="/api/v1/project/%s/" % project.pk,
                                slug=slug,
                                identifier=branch.identifier,
                                verbose_name=branch.verbose_name,
                            )
                        )
                        log.info("New branch found: {0}".format(branch.identifier))
                    except Exception, e:
                        log.error("Failed to create version (branch)", exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
Esempio n. 9
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    update_docs_output = {}
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '{repo_type}' unknown".format(
            repo_type=project.repo_type))

    if version:
        log.info('Checking out version {slug}: {identifier}'.format(
            slug=version.slug, identifier=version.identifier))
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.checkout(
            version.identifier)
    else:
        log.info('Updating to latest revision')
        version_slug = 'latest'
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.update()

    # Ensure we have a conf file (an exception is raised if not)
    conf_file = project.conf_file(version.slug)

    #Do Virtualenv bits:
    if project.use_virtualenv:
        update_docs_output['venv'] = run(
            '{cmd} --distribute --no-site-packages {path}'.format(
                cmd='virtualenv',
                path=project.venv_path(version=version_slug)))
        update_docs_output['sphinx'] = run(
            '{cmd} install -U hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1'
            .format(cmd=project.venv_bin(version=version_slug, bin='pip')))

        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            update_docs_output['requirements'] = run(
                '{cmd} install -r {requirements}'.format(
                    cmd=project.venv_bin(version=version_slug, bin='pip'),
                    requirements=project.requirements_file))
        os.chdir(project.checkout_path(version_slug))
        update_docs_output['install'] = run(
            '{cmd} setup.py install --force'.format(
                cmd=project.venv_bin(version=version_slug, bin='python')))

    # check tags/version
    #XXX:dc: what in this block raises the values error?
    try:
        old_versions = [
            obj['identifier'] for obj in api.version.get(
                project__slug=project.slug, limit=500)['objects']
        ]
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.tags
            for tag in tags:
                if tag.identifier in old_versions:
                    continue
                slug = slugify_uniquely(Version,
                                        tag.verbose_name,
                                        'slug',
                                        255,
                                        project=project)
                try:

                    version_data = api.version.post(
                        dict(project="/api/v1/project/%s/" % project.pk,
                             slug=slug,
                             identifier=tag.identifier,
                             verbose_name=tag.verbose_name))
                    del version_data['resource_uri']
                    project_data = version_data['project']
                    del project_data['users']
                    del project_data['resource_uri']
                    del project_data['absolute_url']
                    version_data['project'] = project
                    ver = Version(**version_data)
                    log.info("New tag found: {0}".format(tag.identifier))
                    ver, highest = project.highest_version[1]
                    ver_obj = mkversion(ver)
                    #TODO: Handle updating higher versions automatically.
                    #This never worked very well, anyways.
                    if highest and ver_obj and ver_obj > highest:
                        log.info("Highest verison known, building docs")
                        update_docs.delay(ver.project.pk, version_pk=ver.pk)
                except Exception, e:
                    log.error("Failed to create version (tag)", exc_info=True)
                    transaction.rollback()
                    #break here to stop updating tags when they will all fail.
                    break
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.branches
            for branch in branches:
                if branch.identifier in old_versions:
                    continue
                slug = slugify_uniquely(Version,
                                        branch.verbose_name,
                                        'slug',
                                        255,
                                        project=project)
                try:
                    api.version.post(
                        dict(project="/api/v1/project/%s/" % project.pk,
                             slug=slug,
                             identifier=branch.identifier,
                             verbose_name=branch.verbose_name))
                    log.info("New branch found: {0}".format(branch.identifier))
                except Exception, e:
                    log.error("Failed to create version (branch)",
                              exc_info=True)
                    transaction.rollback()
                    #break here to stop updating branches when they will all fail.
                    break
            transaction.leave_transaction_management()
Esempio n. 10
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    update_docs_output = {}
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '{repo_type}' unknown".format(
                repo_type=project.repo_type))

    if version:
        log.info('Checking out version {slug}: {identifier}'.format(
            slug=version.slug, identifier=version.identifier))
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.checkout(version.identifier)
    else:
        log.info('Updating to latest revision')
        version_slug = 'latest'
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.update()

    # Ensure we have a conf file (an exception is raised if not)
    conf_file = project.conf_file(version.slug)

    #Do Virtualenv bits:
    if project.use_virtualenv:
        update_docs_output['venv'] = run('{cmd} --distribute --never-download --no-site-packages {path}'.format(
                cmd='virtualenv',
                path=project.venv_path(version=version_slug)))
        update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format(
                cmd=project.venv_bin(version=version_slug, bin='pip')))

        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
                    cmd=project.venv_bin(version=version_slug, bin='pip'),
                    requirements=project.requirements_file))
        os.chdir(project.checkout_path(version_slug))
        update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
                cmd=project.venv_bin(version=version_slug, bin='python')))

    # check tags/version
    #XXX:dc: what in this block raises the values error?
    try:
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.tags
            old_tags = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects']]
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version, tag.verbose_name,
                                        'slug', 255, project=project)
                try:

                    api.version.post(dict(
                        project="/api/v1/project/%s/" % project.pk,
                        slug=slug,
                        identifier=tag.identifier,
                        verbose_name=tag.verbose_name
                    ))
                    log.info("New tag found: {0}".format(tag.identifier))
                    highest = project.highest_version['version']
                    ver_obj = mkversion(ver)
                    #TODO: Handle updating higher versions automatically.
                    #This never worked very well, anyways.
                    if highest and ver_obj and ver_obj > highest:
                        log.info("Highest verison known, building docs")
                        update_docs.delay(ver.project.pk, version_pk=ver.pk)
                except Exception, e:
                    log.error("Failed to create version (tag)", exc_info=True)
                    transaction.rollback()
                    #break
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.branches
            old_branches = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects']]
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version, branch.verbose_name,
                                        'slug', 255, project=project)
                try:
                    api.version.post(dict(
                        project="/api/v1/project/%s/" % project.pk,
                        slug=slug,
                        identifier=branch.identifier,
                        verbose_name=branch.verbose_name
                    ))
                    log.info("New branch found: {0}".format(branch.identifier))
                except Exception, e:
                    log.error("Failed to create version (branch)", exc_info=True)
                    transaction.rollback()
                    #break
            transaction.leave_transaction_management()
Esempio n. 11
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    update_docs_output = {}
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '{repo_type}' unknown".format(
                repo_type=project.repo_type))

    if version:
        print 'Checking out version {slug}: {identifier}'.format(
            slug=version.slug, identifier=version.identifier)
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.checkout(version.identifier)
    else:
        print 'Updating to latest revision'
        version_slug = 'latest'
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.update()

    #Break early without a conf file.
    if not project.conf_file(version.slug):
        raise ProjectImportError("Conf File Missing. Skipping.")

    #Do Virtualenv bits:
    if project.use_virtualenv and project.whitelisted:
        update_docs_output['venv'] = run('{cmd} --no-site-packages {path}'.format(
                cmd='virtualenv',
                path=project.venv_path(version=version_slug)))
        update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format(
                cmd=project.venv_bin(version=version_slug, bin='pip')))

        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
                    cmd=project.venv_bin(version=version_slug, bin='pip'),
                    requirements=project.requirements_file))
        os.chdir(project.checkout_path(version_slug))
        update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
                cmd=project.venv_bin(version=version_slug, bin='python')))

    # check tags/version
    #XXX:dc: what in this block raises the values error?
    try:
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.tags
            old_tags = Version.objects.filter(
                project=project).values_list('identifier', flat=True)
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version, tag.verbose_name,
                                        'slug', 255, project=project)
                try:
                    ver, created = Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=tag.identifier,
                        verbose_name=tag.verbose_name
                    )
                    print "New tag found: %s" % ver
                    highest = project.highest_version['version']
                    ver_obj = mkversion(ver)
                    if highest and ver_obj and ver_obj > highest:
                        print "Highest verison known, building docs"
                        update_docs.delay(ver.project.pk, version_pk=ver.pk)
                except Exception, e:
                    print "Failed to create version (tag): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.branches
            old_branches = Version.objects.filter(
                project=project).values_list('identifier', flat=True)
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version, branch.verbose_name,
                                        'slug', 255, project=project)
                try:
                    ver, created = Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=branch.identifier,
                        verbose_name=branch.verbose_name
                    )
                    print "New branch found: %s" % ver
                except Exception, e:
                    print "Failed to create version (branch): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
            #Kill deleted branches
            Version.objects.filter(
                project=project).exclude(identifier__in=old_branches).delete()
Esempio n. 12
0
def update_imported_docs(version_pk):
    """
    Check out or update the given project's repository.
    """
    version_data = api.version(version_pk).get()
    version = make_api_version(version_data)
    project = version.project

    # Make Dirs
    if not os.path.exists(project.doc_path):
        os.makedirs(project.doc_path)

    with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)):
        update_docs_output = {}
        if not project.vcs_repo():
            raise ProjectImportError(
                ("Repo type '{0}' unknown".format(project.repo_type)))

        # Get the actual code on disk
        if version:
            log.info('Checking out version {slug}: {identifier}'.format(
                slug=version.slug, identifier=version.identifier))
            version_slug = version.slug
            version_repo = project.vcs_repo(version_slug)
            update_docs_output['checkout'] = version_repo.checkout(
                version.identifier)
        else:
            # Does this ever get called?
            log.info('Updating to latest revision')
            version_slug = 'latest'
            version_repo = project.vcs_repo(version_slug)
            update_docs_output['checkout'] = version_repo.update()

        # Ensure we have a conf file (an exception is raised if not)
        project.conf_file(version.slug)

        # Do Virtualenv bits:
        if project.use_virtualenv:
            if project.use_system_packages:
                site_packages = '--system-site-packages'
            else:
                site_packages = '--no-site-packages'
            # Here the command has been modified to support different
            # interpreters.
            update_docs_output['venv'] = run(
                '{cmd} --distribute {site_packages} {path}'.format(
                    cmd='virtualenv -p {interpreter}'.format(
                        interpreter=project.python_interpreter),
                    site_packages=site_packages,
                    path=project.venv_path(version=version_slug)))
            # Other code expects sphinx-build to be installed inside the
            # virtualenv.  Using the -I option makes sure it gets installed
            # even if it is already installed system-wide (and
            # --system-site-packages is used)
            if project.use_system_packages:
                ignore_option = '-I'
            else:
                ignore_option = ''
            sphinx = ('hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c'
                      '#egg=Sphinx')
            if project.python_interpreter != 'python3':
                update_docs_output['sphinx'] = run(
                    ('{cmd} install -U {ignore_option} {sphinx} '
                     'virtualenv==1.8.2 distribute==0.6.28 '
                     'docutils==0.8.1').format(cmd=project.venv_bin(
                         version=version_slug, bin='pip'),
                                               sphinx=sphinx,
                                               ignore_option=ignore_option))
            else:
                # python 3 specific hax
                update_docs_output['sphinx'] = run((
                    '{cmd} install {ignore_option} {sphinx} virtualenv==1.8.2 '
                    'docutils==0.8.1').format(cmd=project.venv_bin(
                        version=version_slug, bin='pip'),
                                              sphinx=sphinx,
                                              ignore_option=ignore_option))

            if project.requirements_file:
                os.chdir(project.checkout_path(version_slug))
                update_docs_output['requirements'] = run(
                    '{cmd} install --exists-action=w -r {requirements}'.format(
                        cmd=project.venv_bin(version=version_slug, bin='pip'),
                        requirements=project.requirements_file))
            os.chdir(project.checkout_path(version_slug))
            if getattr(settings, 'USE_PIP_INSTALL', False):
                update_docs_output['install'] = run(
                    '{cmd} install --ignore-installed .'.format(
                        cmd=project.venv_bin(version=version_slug, bin='pip')))
            else:
                update_docs_output['install'] = run(
                    '{cmd} setup.py install --force'.format(
                        cmd=project.venv_bin(version=version_slug,
                                             bin='python')))

        # check tags/version
        #XXX:dc: what in this block raises the values error?
        try:
            old_versions = [
                obj['identifier']
                for obj in api.version.get(project__slug=project.slug,
                                           limit=5000)['objects']
            ]
            if version_repo.supports_tags:
                transaction.enter_transaction_management(True)
                tags = version_repo.tags
                for tag in tags:
                    if tag.identifier in old_versions:
                        continue
                    log.debug('NEW TAG: (%s not in %s)' %
                              (tag.identifier, old_versions))
                    slug = slugify_uniquely(Version,
                                            tag.verbose_name,
                                            'slug',
                                            255,
                                            project=project)
                    try:

                        version_data = api.version.post(
                            dict(project="/api/v1/project/%s/" % project.pk,
                                 slug=slug,
                                 identifier=tag.identifier,
                                 verbose_name=tag.verbose_name))
                        ver = make_api_version(version_data)
                        log.info("New tag found: {0}".format(tag.identifier))
                        ver, highest = project.highest_version[1]
                        ver_obj = mkversion(ver)
                        # TODO: Handle updating higher versions automatically.
                        # This never worked very well, anyways.
                        if highest and ver_obj and ver_obj > highest:
                            log.info("Highest version known, building docs")
                            update_docs.delay(ver.project.pk,
                                              version_pk=ver.pk)
                    except Exception:
                        log.error("Failed to create version (tag)",
                                  exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
            if version_repo.supports_branches:
                transaction.enter_transaction_management(True)
                branches = version_repo.branches
                for branch in branches:
                    if branch.identifier in old_versions:
                        continue
                    log.debug('NEW BRANCH: (%s not in %s)' %
                              (branch, old_versions))
                    slug = slugify_uniquely(Version,
                                            branch.verbose_name,
                                            'slug',
                                            255,
                                            project=project)
                    try:
                        api.version.post(
                            dict(project="/api/v1/project/%s/" % project.pk,
                                 slug=slug,
                                 identifier=branch.identifier,
                                 verbose_name=branch.verbose_name))
                        log.info(("New branch found: {0}".format(
                            branch.identifier)))
                    except Exception:
                        log.error("Failed to create version (branch)",
                                  exc_info=True)
                        transaction.rollback()
                transaction.leave_transaction_management()
                #TODO: Kill deleted branches
        except ValueError:
            log.error("Error getting tags", exc_info=True)
    return update_docs_output
Esempio n. 13
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
    if version:
        print 'Checking out version %s: %s' % (version.slug, version.identifier)
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        version_repo.checkout(version.identifier)
    else:
        print 'Updating to latest revision'
        version_slug = 'latest'
        version_repo = project.vcs_repo(version_slug)
        version_repo.update()

    #Break early without a conf file.
    if not project.conf_file(version.slug):
        print "Conf File Missing. Skipping."
        return -1

    #Do Virtualenv bits:
    if project.use_virtualenv and project.whitelisted:
        run('virtualenv --no-site-packages %s' % project.venv_path(version=version_slug))
        run('%s install sphinx' % project.venv_bin(version=version_slug,
                                                      bin='pip'))
        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            run('%s install -r %s' % (project.venv_bin(version=version_slug, bin='pip'),
                                    project.requirements_file))
        os.chdir(project.checkout_path(version_slug))
        run('%s setup.py install --force' % project.venv_bin(version=version_slug,
                                                          bin='python'))

    # check tags/version
    try:
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.get_tags()
            old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True)
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project)
                try:
                    ver, created = Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=tag.identifier,
                        verbose_name=tag.verbose_name
                    )
                    print "New tag found: %s" % ver
                    highest = project.highest_version[1]
                    ver_obj = mkversion(ver)
                    if highest and ver_obj and ver_obj > highest:
                        print "Highest verison known, building docs"
                        update_docs.delay(ver.project.pk, version_pk=ver.pk)
                except Exception, e:
                    print "Failed to create version (tag): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.get_branches()
            old_branches = Version.objects.filter(project=project).values_list('identifier', flat=True)
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project)
                try:
                    Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=branch.identifier,
                        verbose_name=branch.verbose_name
                    )
                    print "New branch found: %s" % ver
                except Exception, e:
                    print "Failed to create version (branch): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
            #Kill deleted branches
            Version.objects.filter(project=project).exclude(identifier__in=old_branches).delete()
Esempio n. 14
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
    if version:
        print 'Checking out version %s: %s' % (version.slug,
                                               version.identifier)
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        version_repo.checkout(version.identifier)
    else:
        print 'Updating to latest revision'
        version_slug = 'latest'
        version_repo = project.vcs_repo(version_slug)
        version_repo.update()

    #Break early without a conf file.
    if not project.conf_file(version.slug):
        print "Conf File Missing. Skipping."
        return -1

    #Do Virtualenv bits:
    if project.use_virtualenv and project.whitelisted:
        run('virtualenv --no-site-packages %s' %
            project.venv_path(version=version_slug))
        run('%s install sphinx' %
            project.venv_bin(version=version_slug, bin='pip'))
        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            run('%s install -r %s' % (project.venv_bin(
                version=version_slug, bin='pip'), project.requirements_file))
        os.chdir(project.checkout_path(version_slug))
        run('%s setup.py install --force' %
            project.venv_bin(version=version_slug, bin='python'))

    # check tags/version
    try:
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.get_tags()
            old_tags = Version.objects.filter(project=project).values_list(
                'identifier', flat=True)
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version,
                                        tag.verbose_name,
                                        'slug',
                                        255,
                                        project=project)
                try:
                    Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=tag.identifier,
                        verbose_name=tag.verbose_name)
                except Exception, e:
                    print "Failed to create version (tag): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.get_branches()
            old_branches = Version.objects.filter(project=project).values_list(
                'identifier', flat=True)
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version,
                                        branch.verbose_name,
                                        'slug',
                                        255,
                                        project=project)
                try:
                    Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=branch.identifier,
                        verbose_name=branch.verbose_name)
                except Exception, e:
                    print "Failed to create version (branch): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
            #Kill deleted branches
            Version.objects.filter(project=project).exclude(
                identifier__in=old_branches).delete()
Esempio n. 15
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    update_docs_output = {}
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '{repo_type}' unknown".format(
                repo_type=project.repo_type))

    if version:
        print 'Checking out version {slug}: {identifier}'.format(
            slug=version.slug, identifier=version.identifier)
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.checkout(version.identifier)
    else:
        print 'Updating to latest revision'
        version_slug = 'latest'
        version_repo = project.vcs_repo(version_slug)
        update_docs_output['checkout'] = version_repo.update()

    #Break early without a conf file.
    if not project.conf_file(version.slug):
        raise ProjectImportError("Conf File Missing. Skipping.")

    #Do Virtualenv bits:
    if project.use_virtualenv and project.whitelisted:
        update_docs_output['venv'] = run('{cmd} --no-site-packages {path}'.format(
                cmd='virtualenv',
                path=project.venv_path(version=version_slug)))
        update_docs_output['sphinx'] = run('{cmd} install sphinx'.format(
                cmd=project.venv_bin(version=version_slug, bin='pip')))

        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
                    cmd=project.venv_bin(version=version_slug, bin='pip'),
                    requirements=project.requirements_file))
        os.chdir(project.checkout_path(version_slug))
        update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
                cmd=project.venv_bin(version=version_slug, bin='python')))

    # check tags/version
    #XXX:dc: what in this block raises the values error?
    try:
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.tags
            old_tags = Version.objects.filter(
                project=project).values_list('identifier', flat=True)
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version, tag.verbose_name,
                                        'slug', 255, project=project)
                try:
                    ver, created = Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=tag.identifier,
                        verbose_name=tag.verbose_name
                    )
                    print "New tag found: %s" % ver
                    highest = project.highest_version['version']
                    ver_obj = mkversion(ver)
                    if highest and ver_obj and ver_obj > highest:
                        print "Highest verison known, building docs"
                        update_docs.delay(ver.project.pk, version_pk=ver.pk)
                except Exception, e:
                    print "Failed to create version (tag): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.branches
            old_branches = Version.objects.filter(
                project=project).values_list('identifier', flat=True)
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version, branch.verbose_name,
                                        'slug', 255, project=project)
                try:
                    ver, created = Version.objects.get_or_create(
                        project=project,
                        slug=slug,
                        identifier=branch.identifier,
                        verbose_name=branch.verbose_name
                    )
                    print "New branch found: %s" % ver
                except Exception, e:
                    print "Failed to create version (branch): %s" % e
                    transaction.rollback()
            transaction.leave_transaction_management()
            #Kill deleted branches
            Version.objects.filter(
                project=project).exclude(identifier__in=old_branches).delete()
Esempio n. 16
0
def update_imported_docs(project, version):
    """
    Check out or update the given project's repository.
    """
    update_docs_output = {}
    if not project.vcs_repo():
        raise ProjectImportError("Repo type '{repo_type}' unknown".format(repo_type=project.repo_type))

    if version:
        print "Checking out version {slug}: {identifier}".format(slug=version.slug, identifier=version.identifier)
        version_slug = version.slug
        version_repo = project.vcs_repo(version_slug)
        update_docs_output["checkout"] = version_repo.checkout(version.identifier)
    else:
        print "Updating to latest revision"
        version_slug = "latest"
        version_repo = project.vcs_repo(version_slug)
        update_docs_output["checkout"] = version_repo.update()

    # Break early without a conf file.
    if not project.conf_file(version.slug):
        raise ProjectImportError("Conf File Missing. Skipping.")

    # Do Virtualenv bits:
    if project.use_virtualenv:
        update_docs_output["venv"] = run(
            "{cmd} --distribute --never-download --no-site-packages {path}".format(
                cmd="virtualenv", path=project.venv_path(version=version_slug)
            )
        )
        update_docs_output["sphinx"] = run(
            "{cmd} install -U sphinx".format(cmd=project.venv_bin(version=version_slug, bin="pip"))
        )

        if project.requirements_file:
            os.chdir(project.checkout_path(version_slug))
            update_docs_output["requirements"] = run(
                "{cmd} install -r {requirements}".format(
                    cmd=project.venv_bin(version=version_slug, bin="pip"), requirements=project.requirements_file
                )
            )
        os.chdir(project.checkout_path(version_slug))
        update_docs_output["install"] = run(
            "{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version_slug, bin="python"))
        )

    # check tags/version
    # XXX:dc: what in this block raises the values error?
    try:
        if version_repo.supports_tags:
            transaction.enter_transaction_management(True)
            tags = version_repo.tags
            old_tags = [obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=50)["objects"]]
            for tag in tags:
                if tag.identifier in old_tags:
                    continue
                slug = slugify_uniquely(Version, tag.verbose_name, "slug", 255, project=project)
                try:

                    api.version.post(
                        dict(
                            project="/api/v1/project/%s/" % project.pk,
                            slug=slug,
                            identifier=tag.identifier,
                            verbose_name=tag.verbose_name,
                        )
                    )
                    print "New tag found: %s" % tag.identifier
                    highest = project.highest_version["version"]
                    ver_obj = mkversion(ver)
                    # TODO: Handle updating higher versions automatically.
                    # This never worked very well, anyways.
                    if highest and ver_obj and ver_obj > highest:
                        print "Highest verison known, building docs"
                        update_docs.delay(ver.project.pk, version_pk=ver.pk)
                except Exception, e:
                    print "Failed to create version (tag): %s" % e
                    transaction.rollback()
                    # break
            transaction.leave_transaction_management()
        if version_repo.supports_branches:
            transaction.enter_transaction_management(True)
            branches = version_repo.branches
            old_branches = [
                obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=50)["objects"]
            ]
            for branch in branches:
                if branch.identifier in old_branches:
                    continue
                slug = slugify_uniquely(Version, branch.verbose_name, "slug", 255, project=project)
                try:
                    api.version.post(
                        dict(
                            project="/api/v1/project/%s/" % project.pk,
                            slug=slug,
                            identifier=branch.identifier,
                            verbose_name=branch.verbose_name,
                        )
                    )
                    print "New branch found: %s" % branch.identifier
                except Exception, e:
                    print "Failed to create version (branch): %s" % e
                    transaction.rollback()
                    # break
            transaction.leave_transaction_management()