def version_compare(self, request, **kwargs): project = get_object_or_404(Project, slug=kwargs['project_slug']) highest = highest_version(project.versions.filter(active=True)) base = kwargs.get('base', None) ret_val = { 'project': highest[0], 'version': highest[1], 'is_highest': True, } if highest[0]: ret_val['url'] = highest[0].get_absolute_url() ret_val['slug'] = highest[0].slug, if base and base != 'latest': try: ver_obj = project.versions.get(slug=base) base_ver = mkversion(ver_obj) if base_ver: # This is only place where is_highest can get set. All # error cases will be set to True, for non- standard # versions. ret_val['is_highest'] = base_ver >= highest[1] else: ret_val['is_highest'] = True except (Version.DoesNotExist, TypeError): ret_val['is_highest'] = True return self.create_response(request, ret_val)
def version_compare(self, request, **kwargs): project = get_object_or_404(Project, slug=kwargs['project_slug']) highest = highest_version(project.versions.filter(active=True)) base = kwargs.get('base', None) ret_val = { 'project': highest[0], 'version': highest[1], 'is_highest': True, } if highest[0]: ret_val['url'] = highest[0].get_absolute_url() ret_val['slug'] = highest[0].slug, if base and base != 'latest': try: ver_obj = project.versions.get(slug=base) base_ver = mkversion(ver_obj) if base_ver: #This is only place where is_highest can get set. #All error cases will be set to True, for non- #standard versions. ret_val['is_highest'] = base_ver >= highest[1] else: ret_val['is_highest'] = True except (Version.DoesNotExist, TypeError) as e: ret_val['is_highest'] = True return self.create_response(request, ret_val)
def sort_version_aware(versions): """ Takes a list of versions objects and sort them caring about version schemes """ from distutils2.version import NormalizedVersion from projects.utils import mkversion fallback = NormalizedVersion('99999999.0', error_on_huge_major_num=False) return sorted(versions, key=lambda v:(mkversion(v) or fallback), reverse=True)
def make_version(version): ver = mkversion(version) if not ver: if version.slug == 'latest': return NormalizedVersion('99999.0', error_on_huge_major_num=False) elif version.slug == 'stable': return NormalizedVersion('9999.0', error_on_huge_major_num=False) else: return NormalizedVersion('999.0', error_on_huge_major_num=False) return ver
def make_version(version): ver = mkversion(version) if not ver: if version.slug == LATEST: return NormalizedVersion('99999.0', error_on_huge_major_num=False) elif version.slug == STABLE: return NormalizedVersion('9999.0', error_on_huge_major_num=False) else: return NormalizedVersion('999.0', error_on_huge_major_num=False) return ver
def version_compare(self, request, **kwargs): project = get_object_or_404(Project, slug=kwargs["project_slug"]) highest = highest_version(project.versions.filter(active=True)) base = kwargs.get("base", None) ret_val = {"project": highest[0], "version": highest[1]} if highest[0]: ret_val["url"] = highest[0].get_absolute_url() if base: try: ver_obj = project.versions.get(slug=base) base_ver = mkversion(ver_obj) if base_ver: if base == "latest": ret_val["is_highest"] = True else: ret_val["is_highest"] = base_ver >= highest[1] else: ret_val["is_highest"] = False except Version.DoesNotExist: ret_val["is_highest"] = False return self.create_response(request, ret_val)
def version_compare(self, request, **kwargs): project = get_object_or_404(Project, slug=kwargs["project_slug"]) highest = highest_version(project.versions.filter(active=True)) base = kwargs.get("base", None) ret_val = {"project": highest[0], "version": highest[1], "is_highest": True} if highest[0]: ret_val["url"] = highest[0].get_absolute_url() ret_val["slug"] = (highest[0].slug,) if base and base != "latest": try: ver_obj = project.versions.get(slug=base) base_ver = mkversion(ver_obj) if base_ver: # This is only place where is_highest can get set. # All error cases will be set to True, for non- # standard versions. ret_val["is_highest"] = base_ver >= highest[1] else: ret_val["is_highest"] = True except (Version.DoesNotExist, TypeError) as e: ret_val["is_highest"] = True return self.create_response(request, ret_val)
def version_compare(self, request, **kwargs): project = get_object_or_404(Project, slug=kwargs['project_slug']) highest = highest_version(project.versions.filter(active=True)) base = kwargs.get('base', None) ret_val = { 'project': highest[0], 'version': highest[1], } if highest[0]: ret_val['url'] = highest[0].get_absolute_url() if base: try: ver_obj = project.versions.get(slug=base) base_ver = mkversion(ver_obj) if base_ver: if base == "latest": ret_val['is_highest'] = True else: ret_val['is_highest'] = base_ver >= highest[1] else: ret_val['is_highest'] = False except Version.DoesNotExist: ret_val['is_highest'] = False return self.create_response(request, ret_val)
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format( repo_type=project.repo_type)) if version: log.info('Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier)) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout( version.identifier) else: log.info('Updating to latest revision') version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) conf_file = project.conf_file(version.slug) #Do Virtualenv bits: if project.use_virtualenv: update_docs_output['venv'] = run( '{cmd} --distribute --no-site-packages {path}'.format( cmd='virtualenv', path=project.venv_path(version=version_slug))) update_docs_output['sphinx'] = run( '{cmd} install -U hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1' .format(cmd=project.venv_bin(version=version_slug, bin='pip'))) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run( '{cmd} install -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) update_docs_output['install'] = run( '{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: old_versions = [ obj['identifier'] for obj in api.version.get( project__slug=project.slug, limit=500)['objects'] ] if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags for tag in tags: if tag.identifier in old_versions: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: version_data = api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name)) del version_data['resource_uri'] project_data = version_data['project'] del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] version_data['project'] = project ver = Version(**version_data) log.info("New tag found: {0}".format(tag.identifier)) ver, highest = project.highest_version[1] ver_obj = mkversion(ver) #TODO: Handle updating higher versions automatically. #This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: log.info("Highest verison known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() #break here to stop updating tags when they will all fail. break transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches for branch in branches: if branch.identifier in old_versions: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name)) log.info("New branch found: {0}".format(branch.identifier)) except Exception, e: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() #break here to stop updating branches when they will all fail. break transaction.leave_transaction_management()
def update_imported_docs(version_pk): """ Check out or update the given project's repository. """ version_data = api.version(version_pk).get() version = make_api_version(version_data) project = version.project # Make Dirs if not os.path.exists(project.doc_path): os.makedirs(project.doc_path) with project.repo_lock(getattr(settings, "REPO_LOCK_SECONDS", 30)): update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format(repo_type=project.repo_type)) # Get the actual code on disk if version: log.info( "Checking out version {slug}: {identifier}".format(slug=version.slug, identifier=version.identifier) ) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output["checkout"] = version_repo.checkout(version.identifier) else: # Does this ever get called? log.info("Updating to latest revision") version_slug = "latest" version_repo = project.vcs_repo(version_slug) update_docs_output["checkout"] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) project.conf_file(version.slug) # Do Virtualenv bits: if project.use_virtualenv: if project.use_system_packages: site_packages = "--system-site-packages" else: site_packages = "--no-site-packages" # Here the command has been modified to support different # interpreters. update_docs_output["venv"] = run( "{cmd} --distribute {site_packages} {path}".format( cmd="virtualenv -p {interpreter}".format(interpreter=project.python_interpreter), site_packages=site_packages, path=project.venv_path(version=version_slug), ) ) # Other code expects sphinx-build to be installed inside the virtualenv. # Using the -I option makes sure it gets installed even if it is # already installed system-wide (and --system-site-packages is used) if project.use_system_packages: ignore_option = "-I" else: ignore_option = "" if project.python_interpreter != "python3": update_docs_output["sphinx"] = run( "{cmd} install -U {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1".format( cmd=project.venv_bin(version=version_slug, bin="pip"), ignore_option=ignore_option ) ) else: # python 3 specific hax update_docs_output["sphinx"] = run( "{cmd} install {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 docutils==0.8.1".format( cmd=project.venv_bin(version=version_slug, bin="pip"), ignore_option=ignore_option ) ) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output["requirements"] = run( "{cmd} install -r {requirements}".format( cmd=project.venv_bin(version=version_slug, bin="pip"), requirements=project.requirements_file ) ) os.chdir(project.checkout_path(version_slug)) if getattr(settings, "USE_PIP_INSTALL", False): update_docs_output["install"] = run( "{cmd} install --ignore-installed .".format(cmd=project.venv_bin(version=version_slug, bin="pip")) ) else: update_docs_output["install"] = run( "{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version_slug, bin="python")) ) # check tags/version # XXX:dc: what in this block raises the values error? try: old_versions = [ obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=5000)["objects"] ] if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags for tag in tags: if tag.identifier in old_versions: continue log.debug("NEW TAG: (%s not in %s)" % (tag.identifier, old_versions)) slug = slugify_uniquely(Version, tag.verbose_name, "slug", 255, project=project) try: version_data = api.version.post( dict( project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name, ) ) ver = make_api_version(version_data) log.info("New tag found: {0}".format(tag.identifier)) ver, highest = project.highest_version[1] ver_obj = mkversion(ver) # TODO: Handle updating higher versions automatically. # This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: log.info("Highest version known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches for branch in branches: if branch.identifier in old_versions: continue log.debug("NEW BRANCH: (%s not in %s)" % (branch, old_versions)) slug = slugify_uniquely(Version, branch.verbose_name, "slug", 255, project=project) try: api.version.post( dict( project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name, ) ) log.info("New branch found: {0}".format(branch.identifier)) except Exception, e: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() transaction.leave_transaction_management()
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format( repo_type=project.repo_type)) if version: print 'Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout(version.identifier) else: print 'Updating to latest revision' version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() #Break early without a conf file. if not project.conf_file(version.slug): raise ProjectImportError("Conf File Missing. Skipping.") #Do Virtualenv bits: if project.use_virtualenv and project.whitelisted: update_docs_output['venv'] = run('{cmd} --no-site-packages {path}'.format( cmd='virtualenv', path=project.venv_path(version=version_slug))) update_docs_output['sphinx'] = run('{cmd} install sphinx'.format( cmd=project.venv_bin(version=version_slug, bin='pip'))) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) update_docs_output['install'] = run('{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags old_tags = Version.objects.filter( project=project).values_list('identifier', flat=True) for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: ver, created = Version.objects.get_or_create( project=project, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name ) print "New tag found: %s" % ver highest = project.highest_version['version'] ver_obj = mkversion(ver) if highest and ver_obj and ver_obj > highest: print "Highest verison known, building docs" update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: print "Failed to create version (tag): %s" % e transaction.rollback() transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches old_branches = Version.objects.filter( project=project).values_list('identifier', flat=True) for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: ver, created = Version.objects.get_or_create( project=project, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name ) print "New branch found: %s" % ver except Exception, e: print "Failed to create version (branch): %s" % e transaction.rollback() transaction.leave_transaction_management() #Kill deleted branches Version.objects.filter( project=project).exclude(identifier__in=old_branches).delete()
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format(repo_type=project.repo_type)) if version: print "Checking out version {slug}: {identifier}".format(slug=version.slug, identifier=version.identifier) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output["checkout"] = version_repo.checkout(version.identifier) else: print "Updating to latest revision" version_slug = "latest" version_repo = project.vcs_repo(version_slug) update_docs_output["checkout"] = version_repo.update() # Break early without a conf file. if not project.conf_file(version.slug): raise ProjectImportError("Conf File Missing. Skipping.") # Do Virtualenv bits: if project.use_virtualenv: update_docs_output["venv"] = run( "{cmd} --distribute --never-download --no-site-packages {path}".format( cmd="virtualenv", path=project.venv_path(version=version_slug) ) ) update_docs_output["sphinx"] = run( "{cmd} install -U sphinx".format(cmd=project.venv_bin(version=version_slug, bin="pip")) ) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output["requirements"] = run( "{cmd} install -r {requirements}".format( cmd=project.venv_bin(version=version_slug, bin="pip"), requirements=project.requirements_file ) ) os.chdir(project.checkout_path(version_slug)) update_docs_output["install"] = run( "{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version_slug, bin="python")) ) # check tags/version # XXX:dc: what in this block raises the values error? try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags old_tags = [obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=50)["objects"]] for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, "slug", 255, project=project) try: api.version.post( dict( project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name, ) ) print "New tag found: %s" % tag.identifier highest = project.highest_version["version"] ver_obj = mkversion(ver) # TODO: Handle updating higher versions automatically. # This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: print "Highest verison known, building docs" update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: print "Failed to create version (tag): %s" % e transaction.rollback() # break transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches old_branches = [ obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=50)["objects"] ] for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, "slug", 255, project=project) try: api.version.post( dict( project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name, ) ) print "New branch found: %s" % branch.identifier except Exception, e: print "Failed to create version (branch): %s" % e transaction.rollback() # break transaction.leave_transaction_management()
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format( repo_type=project.repo_type)) if version: log.info('Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier)) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout(version.identifier) else: log.info('Updating to latest revision') version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) conf_file = project.conf_file(version.slug) #Do Virtualenv bits: if project.use_virtualenv: update_docs_output['venv'] = run('{cmd} --distribute --never-download --no-site-packages {path}'.format( cmd='virtualenv', path=project.venv_path(version=version_slug))) update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format( cmd=project.venv_bin(version=version_slug, bin='pip'))) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) update_docs_output['install'] = run('{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags old_tags = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects']] for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: api.version.post(dict( project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name )) log.info("New tag found: {0}".format(tag.identifier)) highest = project.highest_version['version'] ver_obj = mkversion(ver) #TODO: Handle updating higher versions automatically. #This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: log.info("Highest verison known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() #break transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches old_branches = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects']] for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: api.version.post(dict( project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name )) log.info("New branch found: {0}".format(branch.identifier)) except Exception, e: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() #break transaction.leave_transaction_management()
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ if not project.vcs_repo(): raise ProjectImportError("Repo type '%s' unknown" % project.repo_type) if version: print 'Checking out version %s: %s' % (version.slug, version.identifier) version_slug = version.slug version_repo = project.vcs_repo(version_slug) version_repo.checkout(version.identifier) else: print 'Updating to latest revision' version_slug = 'latest' version_repo = project.vcs_repo(version_slug) version_repo.update() #Break early without a conf file. if not project.conf_file(version.slug): print "Conf File Missing. Skipping." return -1 #Do Virtualenv bits: if project.use_virtualenv and project.whitelisted: run('virtualenv --no-site-packages %s' % project.venv_path(version=version_slug)) run('%s install sphinx' % project.venv_bin(version=version_slug, bin='pip')) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) run('%s install -r %s' % (project.venv_bin( version=version_slug, bin='pip'), project.requirements_file)) os.chdir(project.checkout_path(version_slug)) run('%s setup.py install --force' % project.venv_bin(version=version_slug, bin='python')) # check tags/version try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.get_tags() old_tags = Version.objects.filter(project=project).values_list( 'identifier', flat=True) for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: ver, created = Version.objects.get_or_create( project=project, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name) print "New tag found: %s" % ver highest = project.highest_version[1] ver_obj = mkversion(ver) if highest and ver_obj and ver_obj > highest: print "Highest verison known, building docs" update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: print "Failed to create version (tag): %s" % e transaction.rollback() transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.get_branches() old_branches = Version.objects.filter(project=project).values_list( 'identifier', flat=True) for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: Version.objects.get_or_create( project=project, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name) print "New branch found: %s" % ver except Exception, e: print "Failed to create version (branch): %s" % e transaction.rollback() transaction.leave_transaction_management() #Kill deleted branches Version.objects.filter(project=project).exclude( identifier__in=old_branches).delete()
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format( repo_type=project.repo_type)) if version: print 'Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout(version.identifier) else: print 'Updating to latest revision' version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() #Break early without a conf file. if not project.conf_file(version.slug): raise ProjectImportError("Conf File Missing. Skipping.") #Do Virtualenv bits: if project.use_virtualenv and project.whitelisted: update_docs_output['venv'] = run('{cmd} --no-site-packages {path}'.format( cmd='virtualenv', path=project.venv_path(version=version_slug))) update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format( cmd=project.venv_bin(version=version_slug, bin='pip'))) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) update_docs_output['install'] = run('{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags old_tags = Version.objects.filter( project=project).values_list('identifier', flat=True) for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: ver, created = Version.objects.get_or_create( project=project, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name ) print "New tag found: %s" % ver highest = project.highest_version['version'] ver_obj = mkversion(ver) if highest and ver_obj and ver_obj > highest: print "Highest verison known, building docs" update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: print "Failed to create version (tag): %s" % e transaction.rollback() transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches old_branches = Version.objects.filter( project=project).values_list('identifier', flat=True) for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: ver, created = Version.objects.get_or_create( project=project, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name ) print "New branch found: %s" % ver except Exception, e: print "Failed to create version (branch): %s" % e transaction.rollback() transaction.leave_transaction_management() #Kill deleted branches Version.objects.filter( project=project).exclude(identifier__in=old_branches).delete()
def update_imported_docs(version_pk): """ Check out or update the given project's repository. """ version_data = api.version(version_pk).get() version = make_api_version(version_data) project = version.project # Make Dirs if not os.path.exists(project.doc_path): os.makedirs(project.doc_path) with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)): update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError( ("Repo type '{0}' unknown".format(project.repo_type))) # Get the actual code on disk if version: log.info('Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier)) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout( version.identifier) else: # Does this ever get called? log.info('Updating to latest revision') version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) project.conf_file(version.slug) # Do Virtualenv bits: if project.use_virtualenv: if project.use_system_packages: site_packages = '--system-site-packages' else: site_packages = '--no-site-packages' # Here the command has been modified to support different # interpreters. update_docs_output['venv'] = run( '{cmd} --distribute {site_packages} {path}'.format( cmd='virtualenv -p {interpreter}'.format( interpreter=project.python_interpreter), site_packages=site_packages, path=project.venv_path(version=version_slug))) # Other code expects sphinx-build to be installed inside the # virtualenv. Using the -I option makes sure it gets installed # even if it is already installed system-wide (and # --system-site-packages is used) if project.use_system_packages: ignore_option = '-I' else: ignore_option = '' sphinx = ('hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c' '#egg=Sphinx') if project.python_interpreter != 'python3': update_docs_output['sphinx'] = run( ('{cmd} install -U {ignore_option} {sphinx} ' 'virtualenv==1.8.2 distribute==0.6.28 ' 'docutils==0.8.1').format(cmd=project.venv_bin( version=version_slug, bin='pip'), sphinx=sphinx, ignore_option=ignore_option)) else: # python 3 specific hax update_docs_output['sphinx'] = run(( '{cmd} install {ignore_option} {sphinx} virtualenv==1.8.2 ' 'docutils==0.8.1').format(cmd=project.venv_bin( version=version_slug, bin='pip'), sphinx=sphinx, ignore_option=ignore_option)) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run( '{cmd} install --exists-action=w -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) if getattr(settings, 'USE_PIP_INSTALL', False): update_docs_output['install'] = run( '{cmd} install --ignore-installed .'.format( cmd=project.venv_bin(version=version_slug, bin='pip'))) else: update_docs_output['install'] = run( '{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: old_versions = [ obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=5000)['objects'] ] if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags for tag in tags: if tag.identifier in old_versions: continue log.debug('NEW TAG: (%s not in %s)' % (tag.identifier, old_versions)) slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: version_data = api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name)) ver = make_api_version(version_data) log.info("New tag found: {0}".format(tag.identifier)) ver, highest = project.highest_version[1] ver_obj = mkversion(ver) # TODO: Handle updating higher versions automatically. # This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: log.info("Highest version known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches for branch in branches: if branch.identifier in old_versions: continue log.debug('NEW BRANCH: (%s not in %s)' % (branch, old_versions)) slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name)) log.info(("New branch found: {0}".format( branch.identifier))) except Exception: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() transaction.leave_transaction_management() #TODO: Kill deleted branches except ValueError: log.error("Error getting tags", exc_info=True) return update_docs_output
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ if not project.vcs_repo(): raise ProjectImportError("Repo type '%s' unknown" % project.repo_type) if version: print 'Checking out version %s: %s' % (version.slug, version.identifier) version_slug = version.slug version_repo = project.vcs_repo(version_slug) version_repo.checkout(version.identifier) else: print 'Updating to latest revision' version_slug = 'latest' version_repo = project.vcs_repo(version_slug) version_repo.update() #Break early without a conf file. if not project.conf_file(version.slug): print "Conf File Missing. Skipping." return -1 #Do Virtualenv bits: if project.use_virtualenv and project.whitelisted: run('virtualenv --no-site-packages %s' % project.venv_path(version=version_slug)) run('%s install sphinx' % project.venv_bin(version=version_slug, bin='pip')) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) run('%s install -r %s' % (project.venv_bin(version=version_slug, bin='pip'), project.requirements_file)) os.chdir(project.checkout_path(version_slug)) run('%s setup.py install --force' % project.venv_bin(version=version_slug, bin='python')) # check tags/version try: if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.get_tags() old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True) for tag in tags: if tag.identifier in old_tags: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: ver, created = Version.objects.get_or_create( project=project, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name ) print "New tag found: %s" % ver highest = project.highest_version[1] ver_obj = mkversion(ver) if highest and ver_obj and ver_obj > highest: print "Highest verison known, building docs" update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: print "Failed to create version (tag): %s" % e transaction.rollback() transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.get_branches() old_branches = Version.objects.filter(project=project).values_list('identifier', flat=True) for branch in branches: if branch.identifier in old_branches: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: Version.objects.get_or_create( project=project, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name ) print "New branch found: %s" % ver except Exception, e: print "Failed to create version (branch): %s" % e transaction.rollback() transaction.leave_transaction_management() #Kill deleted branches Version.objects.filter(project=project).exclude(identifier__in=old_branches).delete()