def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
if not project.vcs_repo:
raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
if version:
print 'Checking out version %s: %s' % (version.slug, version.identifier)
project.vcs_repo.checkout(version.identifier)
else:
print 'Updating to latest revision'
project.vcs_repo.update()
# check tags/version
try:
if project.vcs_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = project.vcs_repo.get_tags()
old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True)
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project)
try:
Version.objects.get_or_create(
project=project,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name
)
except Exception, e:
print "Failed to create version (tag): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
if project.vcs_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = project.vcs_repo.get_branches()
old_branches = Version.objects.filter(project=project).values_list('identifier', flat=True)
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project)
try:
Version.objects.get_or_create(
project=project,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name
)
except Exception, e:
print "Failed to create version (branch): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
#Kill deleted branches
Version.objects.filter(project=project).exclude(identifier__in=old_branches).delete()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
if not project.vcs_repo:
raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
if version:
print 'Checking out version %s: %s' % (version.slug, version.identifier)
project.vcs_repo.checkout(version.identifier)
else:
print 'Updating to latest revision'
project.vcs_repo.update()
# check tags/version
try:
if project.vcs_repo.supports_tags:
tags = project.vcs_repo.get_tags()
old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True)
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project)
try:
Version.objects.get_or_create(
project=project,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name
)
except Exception, e:
print "Failed to create version: %s" % e
transaction.rollback()
except ValueError, e:
print "Error getting tags: %s" % e
def sync_versions(project, versions):
"""
Update the database with the current versions from the repository.
"""
old_versions = {}
old_version_values = project.versions.values("identifier", "verbose_name")
for version in old_version_values:
old_versions[version["verbose_name"]] = version["identifier"]
added = set()
# Add new versions
for version in versions:
version_id = version["identifier"]
version_name = version["verbose_name"]
if version_name in old_versions.keys():
if version_id == old_versions[version_name]:
# Version is correct
continue
else:
# Update slug with new identifier
Version.objects.filter(project=project, verbose_name=version_name).update(identifier=version_id)
log.info("(Sync Versions) Updated Version: [%s=%s] " % (version["verbose_name"], version["identifier"]))
else:
# New Version
slug = slugify_uniquely(Version, version["verbose_name"], "slug", 255, project=project)
Version.objects.create(
project=project, slug=slug, identifier=version["identifier"], verbose_name=version["verbose_name"]
)
added.add(slug)
if added:
log.info("(Sync Versions) Added Versions: [%s] " % " ".join(added))
return added
def sync_versions(project, versions, type):
"""
Update the database with the current versions from the repository.
"""
# Bookkeeping for keeping tag/branch identifies correct
verbose_names = [v['verbose_name'] for v in versions]
project.versions.filter(verbose_name__in=verbose_names).update(type=type)
old_versions = {}
old_version_values = project.versions.values('identifier', 'verbose_name')
for version in old_version_values:
old_versions[version['verbose_name']] = version['identifier']
added = set()
# Add new versions
for version in versions:
version_id = version['identifier']
version_name = version['verbose_name']
if version_name in old_versions.keys():
if version_id == old_versions[version_name]:
# Version is correct
continue
else:
# Update slug with new identifier
Version.objects.filter(project=project,
verbose_name=version_name).update(
identifier=version_id,
type=type,
machine=False,
)
log.info("(Sync Versions) Updated Version: [%s=%s] " %
(version['verbose_name'], version['identifier']))
else:
# New Version
slug = slugify_uniquely(Version,
version['verbose_name'],
'slug',
255,
project=project)
Version.objects.create(
project=project,
slug=slug,
type=type,
identifier=version['identifier'],
verbose_name=version['verbose_name'],
)
added.add(slug)
if added:
log.info("(Sync Versions) Added Versions: [%s] " % ' '.join(added))
return added
def sync_versions(project, versions, type):
"""
Update the database with the current versions from the repository.
"""
# Bookkeeping for keeping tag/branch identifies correct
verbose_names = [v['verbose_name'] for v in versions]
project.versions.filter(verbose_name__in=verbose_names).update(type=type)
old_versions = {}
old_version_values = project.versions.values('identifier', 'verbose_name')
for version in old_version_values:
old_versions[version['verbose_name']] = version['identifier']
added = set()
# Add new versions
for version in versions:
version_id = version['identifier']
version_name = version['verbose_name']
if version_name in old_versions.keys():
if version_id == old_versions[version_name]:
# Version is correct
continue
else:
# Update slug with new identifier
Version.objects.filter(
project=project, verbose_name=version_name
).update(
identifier=version_id,
type=type,
machine=False,
)
log.info("(Sync Versions) Updated Version: [%s=%s] " % (version['verbose_name'], version['identifier']))
else:
# New Version
slug = slugify_uniquely(Version, version['verbose_name'], 'slug', 255, project=project)
Version.objects.create(
project=project,
slug=slug,
type=type,
identifier=version['identifier'],
verbose_name=version['verbose_name'],
)
added.add(slug)
if added:
log.info("(Sync Versions) Added Versions: [%s] " % ' '.join(added))
return added
def _sync_versions(self, project, versions):
"""
Update the database with the current versions from the repository.
"""
old_versions = project.versions.values_list('identifier', flat=True)
# Add new versions
for version in versions:
if version['identifier'] in old_versions:
continue
slug = slugify_uniquely(Version, version['verbose_name'], 'slug', 255, project=project)
Version.objects.create(
project=project,
slug=slug,
identifier=version['identifier'],
verbose_name=version['verbose_name'],
)
def _sync_versions(self, project, versions):
"""
Update the database with the current versions from the repository.
"""
old_versions = project.versions.values_list('identifier', flat=True)
# Add new versions
for version in versions:
if version['identifier'] in old_versions:
continue
slug = slugify_uniquely(Version,
version['verbose_name'],
'slug',
255,
project=project)
Version.objects.create(
project=project,
slug=slug,
identifier=version['identifier'],
verbose_name=version['verbose_name'],
)
def update_imported_docs(version_pk):
"""
Check out or update the given project's repository.
"""
version_data = api.version(version_pk).get()
version = make_api_version(version_data)
project = version.project
# Make Dirs
if not os.path.exists(project.doc_path):
os.makedirs(project.doc_path)
with project.repo_lock(getattr(settings, "REPO_LOCK_SECONDS", 30)):
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(repo_type=project.repo_type))
# Get the actual code on disk
if version:
log.info(
"Checking out version {slug}: {identifier}".format(slug=version.slug, identifier=version.identifier)
)
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output["checkout"] = version_repo.checkout(version.identifier)
else:
# Does this ever get called?
log.info("Updating to latest revision")
version_slug = "latest"
version_repo = project.vcs_repo(version_slug)
update_docs_output["checkout"] = version_repo.update()
# Ensure we have a conf file (an exception is raised if not)
project.conf_file(version.slug)
# Do Virtualenv bits:
if project.use_virtualenv:
if project.use_system_packages:
site_packages = "--system-site-packages"
else:
site_packages = "--no-site-packages"
# Here the command has been modified to support different
# interpreters.
update_docs_output["venv"] = run(
"{cmd} --distribute {site_packages} {path}".format(
cmd="virtualenv -p {interpreter}".format(interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version_slug),
)
)
# Other code expects sphinx-build to be installed inside the virtualenv.
# Using the -I option makes sure it gets installed even if it is
# already installed system-wide (and --system-site-packages is used)
if project.use_system_packages:
ignore_option = "-I"
else:
ignore_option = ""
if project.python_interpreter != "python3":
update_docs_output["sphinx"] = run(
"{cmd} install -U {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1".format(
cmd=project.venv_bin(version=version_slug, bin="pip"), ignore_option=ignore_option
)
)
else:
# python 3 specific hax
update_docs_output["sphinx"] = run(
"{cmd} install {ignore_option} hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 docutils==0.8.1".format(
cmd=project.venv_bin(version=version_slug, bin="pip"), ignore_option=ignore_option
)
)
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output["requirements"] = run(
"{cmd} install -r {requirements}".format(
cmd=project.venv_bin(version=version_slug, bin="pip"), requirements=project.requirements_file
)
)
os.chdir(project.checkout_path(version_slug))
if getattr(settings, "USE_PIP_INSTALL", False):
update_docs_output["install"] = run(
"{cmd} install --ignore-installed .".format(cmd=project.venv_bin(version=version_slug, bin="pip"))
)
else:
update_docs_output["install"] = run(
"{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version_slug, bin="python"))
)
# check tags/version
# XXX:dc: what in this block raises the values error?
try:
old_versions = [
obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=5000)["objects"]
]
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
for tag in tags:
if tag.identifier in old_versions:
continue
log.debug("NEW TAG: (%s not in %s)" % (tag.identifier, old_versions))
slug = slugify_uniquely(Version, tag.verbose_name, "slug", 255, project=project)
try:
version_data = api.version.post(
dict(
project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name,
)
)
ver = make_api_version(version_data)
log.info("New tag found: {0}".format(tag.identifier))
ver, highest = project.highest_version[1]
ver_obj = mkversion(ver)
# TODO: Handle updating higher versions automatically.
# This never worked very well, anyways.
if highest and ver_obj and ver_obj > highest:
log.info("Highest version known, building docs")
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
log.error("Failed to create version (tag)", exc_info=True)
transaction.rollback()
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
for branch in branches:
if branch.identifier in old_versions:
continue
log.debug("NEW BRANCH: (%s not in %s)" % (branch, old_versions))
slug = slugify_uniquely(Version, branch.verbose_name, "slug", 255, project=project)
try:
api.version.post(
dict(
project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name,
)
)
log.info("New branch found: {0}".format(branch.identifier))
except Exception, e:
log.error("Failed to create version (branch)", exc_info=True)
transaction.rollback()
transaction.leave_transaction_management()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(
repo_type=project.repo_type))
if version:
log.info('Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier))
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.checkout(
version.identifier)
else:
log.info('Updating to latest revision')
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.update()
# Ensure we have a conf file (an exception is raised if not)
conf_file = project.conf_file(version.slug)
#Do Virtualenv bits:
if project.use_virtualenv:
update_docs_output['venv'] = run(
'{cmd} --distribute --no-site-packages {path}'.format(
cmd='virtualenv',
path=project.venv_path(version=version_slug)))
update_docs_output['sphinx'] = run(
'{cmd} install -U hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1'
.format(cmd=project.venv_bin(version=version_slug, bin='pip')))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output['requirements'] = run(
'{cmd} install -r {requirements}'.format(
cmd=project.venv_bin(version=version_slug, bin='pip'),
requirements=project.requirements_file))
os.chdir(project.checkout_path(version_slug))
update_docs_output['install'] = run(
'{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version_slug, bin='python')))
# check tags/version
#XXX:dc: what in this block raises the values error?
try:
old_versions = [
obj['identifier'] for obj in api.version.get(
project__slug=project.slug, limit=500)['objects']
]
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
for tag in tags:
if tag.identifier in old_versions:
continue
slug = slugify_uniquely(Version,
tag.verbose_name,
'slug',
255,
project=project)
try:
version_data = api.version.post(
dict(project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name))
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
version_data['project'] = project
ver = Version(**version_data)
log.info("New tag found: {0}".format(tag.identifier))
ver, highest = project.highest_version[1]
ver_obj = mkversion(ver)
#TODO: Handle updating higher versions automatically.
#This never worked very well, anyways.
if highest and ver_obj and ver_obj > highest:
log.info("Highest verison known, building docs")
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
log.error("Failed to create version (tag)", exc_info=True)
transaction.rollback()
#break here to stop updating tags when they will all fail.
break
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
for branch in branches:
if branch.identifier in old_versions:
continue
slug = slugify_uniquely(Version,
branch.verbose_name,
'slug',
255,
project=project)
try:
api.version.post(
dict(project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name))
log.info("New branch found: {0}".format(branch.identifier))
except Exception, e:
log.error("Failed to create version (branch)",
exc_info=True)
transaction.rollback()
#break here to stop updating branches when they will all fail.
break
transaction.leave_transaction_management()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(
repo_type=project.repo_type))
if version:
log.info('Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier))
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.checkout(version.identifier)
else:
log.info('Updating to latest revision')
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.update()
# Ensure we have a conf file (an exception is raised if not)
conf_file = project.conf_file(version.slug)
#Do Virtualenv bits:
if project.use_virtualenv:
update_docs_output['venv'] = run('{cmd} --distribute --never-download --no-site-packages {path}'.format(
cmd='virtualenv',
path=project.venv_path(version=version_slug)))
update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format(
cmd=project.venv_bin(version=version_slug, bin='pip')))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
cmd=project.venv_bin(version=version_slug, bin='pip'),
requirements=project.requirements_file))
os.chdir(project.checkout_path(version_slug))
update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version_slug, bin='python')))
# check tags/version
#XXX:dc: what in this block raises the values error?
try:
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
old_tags = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects']]
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name,
'slug', 255, project=project)
try:
api.version.post(dict(
project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name
))
log.info("New tag found: {0}".format(tag.identifier))
highest = project.highest_version['version']
ver_obj = mkversion(ver)
#TODO: Handle updating higher versions automatically.
#This never worked very well, anyways.
if highest and ver_obj and ver_obj > highest:
log.info("Highest verison known, building docs")
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
log.error("Failed to create version (tag)", exc_info=True)
transaction.rollback()
#break
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
old_branches = [obj['identifier'] for obj in api.version.get(project__slug=project.slug, limit=50)['objects']]
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version, branch.verbose_name,
'slug', 255, project=project)
try:
api.version.post(dict(
project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name
))
log.info("New branch found: {0}".format(branch.identifier))
except Exception, e:
log.error("Failed to create version (branch)", exc_info=True)
transaction.rollback()
#break
transaction.leave_transaction_management()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(
repo_type=project.repo_type))
if version:
print 'Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier)
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.checkout(version.identifier)
else:
print 'Updating to latest revision'
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.update()
#Break early without a conf file.
if not project.conf_file(version.slug):
raise ProjectImportError("Conf File Missing. Skipping.")
#Do Virtualenv bits:
if project.use_virtualenv and project.whitelisted:
update_docs_output['venv'] = run('{cmd} --no-site-packages {path}'.format(
cmd='virtualenv',
path=project.venv_path(version=version_slug)))
update_docs_output['sphinx'] = run('{cmd} install -U sphinx'.format(
cmd=project.venv_bin(version=version_slug, bin='pip')))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
cmd=project.venv_bin(version=version_slug, bin='pip'),
requirements=project.requirements_file))
os.chdir(project.checkout_path(version_slug))
update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version_slug, bin='python')))
# check tags/version
#XXX:dc: what in this block raises the values error?
try:
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
old_tags = Version.objects.filter(
project=project).values_list('identifier', flat=True)
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name,
'slug', 255, project=project)
try:
ver, created = Version.objects.get_or_create(
project=project,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name
)
print "New tag found: %s" % ver
highest = project.highest_version['version']
ver_obj = mkversion(ver)
if highest and ver_obj and ver_obj > highest:
print "Highest verison known, building docs"
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
print "Failed to create version (tag): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
old_branches = Version.objects.filter(
project=project).values_list('identifier', flat=True)
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version, branch.verbose_name,
'slug', 255, project=project)
try:
ver, created = Version.objects.get_or_create(
project=project,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name
)
print "New branch found: %s" % ver
except Exception, e:
print "Failed to create version (branch): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
#Kill deleted branches
Version.objects.filter(
project=project).exclude(identifier__in=old_branches).delete()
def update_imported_docs(version_pk):
"""
Check out or update the given project's repository.
"""
version_data = api.version(version_pk).get()
version = make_api_version(version_data)
project = version.project
# Make Dirs
if not os.path.exists(project.doc_path):
os.makedirs(project.doc_path)
with project.repo_lock(getattr(settings, 'REPO_LOCK_SECONDS', 30)):
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError(
("Repo type '{0}' unknown".format(project.repo_type)))
# Get the actual code on disk
if version:
log.info('Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier))
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.checkout(
version.identifier)
else:
# Does this ever get called?
log.info('Updating to latest revision')
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.update()
# Ensure we have a conf file (an exception is raised if not)
project.conf_file(version.slug)
# Do Virtualenv bits:
if project.use_virtualenv:
if project.use_system_packages:
site_packages = '--system-site-packages'
else:
site_packages = '--no-site-packages'
# Here the command has been modified to support different
# interpreters.
update_docs_output['venv'] = run(
'{cmd} --distribute {site_packages} {path}'.format(
cmd='virtualenv -p {interpreter}'.format(
interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version_slug)))
# Other code expects sphinx-build to be installed inside the
# virtualenv. Using the -I option makes sure it gets installed
# even if it is already installed system-wide (and
# --system-site-packages is used)
if project.use_system_packages:
ignore_option = '-I'
else:
ignore_option = ''
sphinx = ('hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c'
'#egg=Sphinx')
if project.python_interpreter != 'python3':
update_docs_output['sphinx'] = run(
('{cmd} install -U {ignore_option} {sphinx} '
'virtualenv==1.8.2 distribute==0.6.28 '
'docutils==0.8.1').format(cmd=project.venv_bin(
version=version_slug, bin='pip'),
sphinx=sphinx,
ignore_option=ignore_option))
else:
# python 3 specific hax
update_docs_output['sphinx'] = run((
'{cmd} install {ignore_option} {sphinx} virtualenv==1.8.2 '
'docutils==0.8.1').format(cmd=project.venv_bin(
version=version_slug, bin='pip'),
sphinx=sphinx,
ignore_option=ignore_option))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output['requirements'] = run(
'{cmd} install --exists-action=w -r {requirements}'.format(
cmd=project.venv_bin(version=version_slug, bin='pip'),
requirements=project.requirements_file))
os.chdir(project.checkout_path(version_slug))
if getattr(settings, 'USE_PIP_INSTALL', False):
update_docs_output['install'] = run(
'{cmd} install --ignore-installed .'.format(
cmd=project.venv_bin(version=version_slug, bin='pip')))
else:
update_docs_output['install'] = run(
'{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version_slug,
bin='python')))
# check tags/version
#XXX:dc: what in this block raises the values error?
try:
old_versions = [
obj['identifier']
for obj in api.version.get(project__slug=project.slug,
limit=5000)['objects']
]
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
for tag in tags:
if tag.identifier in old_versions:
continue
log.debug('NEW TAG: (%s not in %s)' %
(tag.identifier, old_versions))
slug = slugify_uniquely(Version,
tag.verbose_name,
'slug',
255,
project=project)
try:
version_data = api.version.post(
dict(project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name))
ver = make_api_version(version_data)
log.info("New tag found: {0}".format(tag.identifier))
ver, highest = project.highest_version[1]
ver_obj = mkversion(ver)
# TODO: Handle updating higher versions automatically.
# This never worked very well, anyways.
if highest and ver_obj and ver_obj > highest:
log.info("Highest version known, building docs")
update_docs.delay(ver.project.pk,
version_pk=ver.pk)
except Exception:
log.error("Failed to create version (tag)",
exc_info=True)
transaction.rollback()
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
for branch in branches:
if branch.identifier in old_versions:
continue
log.debug('NEW BRANCH: (%s not in %s)' %
(branch, old_versions))
slug = slugify_uniquely(Version,
branch.verbose_name,
'slug',
255,
project=project)
try:
api.version.post(
dict(project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name))
log.info(("New branch found: {0}".format(
branch.identifier)))
except Exception:
log.error("Failed to create version (branch)",
exc_info=True)
transaction.rollback()
transaction.leave_transaction_management()
#TODO: Kill deleted branches
except ValueError:
log.error("Error getting tags", exc_info=True)
return update_docs_output
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
if not project.vcs_repo():
raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
if version:
print 'Checking out version %s: %s' % (version.slug, version.identifier)
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
version_repo.checkout(version.identifier)
else:
print 'Updating to latest revision'
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
version_repo.update()
#Break early without a conf file.
if not project.conf_file(version.slug):
print "Conf File Missing. Skipping."
return -1
#Do Virtualenv bits:
if project.use_virtualenv and project.whitelisted:
run('virtualenv --no-site-packages %s' % project.venv_path(version=version_slug))
run('%s install sphinx' % project.venv_bin(version=version_slug,
bin='pip'))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
run('%s install -r %s' % (project.venv_bin(version=version_slug, bin='pip'),
project.requirements_file))
os.chdir(project.checkout_path(version_slug))
run('%s setup.py install --force' % project.venv_bin(version=version_slug,
bin='python'))
# check tags/version
try:
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.get_tags()
old_tags = Version.objects.filter(project=project).values_list('identifier', flat=True)
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project)
try:
ver, created = Version.objects.get_or_create(
project=project,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name
)
print "New tag found: %s" % ver
highest = project.highest_version[1]
ver_obj = mkversion(ver)
if highest and ver_obj and ver_obj > highest:
print "Highest verison known, building docs"
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
print "Failed to create version (tag): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.get_branches()
old_branches = Version.objects.filter(project=project).values_list('identifier', flat=True)
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project)
try:
Version.objects.get_or_create(
project=project,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name
)
print "New branch found: %s" % ver
except Exception, e:
print "Failed to create version (branch): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
#Kill deleted branches
Version.objects.filter(project=project).exclude(identifier__in=old_branches).delete()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
if not project.vcs_repo():
raise ProjectImportError("Repo type '%s' unknown" % project.repo_type)
if version:
print 'Checking out version %s: %s' % (version.slug,
version.identifier)
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
version_repo.checkout(version.identifier)
else:
print 'Updating to latest revision'
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
version_repo.update()
#Break early without a conf file.
if not project.conf_file(version.slug):
print "Conf File Missing. Skipping."
return -1
#Do Virtualenv bits:
if project.use_virtualenv and project.whitelisted:
run('virtualenv --no-site-packages %s' %
project.venv_path(version=version_slug))
run('%s install sphinx' %
project.venv_bin(version=version_slug, bin='pip'))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
run('%s install -r %s' % (project.venv_bin(
version=version_slug, bin='pip'), project.requirements_file))
os.chdir(project.checkout_path(version_slug))
run('%s setup.py install --force' %
project.venv_bin(version=version_slug, bin='python'))
# check tags/version
try:
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.get_tags()
old_tags = Version.objects.filter(project=project).values_list(
'identifier', flat=True)
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version,
tag.verbose_name,
'slug',
255,
project=project)
try:
Version.objects.get_or_create(
project=project,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name)
except Exception, e:
print "Failed to create version (tag): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.get_branches()
old_branches = Version.objects.filter(project=project).values_list(
'identifier', flat=True)
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version,
branch.verbose_name,
'slug',
255,
project=project)
try:
Version.objects.get_or_create(
project=project,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name)
except Exception, e:
print "Failed to create version (branch): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
#Kill deleted branches
Version.objects.filter(project=project).exclude(
identifier__in=old_branches).delete()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(
repo_type=project.repo_type))
if version:
print 'Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier)
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.checkout(version.identifier)
else:
print 'Updating to latest revision'
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.update()
#Break early without a conf file.
if not project.conf_file(version.slug):
raise ProjectImportError("Conf File Missing. Skipping.")
#Do Virtualenv bits:
if project.use_virtualenv and project.whitelisted:
update_docs_output['venv'] = run('{cmd} --no-site-packages {path}'.format(
cmd='virtualenv',
path=project.venv_path(version=version_slug)))
update_docs_output['sphinx'] = run('{cmd} install sphinx'.format(
cmd=project.venv_bin(version=version_slug, bin='pip')))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output['requirements'] = run('{cmd} install -r {requirements}'.format(
cmd=project.venv_bin(version=version_slug, bin='pip'),
requirements=project.requirements_file))
os.chdir(project.checkout_path(version_slug))
update_docs_output['install'] = run('{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version_slug, bin='python')))
# check tags/version
#XXX:dc: what in this block raises the values error?
try:
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
old_tags = Version.objects.filter(
project=project).values_list('identifier', flat=True)
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name,
'slug', 255, project=project)
try:
ver, created = Version.objects.get_or_create(
project=project,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name
)
print "New tag found: %s" % ver
highest = project.highest_version['version']
ver_obj = mkversion(ver)
if highest and ver_obj and ver_obj > highest:
print "Highest verison known, building docs"
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
print "Failed to create version (tag): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
old_branches = Version.objects.filter(
project=project).values_list('identifier', flat=True)
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version, branch.verbose_name,
'slug', 255, project=project)
try:
ver, created = Version.objects.get_or_create(
project=project,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name
)
print "New branch found: %s" % ver
except Exception, e:
print "Failed to create version (branch): %s" % e
transaction.rollback()
transaction.leave_transaction_management()
#Kill deleted branches
Version.objects.filter(
project=project).exclude(identifier__in=old_branches).delete()
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(repo_type=project.repo_type))
if version:
print "Checking out version {slug}: {identifier}".format(slug=version.slug, identifier=version.identifier)
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output["checkout"] = version_repo.checkout(version.identifier)
else:
print "Updating to latest revision"
version_slug = "latest"
version_repo = project.vcs_repo(version_slug)
update_docs_output["checkout"] = version_repo.update()
# Break early without a conf file.
if not project.conf_file(version.slug):
raise ProjectImportError("Conf File Missing. Skipping.")
# Do Virtualenv bits:
if project.use_virtualenv:
update_docs_output["venv"] = run(
"{cmd} --distribute --never-download --no-site-packages {path}".format(
cmd="virtualenv", path=project.venv_path(version=version_slug)
)
)
update_docs_output["sphinx"] = run(
"{cmd} install -U sphinx".format(cmd=project.venv_bin(version=version_slug, bin="pip"))
)
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output["requirements"] = run(
"{cmd} install -r {requirements}".format(
cmd=project.venv_bin(version=version_slug, bin="pip"), requirements=project.requirements_file
)
)
os.chdir(project.checkout_path(version_slug))
update_docs_output["install"] = run(
"{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version_slug, bin="python"))
)
# check tags/version
# XXX:dc: what in this block raises the values error?
try:
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
old_tags = [obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=50)["objects"]]
for tag in tags:
if tag.identifier in old_tags:
continue
slug = slugify_uniquely(Version, tag.verbose_name, "slug", 255, project=project)
try:
api.version.post(
dict(
project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name,
)
)
print "New tag found: %s" % tag.identifier
highest = project.highest_version["version"]
ver_obj = mkversion(ver)
# TODO: Handle updating higher versions automatically.
# This never worked very well, anyways.
if highest and ver_obj and ver_obj > highest:
print "Highest verison known, building docs"
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
print "Failed to create version (tag): %s" % e
transaction.rollback()
# break
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
old_branches = [
obj["identifier"] for obj in api.version.get(project__slug=project.slug, limit=50)["objects"]
]
for branch in branches:
if branch.identifier in old_branches:
continue
slug = slugify_uniquely(Version, branch.verbose_name, "slug", 255, project=project)
try:
api.version.post(
dict(
project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name,
)
)
print "New branch found: %s" % branch.identifier
except Exception, e:
print "Failed to create version (branch): %s" % e
transaction.rollback()
# break
transaction.leave_transaction_management()
