def make_api_version(version_data):
from builds.models import Version
for key in ['resource_uri', 'absolute_url']:
if key in version_data:
del version_data[key]
project_data = version_data['project']
project = make_api_project(project_data)
version_data['project'] = project
ver = Version(**version_data)
ver.save = _new_save
return ver
def make_api_version(version_data):
from builds.models import Version
for key in ['resource_uri', 'absolute_url', 'downloads']:
if key in version_data:
del version_data[key]
project_data = version_data['project']
project = make_api_project(project_data)
version_data['project'] = project
ver = Version(**version_data)
ver.save = _new_save
return ver
def make_api_version(version_data):
from builds.models import Version
del version_data['resource_uri']
project_data = version_data['project']
project = make_api_project(project_data)
version_data['project'] = project
ver = Version(**version_data)
return ver
def api_versions(self):
from builds.models import Version
ret = []
for version_data in api.version.get(project=self.pk,
active=True)['objects']:
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
version_data['project'] = project
ret.append(Version(**version_data))
return sort_version_aware(ret)
def update_intersphinx(version_pk):
version_data = api.version(version_pk).get()
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
version_data['project'] = project
version = Version(**version_data)
try:
object_file = version.project.find('objects.inv', version.slug)[0]
except IndexError, e:
print "Failed to find objects file"
return None
def update_intersphinx(version_pk):
version_data = api.version(version_pk).get()
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
version_data['project'] = project
version = Version(**version_data)
object_file = version.project.find('objects.inv', version.slug)[0]
path = version.project.rtd_build_path(version.slug)
if not path:
log.warning("%s has no path" % version)
return None
app = DictObj()
app.srcdir = path
try:
inv = fetch_inventory(app, path, object_file)
except TypeError, e:
log.error("Failed to fetch inventory for %s" % version, exc_info=True)
return None
def update_docs(pk,
record=True,
pdf=True,
man=True,
epub=True,
version_pk=None,
force=False,
**kwargs):
"""
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands from
the shell, for example.
"""
###
# Handle passed in arguments
###
update_output = kwargs.get('update_output', {})
project_data = api.project(pk).get()
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
# Prevent saving the temporary Project instance
def new_save(*args, **kwargs):
log.warning("Called save on a non-real object.")
return 0
project.save = new_save
log.info("Building %s" % project)
if version_pk:
version_data = api.version(version_pk).get()
del version_data['resource_uri']
else:
#Create or use the 'latest' branch, which is the default for a project.
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
version_data = api.version(
project.slug).get(slug='latest')['objects'][0]
del version_data['resource_uri']
except (slumber.exceptions.HttpClientError, IndexError) as exc:
#if exc.response.status_code in [404,500]:
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
del version_data['resource_uri']
except Exception as e:
log.info("Exception in creating version: %s" % e)
#raise e
version_data['project'] = project
version = Version(**version_data)
version.save = new_save
if not version_pk:
#Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data[
'project'] = "/api/v1/version/%s/" % version_data['project'].pk
api.version(version.pk).put(version_data)
if record:
#Create Build Object.
build = api.build.post(
dict(
project='/api/v1/project/%s/' % project.pk,
version='/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
#Make Dirs
path = project.doc_path
if not os.path.exists(path):
os.makedirs(path)
with project.repo_lock(30):
if project.is_imported:
try:
update_output = update_imported_docs(project, version)
except ProjectImportError, err:
log.error("Failed to import project; skipping build.",
exc_info=True)
build['state'] = 'finished'
build[
'setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
api.build(build['id']).put(build)
return False
else:
def update_imported_docs(project, version):
"""
Check out or update the given project's repository.
"""
update_docs_output = {}
if not project.vcs_repo():
raise ProjectImportError("Repo type '{repo_type}' unknown".format(
repo_type=project.repo_type))
if version:
log.info('Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier))
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.checkout(
version.identifier)
else:
log.info('Updating to latest revision')
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
update_docs_output['checkout'] = version_repo.update()
# Ensure we have a conf file (an exception is raised if not)
conf_file = project.conf_file(version.slug)
#Do Virtualenv bits:
if project.use_virtualenv:
update_docs_output['venv'] = run(
'{cmd} --distribute --no-site-packages {path}'.format(
cmd='virtualenv',
path=project.venv_path(version=version_slug)))
update_docs_output['sphinx'] = run(
'{cmd} install -U hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1'
.format(cmd=project.venv_bin(version=version_slug, bin='pip')))
if project.requirements_file:
os.chdir(project.checkout_path(version_slug))
update_docs_output['requirements'] = run(
'{cmd} install -r {requirements}'.format(
cmd=project.venv_bin(version=version_slug, bin='pip'),
requirements=project.requirements_file))
os.chdir(project.checkout_path(version_slug))
update_docs_output['install'] = run(
'{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version_slug, bin='python')))
# check tags/version
#XXX:dc: what in this block raises the values error?
try:
old_versions = [
obj['identifier'] for obj in api.version.get(
project__slug=project.slug, limit=500)['objects']
]
if version_repo.supports_tags:
transaction.enter_transaction_management(True)
tags = version_repo.tags
for tag in tags:
if tag.identifier in old_versions:
continue
slug = slugify_uniquely(Version,
tag.verbose_name,
'slug',
255,
project=project)
try:
version_data = api.version.post(
dict(project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=tag.identifier,
verbose_name=tag.verbose_name))
del version_data['resource_uri']
project_data = version_data['project']
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
version_data['project'] = project
ver = Version(**version_data)
log.info("New tag found: {0}".format(tag.identifier))
ver, highest = project.highest_version[1]
ver_obj = mkversion(ver)
#TODO: Handle updating higher versions automatically.
#This never worked very well, anyways.
if highest and ver_obj and ver_obj > highest:
log.info("Highest verison known, building docs")
update_docs.delay(ver.project.pk, version_pk=ver.pk)
except Exception, e:
log.error("Failed to create version (tag)", exc_info=True)
transaction.rollback()
#break here to stop updating tags when they will all fail.
break
transaction.leave_transaction_management()
if version_repo.supports_branches:
transaction.enter_transaction_management(True)
branches = version_repo.branches
for branch in branches:
if branch.identifier in old_versions:
continue
slug = slugify_uniquely(Version,
branch.verbose_name,
'slug',
255,
project=project)
try:
api.version.post(
dict(project="/api/v1/project/%s/" % project.pk,
slug=slug,
identifier=branch.identifier,
verbose_name=branch.verbose_name))
log.info("New branch found: {0}".format(branch.identifier))
except Exception, e:
log.error("Failed to create version (branch)",
exc_info=True)
transaction.rollback()
#break here to stop updating branches when they will all fail.
break
transaction.leave_transaction_management()
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
"""
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands from
the shell, for example.
"""
###
# Handle passed in arguments
###
update_output = kwargs.get('update_output', {})
project_data = api.project(pk).get()
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
# Prevent saving the temporary Project instance
def new_save(*args, **kwargs):
log.warning("Called save on a non-real object.")
return 0
project.save = new_save
log.info("Building %s" % project)
if version_pk:
version_data = api.version(version_pk).get()
del version_data['resource_uri']
else:
#Create or use the 'latest' branch, which is the default for a project.
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
version_data = api.version(project.slug).get(slug='latest')['objects'][0]
del version_data['resource_uri']
except (slumber.exceptions.HttpClientError, IndexError) as exc:
#if exc.response.status_code in [404,500]:
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
del version_data['resource_uri']
except Exception as e:
raise e
version_data['project'] = project
version = Version(**version_data)
version.save = new_save
if not version_pk:
#Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data['project'] = "/api/v1/version/%s/" % version_data['project'].pk
api.version(version.pk).put(version_data)
if record:
#Create Build Object.
build = api.build.post(dict(
project= '/api/v1/project/%s/' % project.pk,
version= '/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
#Make Dirs
path = project.doc_path
if not os.path.exists(path):
os.makedirs(path)
with project.repo_lock(30):
if project.is_imported:
try:
update_output = update_imported_docs(project, version)
except ProjectImportError, err:
log.error("Failed to import project; skipping build.", exc_info=True)
return False
else:
def setUp(self):
self.project = Project.objects.get(slug='pip')
self.version = Version(slug='foo', verbose_name='foobar')
self.project.versions.add(self.version)
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
"""
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
"""
###
# Handle passed in arguments
###
update_output = kwargs.get("update_output", {})
project_data = api.project(pk).get()
del project_data["users"]
del project_data["resource_uri"]
del project_data["absolute_url"]
project = Project(**project_data)
def new_save(*args, **kwargs):
# fields = [(field, field.value_to_string(self)) for field in self._meta.fields]
print "*** Called save on a non-real object."
# print fields
# raise TypeError('Not a real model')
return 0
project.save = new_save
print "Building %s" % project
if version_pk:
version_data = api.version(version_pk).get()
del version_data["resource_uri"]
else:
# Create or use the 'latest' branch, which is the default for a project.
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
version_data = api.version(project.slug).get(slug="latest")["objects"][0]
del version_data["resource_uri"]
except (slumber.exceptions.HttpClientError, IndexError) as exc:
# if exc.response.status_code in [404,500]:
version_data = dict(
project="/api/v1/project/%s/" % project.pk,
slug="latest",
active=True,
verbose_name="latest",
identifier=branch,
)
try:
version_data = api.version.post(version_data)
del version_data["resource_uri"]
except Exception as e:
raise e
version_data["project"] = project
version = Version(**version_data)
version.save = new_save
if not version_pk:
# Lots of course correction.
to_save = False
if not version.verbose_name:
version_data["verbose_name"] = "latest"
to_save = True
if not version.active:
version_data["active"] = True
to_save = True
if version.identifier != branch:
version_data["identifier"] = branch
to_save = True
if to_save:
version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk
api.version(version.pk).put(version_data)
if record:
# Create Build Object.
build = api.build.post(
dict(
project="/api/v1/project/%s/" % project.pk,
version="/api/v1/version/%s/" % version.pk,
type="html",
state="triggered",
)
)
else:
build = {}
# Make Dirs
path = project.doc_path
if not os.path.exists(path):
os.makedirs(path)
with project.repo_lock(30):
if project.is_imported:
try:
update_output = update_imported_docs(project, version)
except ProjectImportError, err:
print ("Error importing project: %s. Skipping build." % err)
return False
# scrape_conf_file(version)
else:
