def setup_vcs(version, build, api):
"""
Update the checkout of the repo to make sure it's the latest.
This also syncs versions in the DB.
"""
log.info(
LOG_TEMPLATE.format(project=version.project.slug,
version=version.slug,
msg='Updating docs from VCS'))
try:
update_output = update_imported_docs(version.pk, api)
commit = version.project.vcs_repo(version.slug).commit
if commit:
build['commit'] = commit
except ProjectImportError, err:
log.error(LOG_TEMPLATE.format(
project=version.project.slug,
version=version.slug,
msg='Failed to import project; skipping build'),
exc_info=True)
build['state'] = 'finished'
build['setup_error'] = ('Failed to import project; skipping build.\n'
'\nError\n-----\n\n%s' % err.message)
api.build(build['id']).put(build)
return False
def record_build(api, record, build, results, state, start_time=None):
"""
Record a build by hitting the API.
Returns nothing
"""
if not record:
return None
setup_steps = [
'checkout', 'venv', 'doc_builder', 'requirements', 'install'
]
output_steps = ['html']
all_steps = setup_steps + output_steps
build['state'] = state
if 'html' in results:
build['success'] = results['html'][0] == 0
else:
build['success'] = False
# Set global state
# for step in all_steps:
# if results.get(step, False):
# if results.get(step)[0] != 0:
# results['success'] = False
build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])
build['setup'] = build['setup_error'] = ""
build['output'] = build['error'] = ""
if start_time:
build['length'] = (datetime.datetime.utcnow() -
start_time).total_seconds()
for step in setup_steps:
if step in results:
build['setup'] += "\n\n%s\n-----\n\n" % step
build['setup'] += results.get(step)[1]
build['setup_error'] += "\n\n%s\n-----\n\n" % step
build['setup_error'] += results.get(step)[2]
for step in output_steps:
if step in results:
build['output'] += "\n\n%s\n-----\n\n" % step
build['output'] += results.get(step)[1]
build['error'] += "\n\n%s\n-----\n\n" % step
build['error'] += results.get(step)[2]
# Attempt to stop unicode errors on build reporting
for key, val in build.items():
if isinstance(val, basestring):
build[key] = val.decode('utf-8', 'ignore')
try:
api.build(build['id']).put(build)
except Exception:
log.error("Unable to post a new build", exc_info=True)
def record_build(api, record, build, results, state):
"""
Record a build by hitting the API.
Returns nothing
"""
if not record:
return None
setup_steps = ['checkout', 'venv', 'doc_builder', 'requirements', 'install']
output_steps = ['html']
all_steps = setup_steps + output_steps
build['state'] = state
if 'html' in results:
build['success'] = results['html'][0] == 0
else:
build['success'] = False
# Set global state
# for step in all_steps:
# if results.get(step, False):
# if results.get(step)[0] != 0:
# results['success'] = False
build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])
build['setup'] = build['setup_error'] = ""
build['output'] = build['error'] = ""
for step in setup_steps:
if step in results:
build['setup'] += "\n\n%s\n-----\n\n" % step
build['setup'] += results.get(step)[1]
build['setup_error'] += "\n\n%s\n-----\n\n" % step
build['setup_error'] += results.get(step)[2]
for step in output_steps:
if step in results:
build['output'] += "\n\n%s\n-----\n\n" % step
build['output'] += results.get(step)[1]
build['error'] += "\n\n%s\n-----\n\n" % step
build['error'] += results.get(step)[2]
# Attempt to stop unicode errors on build reporting
for key, val in build.items():
if isinstance(val, basestring):
build[key] = val.decode('utf-8', 'ignore')
try:
api.build(build['id']).put(build)
except Exception:
log.error("Unable to post a new build", exc_info=True)
def record_build(api, record, build, results, state):
"""
Record a build by hitting the API.
Returns nothing
"""
if not record:
return None
setup_steps = [
'checkout', 'venv', 'doc_builder', 'requirements', 'install'
]
output_steps = ['html']
all_steps = setup_steps + output_steps
build['state'] = state
if 'html' in results:
build['success'] = results['html'][0] == 0
else:
build['success'] = False
# Set global state
# for step in all_steps:
# if results.get(step, False):
# if results.get(step)[0] != 0:
# results['success'] = False
build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])
build['setup'] = build.get('setup', '')
build['setup_error'] = build.get('setup_error', '')
build['output'] = build.get('output', '')
build['error'] = build.get('error', '')
for step in setup_steps:
if step in results:
build['setup'] += "\n\n%s\n-----\n\n" % step
build['setup'] += results.get(step)[1]
build['setup_error'] += "\n\n%s\n-----\n\n" % step
build['setup_error'] += results.get(step)[2]
for step in output_steps:
if step in results:
build['output'] += "\n\n%s\n-----\n\n" % step
build['output'] += results.get(step)[1]
build['error'] += "\n\n%s\n-----\n\n" % step
build['error'] += results.get(step)[2]
try:
api.build(build['id']).put(build)
except Exception:
log.error("Unable to post a new build", exc_info=True)
def record_build(api, record, build, results, state):
"""
Record a build by hitting the API.
Returns nothing
"""
if not record:
return None
setup_steps = ['checkout', 'venv', 'doc_builder', 'requirements', 'install']
output_steps = ['html']
all_steps = setup_steps + output_steps
build['state'] = state
if 'html' in results:
build['success'] = results['html'][0] == 0
else:
build['success'] = False
# Set global state
# for step in all_steps:
# if results.get(step, False):
# if results.get(step)[0] != 0:
# results['success'] = False
build['exit_code'] = max([results.get(step, [0])[0] for step in all_steps])
build['setup'] = build.get('setup', '')
build['setup_error'] = build.get('setup_error', '')
build['output'] = build.get('output', '')
build['error'] = build.get('error', '')
for step in setup_steps:
if step in results:
build['setup'] += "\n\n%s\n-----\n\n" % step
build['setup'] += results.get(step)[1]
build['setup_error'] += "\n\n%s\n-----\n\n" % step
build['setup_error'] += results.get(step)[2]
for step in output_steps:
if step in results:
build['output'] += "\n\n%s\n-----\n\n" % step
build['output'] += results.get(step)[1]
build['error'] += "\n\n%s\n-----\n\n" % step
build['error'] += results.get(step)[2]
try:
api.build(build['id']).put(build)
except Exception:
log.error("Unable to post a new build", exc_info=True)
def setup_vcs(version, build, api):
"""
Update the checkout of the repo to make sure it's the latest.
This also syncs versions in the DB.
"""
log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Updating docs from VCS'))
try:
update_output = update_imported_docs(version.pk, api)
except ProjectImportError, err:
log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True)
build['state'] = 'finished'
build['setup_error'] = (
'Failed to import project; skipping build.\n'
'\nError\n-----\n\n%s' % err.message
)
api.build(build['id']).put(build)
return False
def setup_vcs(version, build, api):
"""
Update the checkout of the repo to make sure it's the latest.
This also syncs versions in the DB.
"""
log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Updating docs from VCS"))
try:
update_output = update_imported_docs(version.pk, api)
except ProjectImportError, err:
log.error(
LOG_TEMPLATE.format(
project=version.project.slug, version=version.slug, msg="Failed to import project; skipping build"
),
exc_info=True,
)
build["state"] = "finished"
build["setup_error"] = "Failed to import project; skipping build.\n" "\nError\n-----\n\n%s" % err.message
api.build(build["id"]).put(build)
return False
def create_build(build_pk):
"""
Old placeholder for build creation. Now it just gets it from the database.
"""
if build_pk:
build = api.build(build_pk).get()
for key in ['project', 'version', 'resource_uri', 'absolute_uri']:
if key in build:
del build[key]
else:
build = {}
return build
def create_build(build_pk):
"""
Old placeholder for build creation. Now it just gets it from the database.
"""
if build_pk:
build = api.build(build_pk).get()
for key in ['project', 'version', 'resource_uri', 'absolute_uri']:
if key in build:
del build[key]
else:
build = {}
return build
def record_build(api, record, build, results, state):
"""
Record a build by hitting the API.
Returns nothing
"""
if not record:
return None
setup_steps = ["checkout", "venv", "sphinx", "requirements", "install"]
output_steps = ["html"]
all_steps = setup_steps + output_steps
build["state"] = state
if "html" in results:
build["success"] = results["html"][0] == 0
else:
build["success"] = False
# # Set global state
# for step in all_steps:
# if results.get(step, False):
# if results.get(step)[0] != 0:
# results['success'] = False
build["exit_code"] = max([results.get(step, [0])[0] for step in all_steps])
build["setup"] = build["setup_error"] = ""
build["output"] = build["error"] = ""
for step in setup_steps:
if step in results:
build["setup"] += "\n\n%s\n-----\n\n" % step
build["setup"] += results.get(step)[1]
build["setup_error"] += "\n\n%s\n-----\n\n" % step
build["setup_error"] += results.get(step)[2]
for step in output_steps:
if step in results:
build["output"] += "\n\n%s\n-----\n\n" % step
build["output"] += results.get(step)[1]
build["error"] += "\n\n%s\n-----\n\n" % step
build["error"] += results.get(step)[2]
try:
ret = api.build(build["id"]).put(build)
except Exception, e:
log.error("Unable to post a new build", exc_info=True)
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
"""
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands from
the shell, for example.
"""
###
# Handle passed in arguments
###
update_output = kwargs.get('update_output', {})
project_data = api.project(pk).get()
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
# Prevent saving the temporary Project instance
def new_save(*args, **kwargs):
log.warning("Called save on a non-real object.")
return 0
project.save = new_save
log.info("Building %s" % project)
if version_pk:
version_data = api.version(version_pk).get()
del version_data['resource_uri']
else:
#Create or use the 'latest' branch, which is the default for a project.
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
version_data = api.version(project.slug).get(slug='latest')['objects'][0]
del version_data['resource_uri']
except (slumber.exceptions.HttpClientError, IndexError) as exc:
#if exc.response.status_code in [404,500]:
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
del version_data['resource_uri']
except Exception as e:
log.info("Exception in creating version: %s" % e)
#raise e
version_data['project'] = project
version = Version(**version_data)
version.save = new_save
if not version_pk:
#Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data['project'] = "/api/v1/version/%s/" % version_data['project'].pk
api.version(version.pk).put(version_data)
if record:
#Create Build Object.
build = api.build.post(dict(
project= '/api/v1/project/%s/' % project.pk,
version= '/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
#Make Dirs
path = project.doc_path
if not os.path.exists(path):
os.makedirs(path)
with project.repo_lock(30):
if project.is_imported:
try:
update_output = update_imported_docs(project, version)
except ProjectImportError, err:
log.error("Failed to import project; skipping build.", exc_info=True)
build['state'] = 'finished'
build['setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
api.build(build['id']).put(build)
return False
else:
def update_docs(pk,
record=True,
pdf=True,
man=True,
epub=True,
dash=True,
version_pk=None,
force=False,
**kwargs):
"""The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we
created it. Then it will build the html docs and other requested parts. It
also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands
from the shell, for example.
"""
###
# Handle passed in arguments
###
project_data = api.project(pk).get()
project = make_api_project(project_data)
# Prevent saving the temporary Project instance
def new_save(*args, **kwargs):
log.warning("Called save on a non-real object.")
return 0
project.save = new_save
log.info("Building %s" % project)
if version_pk:
version_data = api.version(version_pk).get()
else:
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
# Use latest version
version_data = (api.version(
project.slug).get(slug='latest')['objects'][0])
except (slumber.exceptions.HttpClientError, IndexError):
# Create the latest version since it doesn't exist
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
except Exception as e:
log.info("Exception in creating version: %s" % e)
raise e
version = make_api_version(version_data)
version.save = new_save
if not version_pk:
# Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data['project'] = ("/api/v1/version/%s/" %
version_data['project'].pk)
api.version(version.pk).put(version_data)
if record:
# Create Build Object.
build = api.build.post(
dict(
project='/api/v1/project/%s/' % project.pk,
version='/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
try:
log.info("Updating docs from VCS")
update_output = update_imported_docs(version.pk)
#update_output = update_result.get()
except ProjectImportError, err:
log.error("Failed to import project; skipping build.", exc_info=True)
build['state'] = 'finished'
build['setup_error'] = ('Failed to import project; skipping build.\n'
'Please make sure your repo is correct and '
'you have a conf.py')
api.build(build['id']).put(build)
return False
if record:
# Update the build with info about the setup
build['state'] = 'building'
output_data = error_data = ''
# Grab all the text from updating the code via VCS.
for key in ['checkout', 'venv', 'sphinx', 'requirements', 'install']:
data = update_output.get(key, None)
if data:
try:
output_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[1])
error_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[2])
except UnicodeDecodeError:
log.debug("Unicode Error in setup")
build['setup'] = output_data
build['setup_error'] = error_data
api.build(build['id']).put(build)
log.info("Building docs")
# This is only checking the results of the HTML build, as it's a canary
try:
results = build_docs(version_pk=version.pk,
pdf=pdf,
man=man,
epub=epub,
dash=dash,
record=record,
force=force)
(html_results, latex_results, pdf_results, man_results, epub_results,
dash_results) = results
(ret, out, err) = html_results
except Exception as e:
if record:
# Update the build with info about the setup
build['state'] = 'building'
output_data = error_data = ''
# Grab all the text from updating the code via VCS.
for key in ['checkout', 'venv', 'sphinx', 'requirements', 'install']:
data = update_output.get(key, None)
if data:
try:
output_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[1])
error_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[2])
except UnicodeDecodeError:
log.debug("Unicode Error in setup")
build['setup'] = output_data
build['setup_error'] = error_data
api.build(build['id']).put(build)
log.info("Building docs")
# This is only checking the results of the HTML build, as it's a canary
try:
results = build_docs(version_pk=version.pk, pdf=pdf, man=man,
epub=epub, dash=dash, record=record, force=force)
(html_results, latex_results, pdf_results, man_results, epub_results,
dash_results) = results
(ret, out, err) = html_results
except Exception as e:
log.error("Exception in flailboat build_docs", exc_info=True)
html_results = (999, "Project build Failed", str(e))
latex_results = (999, "Project build Failed", str(e))
pdf_results = (999, "Project build Failed", str(e))
# These variables aren't currently being used.
def update_docs(pk,
record=True,
pdf=True,
man=True,
epub=True,
version_pk=None,
force=False,
**kwargs):
"""
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands from
the shell, for example.
"""
###
# Handle passed in arguments
###
update_output = kwargs.get('update_output', {})
project_data = api.project(pk).get()
del project_data['users']
del project_data['resource_uri']
del project_data['absolute_url']
project = Project(**project_data)
# Prevent saving the temporary Project instance
def new_save(*args, **kwargs):
log.warning("Called save on a non-real object.")
return 0
project.save = new_save
log.info("Building %s" % project)
if version_pk:
version_data = api.version(version_pk).get()
del version_data['resource_uri']
else:
#Create or use the 'latest' branch, which is the default for a project.
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
version_data = api.version(
project.slug).get(slug='latest')['objects'][0]
del version_data['resource_uri']
except (slumber.exceptions.HttpClientError, IndexError) as exc:
#if exc.response.status_code in [404,500]:
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
del version_data['resource_uri']
except Exception as e:
log.info("Exception in creating version: %s" % e)
#raise e
version_data['project'] = project
version = Version(**version_data)
version.save = new_save
if not version_pk:
#Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data[
'project'] = "/api/v1/version/%s/" % version_data['project'].pk
api.version(version.pk).put(version_data)
if record:
#Create Build Object.
build = api.build.post(
dict(
project='/api/v1/project/%s/' % project.pk,
version='/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
#Make Dirs
path = project.doc_path
if not os.path.exists(path):
os.makedirs(path)
with project.repo_lock(30):
if project.is_imported:
try:
update_output = update_imported_docs(project, version)
except ProjectImportError, err:
log.error("Failed to import project; skipping build.",
exc_info=True)
build['state'] = 'finished'
build[
'setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py'
api.build(build['id']).put(build)
return False
else:
def update_docs(pk, record=True, pdf=True, man=True, epub=True, dash=True,
search=True, version_pk=None, force=False, intersphinx=True,
localmedia=True, api=None, **kwargs):
"""The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we
created it. Then it will build the html docs and other requested parts. It
also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands
from the shell, for example.
"""
if api is None:
api = tastyapi.api
project_data = api.project(pk).get()
project = make_api_project(project_data)
if 'tryton' in project.repo:
# Skip for now
return
log.info(LOG_TEMPLATE.format(project=project.slug, version='', msg='Building'))
if version_pk:
version_data = api.version(version_pk).get()
else:
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
# Use latest version
version_data = (api.version(project.slug)
.get(slug='latest')['objects'][0])
except (slumber.exceptions.HttpClientError, IndexError):
# Create the latest version since it doesn't exist
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
type='branch',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
except Exception as e:
log.info(LOG_TEMPLATE.format(project=project.slug, version='', msg='Exception in creating version: %s' % e))
raise e
version = make_api_version(version_data)
if not version_pk:
# Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data['project'] = ("/api/v1/version/%s/"
% version_data['project'].pk)
api.version(version.pk).put(version_data)
if record:
# Create Build Object.
build = api.build.post(dict(
project='/api/v1/project/%s/' % project.pk,
version='/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
try:
log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Updating docs from VCS'))
update_output = update_imported_docs(version.pk, api)
#update_output = update_result.get()
except ProjectImportError, err:
log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True)
build['state'] = 'finished'
build['setup_error'] = (
'Failed to import project; skipping build.\n'
'\nError\n-----\n\n%s' % err.message
)
api.build(build['id']).put(build)
return False
def update_docs(pk,
record=True,
pdf=True,
man=True,
epub=True,
dash=True,
search=True,
version_pk=None,
force=False,
intersphinx=True,
localmedia=True,
api=None,
**kwargs):
"""The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we
created it. Then it will build the html docs and other requested parts. It
also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands
from the shell, for example.
"""
if api is None:
api = tastyapi.api
project_data = api.project(pk).get()
project = make_api_project(project_data)
if 'tryton' in project.repo:
# Skip for now
return
log.info(
LOG_TEMPLATE.format(project=project.slug, version='', msg='Building'))
if version_pk:
version_data = api.version(version_pk).get()
else:
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
# Use latest version
version_data = (api.version(
project.slug).get(slug='latest')['objects'][0])
except (slumber.exceptions.HttpClientError, IndexError):
# Create the latest version since it doesn't exist
version_data = dict(
project='/api/v1/project/%s/' % project.pk,
slug='latest',
type='branch',
active=True,
verbose_name='latest',
identifier=branch,
)
try:
version_data = api.version.post(version_data)
except Exception as e:
log.info(
LOG_TEMPLATE.format(
project=project.slug,
version='',
msg='Exception in creating version: %s' % e))
raise e
version = make_api_version(version_data)
if not version_pk:
# Lots of course correction.
to_save = False
if not version.verbose_name:
version_data['verbose_name'] = 'latest'
to_save = True
if not version.active:
version_data['active'] = True
to_save = True
if version.identifier != branch:
version_data['identifier'] = branch
to_save = True
if to_save:
version_data['project'] = ("/api/v1/version/%s/" %
version_data['project'].pk)
api.version(version.pk).put(version_data)
if record:
# Create Build Object.
build = api.build.post(
dict(
project='/api/v1/project/%s/' % project.pk,
version='/api/v1/version/%s/' % version.pk,
type='html',
state='triggered',
))
else:
build = {}
try:
log.info(
LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg='Updating docs from VCS'))
update_output = update_imported_docs(version.pk, api)
#update_output = update_result.get()
except ProjectImportError, err:
log.error(LOG_TEMPLATE.format(
project=project.slug,
version=version.slug,
msg='Failed to import project; skipping build'),
exc_info=True)
build['state'] = 'finished'
build['setup_error'] = ('Failed to import project; skipping build.\n'
'\nError\n-----\n\n%s' % err.message)
api.build(build['id']).put(build)
return False
def update_docs(
pk,
record=True,
pdf=True,
man=True,
epub=True,
dash=True,
search=True,
version_pk=None,
force=False,
intersphinx=True,
api=None,
**kwargs
):
"""The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we
created it. Then it will build the html docs and other requested parts. It
also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands
from the shell, for example.
"""
if api is None:
api = tastyapi.api
project_data = api.project(pk).get()
project = make_api_project(project_data)
# if 'edx-platform' in project.repo:
# Skip edx for now
# return
log.info(LOG_TEMPLATE.format(project=project.slug, version="", msg="Building"))
if version_pk:
version_data = api.version(version_pk).get()
else:
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
# Use latest version
version_data = api.version(project.slug).get(slug="latest")["objects"][0]
except (slumber.exceptions.HttpClientError, IndexError):
# Create the latest version since it doesn't exist
version_data = dict(
project="/api/v1/project/%s/" % project.pk,
slug="latest",
type="branch",
active=True,
verbose_name="latest",
identifier=branch,
)
try:
version_data = api.version.post(version_data)
except Exception as e:
log.info(
LOG_TEMPLATE.format(project=project.slug, version="", msg="Exception in creating version: %s" % e)
)
raise e
version = make_api_version(version_data)
if not version_pk:
# Lots of course correction.
to_save = False
if not version.verbose_name:
version_data["verbose_name"] = "latest"
to_save = True
if not version.active:
version_data["active"] = True
to_save = True
if version.identifier != branch:
version_data["identifier"] = branch
to_save = True
if to_save:
version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk
api.version(version.pk).put(version_data)
if record:
# Create Build Object.
build = api.build.post(
dict(
project="/api/v1/project/%s/" % project.pk,
version="/api/v1/version/%s/" % version.pk,
type="html",
state="triggered",
)
)
else:
build = {}
try:
log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="Updating docs from VCS"))
update_output = update_imported_docs(version.pk, api)
# update_output = update_result.get()
except ProjectImportError, err:
log.error(
LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="Failed to import project; skipping build"
),
exc_info=True,
)
build["state"] = "finished"
build["setup_error"] = "Failed to import project; skipping build.\n" "\nError\n-----\n\n%s" % err.message
api.build(build["id"]).put(build)
return False
if record:
# Update the build with info about the setup
build["state"] = "building"
output_data = error_data = ""
# Grab all the text from updating the code via VCS.
for key in ["checkout", "venv", "sphinx", "requirements", "install"]:
data = update_output.get(key, None)
if data:
try:
output_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[1])
error_data += u"\n\n%s\n\n%s\n\n" % (key.upper(), data[2])
except UnicodeDecodeError:
log.debug("Unicode Error in setup")
build["setup"] = output_data
build["setup_error"] = error_data
api.build(build["id"]).put(build)
log.info("Building docs")
# This is only checking the results of the HTML build, as it's a canary
try:
(html_results, latex_results, pdf_results, man_results, epub_results) = build_docs(
version_pk=version.pk, pdf=pdf, man=man, epub=epub, record=record, force=force
)
(ret, out, err) = html_results
except Exception as e:
log.error("Exception in flailboat build_docs", exc_info=True)
html_results = (999, "Project build Failed", str(e))
latex_results = (999, "Project build Failed", str(e))
pdf_results = (999, "Project build Failed", str(e))
man_results = (999, "Project build Failed", str(e))
epub_results = (999, "Project build Failed", str(e))
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs):
"""
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
`pk`
Primary key of the project to update
`record`
Whether or not to keep a record of the update in the database. Useful
for preventing changes visible to the end-user when running commands from
the shell, for example.
"""
###
# Handle passed in arguments
###
project_data = api.project(pk).get()
project = make_api_project(project_data)
# Prevent saving the temporary Project instance
def new_save(*args, **kwargs):
log.warning("Called save on a non-real object.")
return 0
project.save = new_save
log.info("Building %s" % project)
if version_pk:
version_data = api.version(version_pk).get()
else:
branch = project.default_branch or project.vcs_repo().fallback_branch
try:
# Use latest version
version_data = api.version(project.slug).get(slug="latest")["objects"][0]
except (slumber.exceptions.HttpClientError, IndexError):
# Create the latest version since it doesn't exist
version_data = dict(
project="/api/v1/project/%s/" % project.pk,
slug="latest",
active=True,
verbose_name="latest",
identifier=branch,
)
try:
version_data = api.version.post(version_data)
except Exception as e:
log.info("Exception in creating version: %s" % e)
raise e
version = make_api_version(version_data)
version.save = new_save
if not version_pk:
# Lots of course correction.
to_save = False
if not version.verbose_name:
version_data["verbose_name"] = "latest"
to_save = True
if not version.active:
version_data["active"] = True
to_save = True
if version.identifier != branch:
version_data["identifier"] = branch
to_save = True
if to_save:
version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk
api.version(version.pk).put(version_data)
if record:
# Create Build Object.
build = api.build.post(
dict(
project="/api/v1/project/%s/" % project.pk,
version="/api/v1/version/%s/" % version.pk,
type="html",
state="triggered",
)
)
else:
build = {}
try:
log.info("Updating docs from VCS")
update_output = update_imported_docs(version.pk)
# update_output = update_result.get()
except ProjectImportError, err:
log.error("Failed to import project; skipping build.", exc_info=True)
build["state"] = "finished"
build[
"setup_error"
] = "Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py"
api.build(build["id"]).put(build)
return False
def record_build(api, record, build, results, state, start_time=None):
"""
Record a build by hitting the API.
Returns nothing
"""
if not record:
return None
build["builder"] = socket.gethostname()
setup_steps = ["checkout", "venv", "doc_builder", "requirements", "install"]
output_steps = ["html"]
all_steps = setup_steps + output_steps
build["state"] = state
if "html" in results:
build["success"] = results["html"][0] == 0
else:
build["success"] = False
# Set global state
# for step in all_steps:
# if results.get(step, False):
# if results.get(step)[0] != 0:
# results['success'] = False
build["exit_code"] = max([results.get(step, [0])[0] for step in all_steps])
build["setup"] = build["setup_error"] = ""
build["output"] = build["error"] = ""
if start_time:
build["length"] = (datetime.datetime.utcnow() - start_time).total_seconds()
for step in setup_steps:
if step in results:
build["setup"] += "\n\n%s\n-----\n\n" % step
try:
build["setup"] += results.get(step)[1]
except (IndexError, TypeError):
pass
build["setup_error"] += "\n\n%s\n-----\n\n" % step
try:
build["setup_error"] += results.get(step)[2]
except (IndexError, TypeError):
pass
for step in output_steps:
if step in results:
build["output"] += "\n\n%s\n-----\n\n" % step
try:
build["output"] += results.get(step)[1]
except (IndexError, TypeError):
pass
build["error"] += "\n\n%s\n-----\n\n" % step
try:
build["error"] += results.get(step)[2]
except (IndexError, TypeError):
pass
# Attempt to stop unicode errors on build reporting
for key, val in build.items():
if isinstance(val, basestring):
build[key] = val.decode("utf-8", "ignore")
try:
api.build(build["id"]).put(build)
except Exception:
log.error("Unable to post a new build", exc_info=True)
