def test_local_build(self):
test_dir = os.path.dirname(os.path.realpath(__file__))
docs_dir = os.path.normpath(os.path.join(test_dir, os.pardir, os.pardir, 'docs'))
state = BuildState(root=docs_dir)
BuilderClass = loading.get('sphinx')
builder = BuilderClass(state=state)
builder.append_conf()
builder.build()
self.assertIn('index.html', os.listdir(state.output_path))
shutil.rmtree(state.output_path)
def test_local_build(self):
test_dir = os.path.dirname(os.path.realpath(__file__))
docs_dir = os.path.normpath(
os.path.join(test_dir, os.pardir, os.pardir, 'docs'))
state = BuildState(root=docs_dir)
BuilderClass = loading.get('sphinx')
builder = BuilderClass(state=state)
builder.append_conf()
builder.build()
self.assertIn('index.html', os.listdir(state.output_path))
shutil.rmtree(state.output_path)
def test_total_build(self):
self.root = tempfile.mkdtemp()
repo_url = 'file://' + subprocess.check_output('git rev-parse --show-toplevel', shell=True).strip()
state = BuildState(root=self.root, repo=repo_url)
BuilderClass = loading.get('sphinx')
builder = BuilderClass(state=state)
builder.checkout_code()
builder.setup_environment()
builder.append_conf()
builder.build()
self.assertIn('index.html', os.listdir(state.output_path))
shutil.rmtree(self.root)
def test_total_build(self):
self.root = tempfile.mkdtemp()
repo_url = 'file://' + subprocess.check_output(
'git rev-parse --show-toplevel', shell=True).strip()
state = BuildState(root=self.root, repo=repo_url)
BuilderClass = loading.get('sphinx')
builder = BuilderClass(state=state)
builder.checkout_code()
builder.setup_environment()
builder.append_conf()
builder.build()
self.assertIn('index.html', os.listdir(state.output_path))
shutil.rmtree(self.root)
def build_docs(version, force, pdf, man, epub, dash, search, localmedia):
"""
This handles the actual building of the documentation
"""
project = version.project
results = {}
before_build.send(sender=version)
with project.repo_nonblockinglock(version=version,
max_lock_age=getattr(
settings, 'REPO_LOCK_SECONDS', 30)):
html_builder = builder_loading.get(project.documentation_type)(version)
if force:
html_builder.force()
html_builder.append_conf()
results['html'] = html_builder.build()
if results['html'][0] == 0:
html_builder.move()
# Gracefully attempt to move files via task on web workers.
try:
move_files.delay(
version_pk=version.pk,
html=True,
hostname=socket.gethostname(),
)
except socket.error:
pass
fake_results = (999, "Project Skipped, Didn't build",
"Project Skipped, Didn't build")
if 'mkdocs' in project.documentation_type:
if search:
try:
search_builder = builder_loading.get('mkdocs_json')(
version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg="JSON Build Error"),
exc_info=True)
if 'sphinx' in project.documentation_type:
# Search builder. Creates JSON from docs and sends it to the
# server.
if search:
try:
search_builder = builder_loading.get('sphinx_search')(
version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
# Copy json for safe keeping
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg="JSON Build Error"),
exc_info=True)
# Local media builder for singlepage HTML download archive
if localmedia:
try:
localmedia_builder = builder_loading.get(
'sphinx_singlehtmllocalmedia')(version)
results['localmedia'] = localmedia_builder.build()
if results['localmedia'][0] == 0:
localmedia_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug,
version=version.slug,
msg="Local Media HTML Build Error"),
exc_info=True)
# Optional build steps
if version.project.slug not in HTML_ONLY and not project.skip:
if pdf:
pdf_builder = builder_loading.get('sphinx_pdf')(version)
results['pdf'] = pdf_builder.build()
# Always move pdf results even when there's an error.
# if pdf_results[0] == 0:
pdf_builder.move()
else:
results['pdf'] = fake_results
if epub:
epub_builder = builder_loading.get('sphinx_epub')(version)
results['epub'] = epub_builder.build()
if results['epub'][0] == 0:
epub_builder.move()
else:
results['epub'] = fake_results
after_build.send(sender=version)
return results
def setup_environment(version):
"""
Build the virtualenv and install the project into it.
Always build projects with a virtualenv.
"""
ret_dict = {}
project = version.project
build_dir = os.path.join(project.venv_path(version=version.slug), 'build')
if os.path.exists(build_dir):
log.info(
LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg='Removing existing build dir'))
shutil.rmtree(build_dir)
if project.use_system_packages:
site_packages = '--system-site-packages'
else:
site_packages = '--no-site-packages'
# Here the command has been modified to support different
# interpreters.
ret_dict['venv'] = run('{cmd} {site_packages} {path}'.format(
cmd='virtualenv-2.7 -p {interpreter}'.format(
interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version.slug)))
# Other code expects sphinx-build to be installed inside the
# virtualenv. Using the -I option makes sure it gets installed
# even if it is already installed system-wide (and
# --system-site-packages is used)
if project.use_system_packages:
ignore_option = '-I'
else:
ignore_option = ''
wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels')
ret_dict['doc_builder'] = run((
'{cmd} install --use-wheel --find-links={wheeldir} -U {ignore_option} '
'sphinx==1.2.2 virtualenv==1.10.1 setuptools==1.1 docutils==0.11 mkdocs==0.11.1 mock==1.0.1 pillow==2.6.1'
' readthedocs-sphinx-ext==0.4.4 sphinx-rtd-theme==0.1.6 ').format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
ignore_option=ignore_option,
wheeldir=wheeldir,
))
# Handle requirements
requirements_file_path = project.requirements_file
checkout_path = project.checkout_path(version.slug)
if not requirements_file_path:
docs_dir = builder_loading.get(
project.documentation_type)(version).docs_dir()
for path in [docs_dir, '']:
for req_file in ['pip_requirements.txt', 'requirements.txt']:
test_path = os.path.join(checkout_path, path, req_file)
print('Testing %s' % test_path)
if os.path.exists(test_path):
requirements_file_path = test_path
break
if requirements_file_path:
os.chdir(checkout_path)
ret_dict['requirements'] = run(
'{cmd} install --exists-action=w -r {requirements}'.format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
requirements=requirements_file_path))
# Handle setup.py
os.chdir(project.checkout_path(version.slug))
if os.path.isfile("setup.py"):
if getattr(settings, 'USE_PIP_INSTALL', False):
ret_dict['install'] = run(
'{cmd} install --ignore-installed .'.format(
cmd=project.venv_bin(version=version.slug, bin='pip')))
else:
ret_dict['install'] = run('{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version.slug, bin='python')))
else:
ret_dict['install'] = (999, "", "No setup.py, skipping install")
return ret_dict
def build_docs(version, force, pdf, man, epub, dash, search, localmedia):
"""
This handles the actual building of the documentation
"""
project = version.project
results = {}
if 'sphinx' in project.documentation_type:
try:
project.conf_file(version.slug)
except ProjectImportError:
results['html'] = (999, 'Conf file not found.', '')
return results
with project.repo_nonblockinglock(version=version,
max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):
html_builder = builder_loading.get(project.documentation_type)(version)
if force:
html_builder.force()
html_builder.append_conf()
results['html'] = html_builder.build()
if results['html'][0] == 0:
html_builder.move()
fake_results = (999, "Project Skipped, Didn't build",
"Project Skipped, Didn't build")
if 'mkdocs' in project.documentation_type:
if search:
try:
search_builder = builder_loading.get('mkdocs_json')(version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True)
if 'sphinx' in project.documentation_type:
# Search builder. Creates JSON from docs and sends it to the
# server.
if search:
try:
search_builder = builder_loading.get(
'sphinx_search')(version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
# Copy json for safe keeping
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True)
# Local media builder for singlepage HTML download archive
if localmedia:
try:
localmedia_builder = builder_loading.get(
'sphinx_singlehtmllocalmedia')(version)
results['localmedia'] = localmedia_builder.build()
if results['localmedia'][0] == 0:
localmedia_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True)
# Optional build steps
if version.project.slug not in HTML_ONLY and not project.skip:
if pdf:
pdf_builder = builder_loading.get('sphinx_pdf')(version)
results['pdf'] = pdf_builder.build()
# Always move pdf results even when there's an error.
# if pdf_results[0] == 0:
pdf_builder.move()
else:
results['pdf'] = fake_results
if epub:
epub_builder = builder_loading.get('sphinx_epub')(version)
results['epub'] = epub_builder.build()
if results['epub'][0] == 0:
epub_builder.move()
else:
results['epub'] = fake_results
return results
def doc_builder(self):
return loading.get(self.documentation_type)
def doc_builder(self):
return loading.get(self.documentation_type)
def run_build(self, force=False):
"""
This handles the actual building of the documentation
"""
from doc_builder.loader import loading as builder_loading
from projects.tasks import move_files
results = {}
if signals.USE_SIGNALS:
signals.before_build.send(sender=self.state)
with NonBlockingLock(version=self.state.version,
project=self.state.project,
doc_path=self.state.root,
max_lock_age=self.state.REPO_LOCK_SECONDS):
html_builder = builder_loading.get(self.state.documentation_type)(
self.state)
if force:
html_builder.force()
html_builder.append_conf()
results['html'] = html_builder.build()
if results['html'][0] == 0:
html_builder.move()
# Gracefully attempt to move files via task on web workers.
try:
move_files.delay(
state=self.state,
html=True,
hostname=socket.gethostname(),
)
except socket.error:
pass
fake_results = (999, "Project Skipped, Didn't build",
"Project Skipped, Didn't build")
if 'mkdocs' in self.state.documentation_type:
if 'search' in self.state.build_types:
try:
search_builder = builder_loading.get('mkdocs_json')(
self.state)
results['search'] = search_builder.build()
if results['search'][0] == 0:
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=self.state.project,
version=self.state.version,
msg="JSON Build Error"),
exc_info=True)
if 'sphinx' in self.state.documentation_type:
# Search builder. Creates JSON from docs and sends it to the
# server.
if 'search' in self.state.build_types:
try:
search_builder = builder_loading.get('sphinx_search')(
self.state)
results['search'] = search_builder.build()
if results['search'][0] == 0:
# Copy json for safe keeping
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=self.state.project,
version=self.state.version,
msg="JSON Build Error"),
exc_info=True)
# Local media builder for singlepage HTML download archive
if 'localmedia' in self.state.build_types:
try:
localmedia_builder = builder_loading.get(
'sphinx_singlehtmllocalmedia')(self.state)
results['localmedia'] = localmedia_builder.build()
if results['localmedia'][0] == 0:
localmedia_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=self.state.project,
version=self.state.version,
msg="Local Media HTML Build Error"),
exc_info=True)
# Optional build steps
if self.state.project not in self.state.HTML_ONLY:
if 'pdf' in self.state.build_types:
pdf_builder = builder_loading.get('sphinx_pdf')(
self.state)
results['pdf'] = pdf_builder.build()
# Always move pdf results even when there's an error.
# if pdf_results[0] == 0:
pdf_builder.move()
else:
results['pdf'] = fake_results
if 'epub' in self.state.build_types:
epub_builder = builder_loading.get('sphinx_epub')(
self.state)
results['epub'] = epub_builder.build()
if results['epub'][0] == 0:
epub_builder.move()
else:
results['epub'] = fake_results
if signals.USE_SIGNALS:
signals.after_build.send(sender=self.state)
return results
def setup_environment(version):
"""
Build the virtualenv and install the project into it.
Always build projects with a virtualenv.
"""
ret_dict = {}
project = version.project
build_dir = os.path.join(project.venv_path(version=version.slug), 'build')
if os.path.exists(build_dir):
log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Removing existing build dir'))
shutil.rmtree(build_dir)
if project.use_system_packages:
site_packages = '--system-site-packages'
else:
site_packages = '--no-site-packages'
# Here the command has been modified to support different
# interpreters.
ret_dict['venv'] = run(
'{cmd} {site_packages} {path}'.format(
cmd='virtualenv-2.7 -p {interpreter}'.format(
interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version.slug)
)
)
# Other code expects sphinx-build to be installed inside the
# virtualenv. Using the -I option makes sure it gets installed
# even if it is already installed system-wide (and
# --system-site-packages is used)
if project.use_system_packages:
ignore_option = '-I'
else:
ignore_option = ''
requirements = ' '.join([
'sphinx==1.3.1',
'Pygments==2.0.2',
'virtualenv==1.10.1',
'setuptools==1.1',
'docutils==0.11',
'mkdocs==0.11.1',
'mock==1.0.1',
'pillow==2.6.1',
'readthedocs-sphinx-ext==0.4.4',
'sphinx-rtd-theme==0.1.8',
'recommonmark==0.1.1',
])
wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels')
ret_dict['doc_builder'] = run(
(
'{cmd} install --use-wheel --find-links={wheeldir} -U '
'{ignore_option} {requirements}'
).format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
ignore_option=ignore_option,
wheeldir=wheeldir,
requirements=requirements,
)
)
# Handle requirements
requirements_file_path = project.requirements_file
checkout_path = project.checkout_path(version.slug)
if not requirements_file_path:
docs_dir = builder_loading.get(project.documentation_type)(version).docs_dir()
for path in [docs_dir, '']:
for req_file in ['pip_requirements.txt', 'requirements.txt']:
test_path = os.path.join(checkout_path, path, req_file)
print('Testing %s' % test_path)
if os.path.exists(test_path):
requirements_file_path = test_path
break
if requirements_file_path:
os.chdir(checkout_path)
ret_dict['requirements'] = run(
'{cmd} install --exists-action=w -r {requirements}'.format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
requirements=requirements_file_path))
# Handle setup.py
os.chdir(project.checkout_path(version.slug))
if os.path.isfile("setup.py"):
if getattr(settings, 'USE_PIP_INSTALL', False):
ret_dict['install'] = run(
'{cmd} install --ignore-installed .'.format(
cmd=project.venv_bin(version=version.slug, bin='pip')))
else:
ret_dict['install'] = run(
'{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version.slug,
bin='python')))
else:
ret_dict['install'] = (999, "", "No setup.py, skipping install")
return ret_dict
def main(args, options=None):
"""
Processes build specific options.
Uses a readthedocs.yml file to specify build-time settings.
:param: output - Where files get rendered
:param: full - Do a full build, creating a temp directory and use a virtualenv
:param: config - Where the readthedocs.yml config is located
"""
output_path = options.get('output', None)
full = options.get('full', False)
_open = options.get('open', False)
config = options.get('config', None)
serve = options.get('serve', None)
if full:
temp = env = True
else:
temp = options.get('temp', False)
env = options.get('env', False)
# Read file config
if config:
print "Using config %s" % config
state = BuildState.from_build_state(config)
else:
state = BuildState(root=os.getcwd())
matches = state.full_find('readthedocs.yml')
if len(matches) == 1:
print "Using config %s" % matches[0]
state = BuildState.from_build_state(matches[0])
elif len(matches) > 1:
print "ERROR: Multiple config files found: "
print '\n'.join(matches)
sys.exit(1)
else:
print "No configuration file given, using defaults"
if serve:
os.chdir(state.output_path)
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", 8001), Handler)
print "serving at port", 8001
httpd.serve_forever()
if temp:
docs_dir = tempfile.mkdtemp()
repo_url = 'file://' + subprocess.check_output(
'git rev-parse --show-toplevel', shell=True).strip()
state.docs_dir = docs_dir
state.repo_url = repo_url
#state = BuildState(root=docs_dir, repo=repo_url, output_path=output_path)
# else:
# docs_dir = os.getcwd()
# state = BuildState(root=docs_dir, output_path=output_path)
# State keeping
if output_path:
if os.path.isabs(output_path):
state.output_path = output_path
else:
state.output_path = os.path.normpath(
os.path.join(os.getcwd(), output_path))
os.chdir(state.root)
BuilderClass = loading.get(state.documentation_type)
builder = BuilderClass(state=state)
print "Building docs to " + state.output_path
if temp:
print "Checking out code to %s" % docs_dir
builder.checkout_code()
if env:
print "Setting up virtualenv"
builder.setup_environment()
try:
builder.append_conf()
builder.build()
finally:
builder.clean_conf()
if _open:
to_open = 'file://' + os.path.join(state.output_path, 'index.html')
print "Opening %s" % to_open
webbrowser.open(to_open)
