def test_local_build(self): test_dir = os.path.dirname(os.path.realpath(__file__)) docs_dir = os.path.normpath(os.path.join(test_dir, os.pardir, os.pardir, 'docs')) state = BuildState(root=docs_dir) BuilderClass = loading.get('sphinx') builder = BuilderClass(state=state) builder.append_conf() builder.build() self.assertIn('index.html', os.listdir(state.output_path)) shutil.rmtree(state.output_path)
def test_local_build(self): test_dir = os.path.dirname(os.path.realpath(__file__)) docs_dir = os.path.normpath( os.path.join(test_dir, os.pardir, os.pardir, 'docs')) state = BuildState(root=docs_dir) BuilderClass = loading.get('sphinx') builder = BuilderClass(state=state) builder.append_conf() builder.build() self.assertIn('index.html', os.listdir(state.output_path)) shutil.rmtree(state.output_path)
def test_total_build(self): self.root = tempfile.mkdtemp() repo_url = 'file://' + subprocess.check_output('git rev-parse --show-toplevel', shell=True).strip() state = BuildState(root=self.root, repo=repo_url) BuilderClass = loading.get('sphinx') builder = BuilderClass(state=state) builder.checkout_code() builder.setup_environment() builder.append_conf() builder.build() self.assertIn('index.html', os.listdir(state.output_path)) shutil.rmtree(self.root)
def test_total_build(self): self.root = tempfile.mkdtemp() repo_url = 'file://' + subprocess.check_output( 'git rev-parse --show-toplevel', shell=True).strip() state = BuildState(root=self.root, repo=repo_url) BuilderClass = loading.get('sphinx') builder = BuilderClass(state=state) builder.checkout_code() builder.setup_environment() builder.append_conf() builder.build() self.assertIn('index.html', os.listdir(state.output_path)) shutil.rmtree(self.root)
def build_docs(version, force, pdf, man, epub, dash, search, localmedia): """ This handles the actual building of the documentation """ project = version.project results = {} before_build.send(sender=version) with project.repo_nonblockinglock(version=version, max_lock_age=getattr( settings, 'REPO_LOCK_SECONDS', 30)): html_builder = builder_loading.get(project.documentation_type)(version) if force: html_builder.force() html_builder.append_conf() results['html'] = html_builder.build() if results['html'][0] == 0: html_builder.move() # Gracefully attempt to move files via task on web workers. try: move_files.delay( version_pk=version.pk, html=True, hostname=socket.gethostname(), ) except socket.error: pass fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build") if 'mkdocs' in project.documentation_type: if search: try: search_builder = builder_loading.get('mkdocs_json')( version) results['search'] = search_builder.build() if results['search'][0] == 0: search_builder.move() except: log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) if 'sphinx' in project.documentation_type: # Search builder. Creates JSON from docs and sends it to the # server. if search: try: search_builder = builder_loading.get('sphinx_search')( version) results['search'] = search_builder.build() if results['search'][0] == 0: # Copy json for safe keeping search_builder.move() except: log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) # Local media builder for singlepage HTML download archive if localmedia: try: localmedia_builder = builder_loading.get( 'sphinx_singlehtmllocalmedia')(version) results['localmedia'] = localmedia_builder.build() if results['localmedia'][0] == 0: localmedia_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True) # Optional build steps if version.project.slug not in HTML_ONLY and not project.skip: if pdf: pdf_builder = builder_loading.get('sphinx_pdf')(version) results['pdf'] = pdf_builder.build() # Always move pdf results even when there's an error. # if pdf_results[0] == 0: pdf_builder.move() else: results['pdf'] = fake_results if epub: epub_builder = builder_loading.get('sphinx_epub')(version) results['epub'] = epub_builder.build() if results['epub'][0] == 0: epub_builder.move() else: results['epub'] = fake_results after_build.send(sender=version) return results
def setup_environment(version): """ Build the virtualenv and install the project into it. Always build projects with a virtualenv. """ ret_dict = {} project = version.project build_dir = os.path.join(project.venv_path(version=version.slug), 'build') if os.path.exists(build_dir): log.info( LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Removing existing build dir')) shutil.rmtree(build_dir) if project.use_system_packages: site_packages = '--system-site-packages' else: site_packages = '--no-site-packages' # Here the command has been modified to support different # interpreters. ret_dict['venv'] = run('{cmd} {site_packages} {path}'.format( cmd='virtualenv-2.7 -p {interpreter}'.format( interpreter=project.python_interpreter), site_packages=site_packages, path=project.venv_path(version=version.slug))) # Other code expects sphinx-build to be installed inside the # virtualenv. Using the -I option makes sure it gets installed # even if it is already installed system-wide (and # --system-site-packages is used) if project.use_system_packages: ignore_option = '-I' else: ignore_option = '' wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels') ret_dict['doc_builder'] = run(( '{cmd} install --use-wheel --find-links={wheeldir} -U {ignore_option} ' 'sphinx==1.2.2 virtualenv==1.10.1 setuptools==1.1 docutils==0.11 mkdocs==0.11.1 mock==1.0.1 pillow==2.6.1' ' readthedocs-sphinx-ext==0.4.4 sphinx-rtd-theme==0.1.6 ').format( cmd=project.venv_bin(version=version.slug, bin='pip'), ignore_option=ignore_option, wheeldir=wheeldir, )) # Handle requirements requirements_file_path = project.requirements_file checkout_path = project.checkout_path(version.slug) if not requirements_file_path: docs_dir = builder_loading.get( project.documentation_type)(version).docs_dir() for path in [docs_dir, '']: for req_file in ['pip_requirements.txt', 'requirements.txt']: test_path = os.path.join(checkout_path, path, req_file) print('Testing %s' % test_path) if os.path.exists(test_path): requirements_file_path = test_path break if requirements_file_path: os.chdir(checkout_path) ret_dict['requirements'] = run( '{cmd} install --exists-action=w -r {requirements}'.format( cmd=project.venv_bin(version=version.slug, bin='pip'), requirements=requirements_file_path)) # Handle setup.py os.chdir(project.checkout_path(version.slug)) if os.path.isfile("setup.py"): if getattr(settings, 'USE_PIP_INSTALL', False): ret_dict['install'] = run( '{cmd} install --ignore-installed .'.format( cmd=project.venv_bin(version=version.slug, bin='pip'))) else: ret_dict['install'] = run('{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version.slug, bin='python'))) else: ret_dict['install'] = (999, "", "No setup.py, skipping install") return ret_dict
def build_docs(version, force, pdf, man, epub, dash, search, localmedia): """ This handles the actual building of the documentation """ project = version.project results = {} if 'sphinx' in project.documentation_type: try: project.conf_file(version.slug) except ProjectImportError: results['html'] = (999, 'Conf file not found.', '') return results with project.repo_nonblockinglock(version=version, max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)): html_builder = builder_loading.get(project.documentation_type)(version) if force: html_builder.force() html_builder.append_conf() results['html'] = html_builder.build() if results['html'][0] == 0: html_builder.move() fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build") if 'mkdocs' in project.documentation_type: if search: try: search_builder = builder_loading.get('mkdocs_json')(version) results['search'] = search_builder.build() if results['search'][0] == 0: search_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) if 'sphinx' in project.documentation_type: # Search builder. Creates JSON from docs and sends it to the # server. if search: try: search_builder = builder_loading.get( 'sphinx_search')(version) results['search'] = search_builder.build() if results['search'][0] == 0: # Copy json for safe keeping search_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) # Local media builder for singlepage HTML download archive if localmedia: try: localmedia_builder = builder_loading.get( 'sphinx_singlehtmllocalmedia')(version) results['localmedia'] = localmedia_builder.build() if results['localmedia'][0] == 0: localmedia_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True) # Optional build steps if version.project.slug not in HTML_ONLY and not project.skip: if pdf: pdf_builder = builder_loading.get('sphinx_pdf')(version) results['pdf'] = pdf_builder.build() # Always move pdf results even when there's an error. # if pdf_results[0] == 0: pdf_builder.move() else: results['pdf'] = fake_results if epub: epub_builder = builder_loading.get('sphinx_epub')(version) results['epub'] = epub_builder.build() if results['epub'][0] == 0: epub_builder.move() else: results['epub'] = fake_results return results
def doc_builder(self): return loading.get(self.documentation_type)
def run_build(self, force=False): """ This handles the actual building of the documentation """ from doc_builder.loader import loading as builder_loading from projects.tasks import move_files results = {} if signals.USE_SIGNALS: signals.before_build.send(sender=self.state) with NonBlockingLock(version=self.state.version, project=self.state.project, doc_path=self.state.root, max_lock_age=self.state.REPO_LOCK_SECONDS): html_builder = builder_loading.get(self.state.documentation_type)( self.state) if force: html_builder.force() html_builder.append_conf() results['html'] = html_builder.build() if results['html'][0] == 0: html_builder.move() # Gracefully attempt to move files via task on web workers. try: move_files.delay( state=self.state, html=True, hostname=socket.gethostname(), ) except socket.error: pass fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build") if 'mkdocs' in self.state.documentation_type: if 'search' in self.state.build_types: try: search_builder = builder_loading.get('mkdocs_json')( self.state) results['search'] = search_builder.build() if results['search'][0] == 0: search_builder.move() except: log.error(LOG_TEMPLATE.format( project=self.state.project, version=self.state.version, msg="JSON Build Error"), exc_info=True) if 'sphinx' in self.state.documentation_type: # Search builder. Creates JSON from docs and sends it to the # server. if 'search' in self.state.build_types: try: search_builder = builder_loading.get('sphinx_search')( self.state) results['search'] = search_builder.build() if results['search'][0] == 0: # Copy json for safe keeping search_builder.move() except: log.error(LOG_TEMPLATE.format( project=self.state.project, version=self.state.version, msg="JSON Build Error"), exc_info=True) # Local media builder for singlepage HTML download archive if 'localmedia' in self.state.build_types: try: localmedia_builder = builder_loading.get( 'sphinx_singlehtmllocalmedia')(self.state) results['localmedia'] = localmedia_builder.build() if results['localmedia'][0] == 0: localmedia_builder.move() except: log.error(LOG_TEMPLATE.format( project=self.state.project, version=self.state.version, msg="Local Media HTML Build Error"), exc_info=True) # Optional build steps if self.state.project not in self.state.HTML_ONLY: if 'pdf' in self.state.build_types: pdf_builder = builder_loading.get('sphinx_pdf')( self.state) results['pdf'] = pdf_builder.build() # Always move pdf results even when there's an error. # if pdf_results[0] == 0: pdf_builder.move() else: results['pdf'] = fake_results if 'epub' in self.state.build_types: epub_builder = builder_loading.get('sphinx_epub')( self.state) results['epub'] = epub_builder.build() if results['epub'][0] == 0: epub_builder.move() else: results['epub'] = fake_results if signals.USE_SIGNALS: signals.after_build.send(sender=self.state) return results
def setup_environment(version): """ Build the virtualenv and install the project into it. Always build projects with a virtualenv. """ ret_dict = {} project = version.project build_dir = os.path.join(project.venv_path(version=version.slug), 'build') if os.path.exists(build_dir): log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Removing existing build dir')) shutil.rmtree(build_dir) if project.use_system_packages: site_packages = '--system-site-packages' else: site_packages = '--no-site-packages' # Here the command has been modified to support different # interpreters. ret_dict['venv'] = run( '{cmd} {site_packages} {path}'.format( cmd='virtualenv-2.7 -p {interpreter}'.format( interpreter=project.python_interpreter), site_packages=site_packages, path=project.venv_path(version=version.slug) ) ) # Other code expects sphinx-build to be installed inside the # virtualenv. Using the -I option makes sure it gets installed # even if it is already installed system-wide (and # --system-site-packages is used) if project.use_system_packages: ignore_option = '-I' else: ignore_option = '' requirements = ' '.join([ 'sphinx==1.3.1', 'Pygments==2.0.2', 'virtualenv==1.10.1', 'setuptools==1.1', 'docutils==0.11', 'mkdocs==0.11.1', 'mock==1.0.1', 'pillow==2.6.1', 'readthedocs-sphinx-ext==0.4.4', 'sphinx-rtd-theme==0.1.8', 'recommonmark==0.1.1', ]) wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels') ret_dict['doc_builder'] = run( ( '{cmd} install --use-wheel --find-links={wheeldir} -U ' '{ignore_option} {requirements}' ).format( cmd=project.venv_bin(version=version.slug, bin='pip'), ignore_option=ignore_option, wheeldir=wheeldir, requirements=requirements, ) ) # Handle requirements requirements_file_path = project.requirements_file checkout_path = project.checkout_path(version.slug) if not requirements_file_path: docs_dir = builder_loading.get(project.documentation_type)(version).docs_dir() for path in [docs_dir, '']: for req_file in ['pip_requirements.txt', 'requirements.txt']: test_path = os.path.join(checkout_path, path, req_file) print('Testing %s' % test_path) if os.path.exists(test_path): requirements_file_path = test_path break if requirements_file_path: os.chdir(checkout_path) ret_dict['requirements'] = run( '{cmd} install --exists-action=w -r {requirements}'.format( cmd=project.venv_bin(version=version.slug, bin='pip'), requirements=requirements_file_path)) # Handle setup.py os.chdir(project.checkout_path(version.slug)) if os.path.isfile("setup.py"): if getattr(settings, 'USE_PIP_INSTALL', False): ret_dict['install'] = run( '{cmd} install --ignore-installed .'.format( cmd=project.venv_bin(version=version.slug, bin='pip'))) else: ret_dict['install'] = run( '{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version.slug, bin='python'))) else: ret_dict['install'] = (999, "", "No setup.py, skipping install") return ret_dict
def main(args, options=None): """ Processes build specific options. Uses a readthedocs.yml file to specify build-time settings. :param: output - Where files get rendered :param: full - Do a full build, creating a temp directory and use a virtualenv :param: config - Where the readthedocs.yml config is located """ output_path = options.get('output', None) full = options.get('full', False) _open = options.get('open', False) config = options.get('config', None) serve = options.get('serve', None) if full: temp = env = True else: temp = options.get('temp', False) env = options.get('env', False) # Read file config if config: print "Using config %s" % config state = BuildState.from_build_state(config) else: state = BuildState(root=os.getcwd()) matches = state.full_find('readthedocs.yml') if len(matches) == 1: print "Using config %s" % matches[0] state = BuildState.from_build_state(matches[0]) elif len(matches) > 1: print "ERROR: Multiple config files found: " print '\n'.join(matches) sys.exit(1) else: print "No configuration file given, using defaults" if serve: os.chdir(state.output_path) Handler = SimpleHTTPServer.SimpleHTTPRequestHandler httpd = SocketServer.TCPServer(("", 8001), Handler) print "serving at port", 8001 httpd.serve_forever() if temp: docs_dir = tempfile.mkdtemp() repo_url = 'file://' + subprocess.check_output( 'git rev-parse --show-toplevel', shell=True).strip() state.docs_dir = docs_dir state.repo_url = repo_url #state = BuildState(root=docs_dir, repo=repo_url, output_path=output_path) # else: # docs_dir = os.getcwd() # state = BuildState(root=docs_dir, output_path=output_path) # State keeping if output_path: if os.path.isabs(output_path): state.output_path = output_path else: state.output_path = os.path.normpath( os.path.join(os.getcwd(), output_path)) os.chdir(state.root) BuilderClass = loading.get(state.documentation_type) builder = BuilderClass(state=state) print "Building docs to " + state.output_path if temp: print "Checking out code to %s" % docs_dir builder.checkout_code() if env: print "Setting up virtualenv" builder.setup_environment() try: builder.append_conf() builder.build() finally: builder.clean_conf() if _open: to_open = 'file://' + os.path.join(state.output_path, 'index.html') print "Opening %s" % to_open webbrowser.open(to_open)