def test_build(self, mock_Popen, mock_api_versions, mock_chdir,
mock_apiv2_downloads):
# subprocess mock logic
mock_process = mock.Mock()
process_return_dict = {'communicate.return_value': ('SOMEGITHASH', '')}
mock_process.configure_mock(**process_return_dict)
mock_Popen.return_value = mock_process
mock_Popen.side_effect = build_subprocess_side_effect
project = ProjectFactory(allow_comments=True)
version = project.versions.all()[0]
mock_api_versions.return_value = [version]
mock_apiv2_downloads.get.return_value = {'downloads': "no_url_here"}
conf_path = os.path.join(project.checkout_path(version.slug),
project.conf_py_file)
# Mock open to simulate existing conf.py file
with mock.patch('codecs.open', mock.mock_open(), create=True):
with fake_paths(conf_path):
built_docs = build_docs(
version,
False,
False,
False,
)
builder_class = get_builder_class(project.documentation_type)
builder = builder_class(version)
self.assertIn(builder.sphinx_builder,
str(mock_Popen.call_args_list[1]))
def test_build(self, mock_Popen, mock_api_versions, mock_chdir, mock_apiv2_downloads):
# subprocess mock logic
mock_process = mock.Mock()
process_return_dict = {'communicate.return_value': ('SOMEGITHASH', '')}
mock_process.configure_mock(**process_return_dict)
mock_Popen.return_value = mock_process
mock_Popen.side_effect = build_subprocess_side_effect
project = ProjectFactory(allow_comments=True)
version = project.versions.all()[0]
mock_api_versions.return_value = [version]
mock_apiv2_downloads.get.return_value = {'downloads': "no_url_here"}
conf_path = os.path.join(
project.checkout_path(version.slug),
project.conf_py_file)
# Mock open to simulate existing conf.py file
with mock.patch('codecs.open', mock.mock_open(), create=True):
with fake_paths_lookup({conf_path: True}):
built_docs = build_docs(version,
False,
False,
False,
)
builder_class = get_builder_class(project.documentation_type)
builder = builder_class(version)
self.assertIn(builder.sphinx_builder,
str(mock_Popen.call_args_list[1])
)
def test_builder_comments(self):
# Normal build
project = ProjectFactory(allow_comments=True)
version = project.versions.all()[0]
builder_class = get_builder_class(project.documentation_type)
builder = builder_class(version)
self.assertEqual(builder.sphinx_builder, 'readthedocs-comments')
def test_builder_no_comments(self):
# Normal build
project = ProjectFactory(allow_comments=False)
version = project.versions.all()[0]
builder_class = get_builder_class(project.documentation_type)
builder = builder_class(version)
self.assertEqual(builder.sphinx_builder, 'readthedocs')
def build_docs(version, force, search, localmedia):
"""
This handles the actual building of the documentation
"""
project = version.project
results = {}
before_build.send(sender=version)
with project.repo_nonblockinglock(version=version,
max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)):
html_builder = get_builder_class(project.documentation_type)(version)
if force:
html_builder.force()
html_builder.append_conf()
results['html'] = html_builder.build()
if results['html'][0] == 0:
html_builder.move()
# Gracefully attempt to move files via task on web workers.
try:
move_files.delay(
version_pk=version.pk,
html=True,
hostname=socket.gethostname(),
)
except socket.error:
pass
fake_results = (999, "Project Skipped, Didn't build",
"Project Skipped, Didn't build")
if 'mkdocs' in project.documentation_type:
if search:
try:
search_builder = get_builder_class('mkdocs_json')(version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True)
if 'sphinx' in project.documentation_type:
# Search builder. Creates JSON from docs and sends it to the
# server.
if search:
try:
search_builder = get_builder_class('sphinx_search')(version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
# Copy json for safe keeping
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True)
# Local media builder for singlepage HTML download archive
if localmedia:
try:
localmedia_builder = get_builder_class('sphinx_singlehtmllocalmedia')(version)
results['localmedia'] = localmedia_builder.build()
if results['localmedia'][0] == 0:
localmedia_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True)
# Optional build steps
if version.project.slug not in HTML_ONLY and not project.skip:
if project.enable_pdf_build:
pdf_builder = get_builder_class('sphinx_pdf')(version)
results['pdf'] = pdf_builder.build()
# Always move pdf results even when there's an error.
# if pdf_results[0] == 0:
pdf_builder.move()
else:
results['pdf'] = fake_results
if project.enable_epub_build:
epub_builder = get_builder_class('sphinx_epub')(version)
results['epub'] = epub_builder.build()
if results['epub'][0] == 0:
epub_builder.move()
else:
results['epub'] = fake_results
after_build.send(sender=version)
return results
def setup_environment(version):
"""
Build the virtualenv and install the project into it.
Always build projects with a virtualenv.
"""
ret_dict = {}
project = version.project
build_dir = os.path.join(project.venv_path(version=version.slug), 'build')
if os.path.exists(build_dir):
log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Removing existing build dir'))
shutil.rmtree(build_dir)
if project.use_system_packages:
site_packages = '--system-site-packages'
else:
site_packages = '--no-site-packages'
# Here the command has been modified to support different
# interpreters.
ret_dict['venv'] = run(
'{cmd} {site_packages} {path}'.format(
cmd='{interpreter} -m virtualenv'.format(
interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version.slug)
)
)
# Other code expects sphinx-build to be installed inside the
# virtualenv. Using the -I option makes sure it gets installed
# even if it is already installed system-wide (and
# --system-site-packages is used)
if project.use_system_packages:
ignore_option = '-I'
else:
ignore_option = ''
requirements = ' '.join([
'sphinx==1.3.1',
'Pygments==2.0.2',
'virtualenv==1.10.1',
'setuptools==1.1',
'docutils==0.11',
'mkdocs==0.13.3',
'mock==1.0.1',
'pillow==2.6.1',
'readthedocs-sphinx-ext==0.5.4',
'sphinx-rtd-theme==0.1.8',
'alabaster>=0.7,<0.8,!=0.7.5',
'recommonmark==0.1.1',
])
wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels')
ret_dict['doc_builder'] = run(
(
'{cmd} install --use-wheel --find-links={wheeldir} -U '
'{ignore_option} {requirements}'
).format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
ignore_option=ignore_option,
wheeldir=wheeldir,
requirements=requirements,
)
)
# Handle requirements
requirements_file_path = project.requirements_file
checkout_path = project.checkout_path(version.slug)
if not requirements_file_path:
builder_class = get_builder_class(project.documentation_type)
docs_dir = builder_class(version).docs_dir()
for path in [docs_dir, '']:
for req_file in ['pip_requirements.txt', 'requirements.txt']:
test_path = os.path.join(checkout_path, path, req_file)
print('Testing %s' % test_path)
if os.path.exists(test_path):
requirements_file_path = test_path
break
if requirements_file_path:
os.chdir(checkout_path)
ret_dict['requirements'] = run(
'{cmd} install --exists-action=w -r {requirements}'.format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
requirements=requirements_file_path))
# Handle setup.py
os.chdir(project.checkout_path(version.slug))
if os.path.isfile("setup.py"):
if getattr(settings, 'USE_PIP_INSTALL', False):
ret_dict['install'] = run(
'{cmd} install --ignore-installed .'.format(
cmd=project.venv_bin(version=version.slug, bin='pip')))
else:
ret_dict['install'] = run(
'{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version.slug,
bin='python')))
else:
ret_dict['install'] = (999, "", "No setup.py, skipping install")
return ret_dict
def build_docs(version, force, search, localmedia):
"""
This handles the actual building of the documentation
"""
project = version.project
results = {}
before_build.send(sender=version)
with project.repo_nonblockinglock(version=version,
max_lock_age=getattr(
settings, 'REPO_LOCK_SECONDS', 30)):
html_builder = get_builder_class(project.documentation_type)(version)
if force:
html_builder.force()
html_builder.append_conf()
results['html'] = html_builder.build()
if results['html'][0] == 0:
html_builder.move()
# Gracefully attempt to move files via task on web workers.
try:
move_files.delay(
version_pk=version.pk,
html=True,
hostname=socket.gethostname(),
)
except socket.error:
pass
fake_results = (999, "Project Skipped, Didn't build",
"Project Skipped, Didn't build")
if 'mkdocs' in project.documentation_type:
if search:
try:
search_builder = get_builder_class('mkdocs_json')(version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg="JSON Build Error"),
exc_info=True)
if 'sphinx' in project.documentation_type:
# Search builder. Creates JSON from docs and sends it to the
# server.
if search:
try:
search_builder = get_builder_class('sphinx_search')(
version)
results['search'] = search_builder.build()
if results['search'][0] == 0:
# Copy json for safe keeping
search_builder.move()
except:
log.error(LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg="JSON Build Error"),
exc_info=True)
# Local media builder for singlepage HTML download archive
if localmedia:
try:
localmedia_builder = get_builder_class(
'sphinx_singlehtmllocalmedia')(version)
results['localmedia'] = localmedia_builder.build()
if results['localmedia'][0] == 0:
localmedia_builder.move()
except:
log.error(LOG_TEMPLATE.format(
project=project.slug,
version=version.slug,
msg="Local Media HTML Build Error"),
exc_info=True)
# Optional build steps
if version.project.slug not in HTML_ONLY and not project.skip:
if project.enable_pdf_build:
pdf_builder = get_builder_class('sphinx_pdf')(version)
results['pdf'] = pdf_builder.build()
# Always move pdf results even when there's an error.
# if pdf_results[0] == 0:
pdf_builder.move()
else:
results['pdf'] = fake_results
if project.enable_epub_build:
epub_builder = get_builder_class('sphinx_epub')(version)
results['epub'] = epub_builder.build()
if results['epub'][0] == 0:
epub_builder.move()
else:
results['epub'] = fake_results
after_build.send(sender=version)
return results
def setup_environment(version):
"""
Build the virtualenv and install the project into it.
Always build projects with a virtualenv.
"""
ret_dict = {}
project = version.project
build_dir = os.path.join(project.venv_path(version=version.slug), 'build')
if os.path.exists(build_dir):
log.info(
LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg='Removing existing build dir'))
shutil.rmtree(build_dir)
if project.use_system_packages:
site_packages = '--system-site-packages'
else:
site_packages = '--no-site-packages'
# Here the command has been modified to support different
# interpreters.
ret_dict['venv'] = run('{cmd} {site_packages} {path}'.format(
cmd='{interpreter} -m virtualenv'.format(
interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version.slug)))
# Other code expects sphinx-build to be installed inside the
# virtualenv. Using the -I option makes sure it gets installed
# even if it is already installed system-wide (and
# --system-site-packages is used)
if project.use_system_packages:
ignore_option = '-I'
else:
ignore_option = ''
requirements = ' '.join([
'sphinx==1.3.1',
'Pygments==2.0.2',
'virtualenv==13.1.0',
'setuptools==18.0.1',
'docutils==0.11',
'mkdocs==0.14.0',
'mock==1.0.1',
'pillow==2.6.1',
'readthedocs-sphinx-ext==0.5.4',
'sphinx-rtd-theme==0.1.8',
'alabaster>=0.7,<0.8,!=0.7.5',
'recommonmark==0.1.1',
])
wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels')
ret_dict['doc_builder'] = run(
('{cmd} install --use-wheel --find-links={wheeldir} -U '
'{ignore_option} {requirements}').format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
ignore_option=ignore_option,
wheeldir=wheeldir,
requirements=requirements,
))
# Handle requirements
requirements_file_path = project.requirements_file
checkout_path = project.checkout_path(version.slug)
if not requirements_file_path:
builder_class = get_builder_class(project.documentation_type)
docs_dir = builder_class(version).docs_dir()
for path in [docs_dir, '']:
for req_file in ['pip_requirements.txt', 'requirements.txt']:
test_path = os.path.join(checkout_path, path, req_file)
print('Testing %s' % test_path)
if os.path.exists(test_path):
requirements_file_path = test_path
break
if requirements_file_path:
os.chdir(checkout_path)
ret_dict['requirements'] = run(
'{cmd} install --exists-action=w -r {requirements}'.format(
cmd=project.venv_bin(version=version.slug, bin='pip'),
requirements=requirements_file_path))
# Handle setup.py
os.chdir(project.checkout_path(version.slug))
if os.path.isfile("setup.py"):
if getattr(settings, 'USE_PIP_INSTALL', False):
ret_dict['install'] = run(
'{cmd} install --ignore-installed .'.format(
cmd=project.venv_bin(version=version.slug, bin='pip')))
else:
ret_dict['install'] = run('{cmd} setup.py install --force'.format(
cmd=project.venv_bin(version=version.slug, bin='python')))
else:
ret_dict['install'] = (999, "", "No setup.py, skipping install")
return ret_dict
def setup_environment(version):
"""
Build the virtualenv and install the project into it.
Always build projects with a virtualenv.
"""
ret_dict = {}
project = version.project
build_dir = os.path.join(project.venv_path(version=version.slug), "build")
if os.path.exists(build_dir):
log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="Removing existing build dir"))
shutil.rmtree(build_dir)
if project.use_system_packages:
site_packages = "--system-site-packages"
else:
site_packages = "--no-site-packages"
# Here the command has been modified to support different
# interpreters.
ret_dict["venv"] = run(
"{cmd} {site_packages} {path}".format(
cmd="{interpreter} -m virtualenv".format(interpreter=project.python_interpreter),
site_packages=site_packages,
path=project.venv_path(version=version.slug),
)
)
# Other code expects sphinx-build to be installed inside the
# virtualenv. Using the -I option makes sure it gets installed
# even if it is already installed system-wide (and
# --system-site-packages is used)
if project.use_system_packages:
ignore_option = "-I"
else:
ignore_option = ""
requirements = " ".join(
[
"sphinx==1.3.1",
"Pygments==2.0.2",
"virtualenv==1.10.1",
"setuptools==1.1",
"docutils==0.11",
"mkdocs==0.14.0",
"mock==1.0.1",
"pillow==2.6.1",
"readthedocs-sphinx-ext==0.5.4",
"sphinx-rtd-theme==0.1.8",
"alabaster>=0.7,<0.8,!=0.7.5",
"recommonmark==0.1.1",
]
)
wheeldir = os.path.join(settings.SITE_ROOT, "deploy", "wheels")
ret_dict["doc_builder"] = run(
("{cmd} install --use-wheel --find-links={wheeldir} -U " "{ignore_option} {requirements}").format(
cmd=project.venv_bin(version=version.slug, bin="pip"),
ignore_option=ignore_option,
wheeldir=wheeldir,
requirements=requirements,
)
)
# Handle requirements
requirements_file_path = project.requirements_file
checkout_path = project.checkout_path(version.slug)
if not requirements_file_path:
builder_class = get_builder_class(project.documentation_type)
docs_dir = builder_class(version).docs_dir()
for path in [docs_dir, ""]:
for req_file in ["pip_requirements.txt", "requirements.txt"]:
test_path = os.path.join(checkout_path, path, req_file)
print ("Testing %s" % test_path)
if os.path.exists(test_path):
requirements_file_path = test_path
break
if requirements_file_path:
os.chdir(checkout_path)
ret_dict["requirements"] = run(
"{cmd} install --exists-action=w -r {requirements}".format(
cmd=project.venv_bin(version=version.slug, bin="pip"), requirements=requirements_file_path
)
)
# Handle setup.py
os.chdir(project.checkout_path(version.slug))
if os.path.isfile("setup.py"):
if getattr(settings, "USE_PIP_INSTALL", False):
ret_dict["install"] = run(
"{cmd} install --ignore-installed .".format(cmd=project.venv_bin(version=version.slug, bin="pip"))
)
else:
ret_dict["install"] = run(
"{cmd} setup.py install --force".format(cmd=project.venv_bin(version=version.slug, bin="python"))
)
else:
ret_dict["install"] = (999, "", "No setup.py, skipping install")
return ret_dict