def pre_build(local_root, versions):
"""Build docs for all versions to determine root directory and master_doc names.
Need to build docs to (a) avoid filename collision with files from root_ref and branch/tag names and (b) determine
master_doc config values for all versions (in case master_doc changes from e.g. contents.rst to index.rst between
versions).
Exports all commits into a temporary directory and returns the path to avoid re-exporting during the final build.
:param str local_root: Local path to git root directory.
:param sphinxcontrib.versioning.versions.Versions versions: Versions class instance.
:return: Tempdir path with exported commits as subdirectories.
:rtype: str
"""
log = logging.getLogger(__name__)
exported_root = TempDir(True).name
# Extract all.
for sha in {r['sha'] for r in versions.remotes}:
target = os.path.join(exported_root, sha)
log.debug('Exporting %s to temporary directory.', sha)
export(local_root, sha, target)
# Build root.
remote = versions[Config.from_context().root_ref]
with TempDir() as temp_dir:
log.debug('Building root (before setting root_dirs) in temporary directory: %s', temp_dir)
source = os.path.dirname(os.path.join(exported_root, remote['sha'], remote['conf_rel_path']))
build(source, temp_dir, versions, remote['name'], True)
existing = os.listdir(temp_dir)
# Define root_dir for all versions to avoid file name collisions.
for remote in versions.remotes:
root_dir = RE_INVALID_FILENAME.sub('_', remote['name'])
while root_dir in existing:
root_dir += '_'
remote['root_dir'] = root_dir
log.debug('%s root directory is %s', remote['name'], root_dir)
existing.append(root_dir)
# Get found_docs and master_doc values for all versions.
for remote in list(versions.remotes):
log.debug('Partially running sphinx-build to read configuration for: %s', remote['name'])
source = os.path.dirname(os.path.join(exported_root, remote['sha'], remote['conf_rel_path']))
try:
config = read_config(source, remote['name'])
except HandledError:
log.warning('Skipping. Will not be building: %s', remote['name'])
versions.remotes.pop(versions.remotes.index(remote))
continue
remote['found_docs'] = config['found_docs']
remote['master_doc'] = config['master_doc']
return exported_root
def read_config(source, current_name):
"""Read the Sphinx config for one version.
:raise HandledError: If sphinx-build fails. Will be logged before raising.
:param str source: Source directory to pass to sphinx-build.
:param str current_name: The ref name of the current version being built.
:return: Specific Sphinx config values.
:rtype: dict
"""
log = logging.getLogger(__name__)
queue = multiprocessing.Queue()
config = Config.from_context()
with TempDir() as temp_dir:
argv = ('sphinx-build', source, temp_dir)
log.debug('Running sphinx-build for config values with args: %s',
str(argv))
child = multiprocessing.Process(target=_read_config,
args=(argv, config, current_name,
queue))
child.start()
child.join() # Block.
if child.exitcode != 0:
log.error(
'sphinx-build failed for branch/tag while reading config: %s',
current_name)
raise HandledError
config = queue.get()
return config
def push(ctx, config, rel_source, dest_branch, rel_dest, **options):
"""Build locally and then push to remote branch.
First the build sub command is invoked which takes care of building all versions of your documentation in a
temporary directory. If that succeeds then all built documents will be pushed to a remote branch.
REL_SOURCE is the path to the docs directory relative to the git root. If the source directory has moved around
between git tags you can specify additional directories.
DEST_BRANCH is the branch name where generated docs will be committed to. The branch will then be pushed to remote.
If there is a race condition with another job pushing to remote the docs will be re-generated and pushed again.
REL_DEST is the path to the directory that will hold all generated docs for all versions relative to the git roof of
DEST_BRANCH.
To pass options to sphinx-build (run for every branch/tag) use a double hyphen
(e.g. push docs gh-pages . -- -D setting=value).
\f
:param click.core.Context ctx: Click context.
:param sphinxcontrib.versioning.lib.Config config: Runtime configuration.
:param tuple rel_source: Possible relative paths (to git root) of Sphinx directory containing conf.py (e.g. docs).
:param str dest_branch: Branch to clone and push to.
:param str rel_dest: Relative path (to git root) to write generated docs to.
:param dict options: Additional Click options.
"""
if 'pre' in config:
config.pop('pre')(rel_source)
config.update({k: v for k, v in options.items() if v})
if config.local_conf:
config.update(read_local_conf(config.local_conf), ignore_set=True)
if NO_EXECUTE:
raise RuntimeError(config, rel_source, dest_branch, rel_dest)
log = logging.getLogger(__name__)
# Clone, build, push.
for _ in range(PUSH_RETRIES):
with TempDir() as temp_dir:
log.info('Cloning %s into temporary directory...', dest_branch)
try:
clone(config.git_root, temp_dir, config.push_remote,
dest_branch, rel_dest, config.grm_exclude)
except GitError as exc:
log.error(exc.message)
log.error(exc.output)
raise HandledError
log.info('Building docs...')
ctx.invoke(build,
rel_source=rel_source,
destination=os.path.join(temp_dir, rel_dest))
versions = config.pop('versions')
log.info('Attempting to push to branch %s on remote repository.',
dest_branch)
try:
if commit_and_push(temp_dir, config.push_remote, versions):
return
except GitError as exc:
log.error(exc.message)
log.error(exc.output)
raise HandledError
log.warning(
'Failed to push to remote repository. Retrying in %d seconds...',
PUSH_SLEEP)
time.sleep(PUSH_SLEEP)
# Failed if this is reached.
log.error('Ran out of retries, giving up.')
raise HandledError
def _build(argv, config, versions, remote, is_root):
"""Build Sphinx docs via multiprocessing for isolation.
:param tuple argv: Arguments to pass to Sphinx.
:param sphinxcontrib.versioning.lib.Config config: Runtime configuration.
:param sphinxcontrib.versioning.versions.Versions versions: Versions class instance.
:param dict remote: The remote of the current version being built.
:param bool is_root: Is this build in the web root?
"""
# Patch.
application.Config = ConfigInject
if config.show_banner:
EventHandlers.BANNER_GREATEST_TAG = config.banner_greatest_tag
EventHandlers.BANNER_MAIN_VERSION = config.banner_main_ref
EventHandlers.BANNER_RECENT_TAG = config.banner_recent_tag
EventHandlers.SHOW_BANNER = True
EventHandlers.CURRENT_VERSION = remote['name']
EventHandlers.IS_ROOT = is_root
EventHandlers.VERSIONS = versions
SC_VERSIONING_VERSIONS[:] = [
p for r in versions.remotes for p in sorted(r.items())
if p[0] not in ('sha', 'date')
]
# Update argv.
if config.verbose > 1:
argv += ('-v', ) * (config.verbose - 1)
if config.no_colors:
argv += ('-N', )
if config.overflow:
argv += config.overflow
log = logging.getLogger(__name__)
with tempfile.NamedTemporaryFile(delete=False) as requirements:
log.info('Freezing requirements to %s', requirements.name)
requirements.write(
subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']), )
with TempDir() as temp_dir:
log.info('Creating a venv at %s...', temp_dir)
try:
venv.EnvBuilder(with_pip=True).create(temp_dir)
#subprocess.check_output([sys.executable, '-m', 'venv', temp_dir])
except subprocess.CalledProcessError as exc:
raise Exception(exc.stdout)
venv_python = os.path.join(temp_dir, 'bin', 'python')
log.info('Installing the frozen requirements...')
subprocess.run(
[venv_python, '-m', 'pip', 'install', '-r', requirements.name],
check=True,
)
# Install the project before building.
# This works because _build runs in a subprocess
# (which is not affected by other subprocesses or their imports);
# however, the package at pkg_path MUST NOT be imported yet
# (because the already-loaded module will be used instead).
rel_path = remote['conf_rel_path']
while not argv[0].endswith(rel_path):
assert rel_path != '/'
rel_path = os.path.dirname(rel_path)
assert argv[0].endswith(rel_path), (argv[0], rel_path)
pkg_path = argv[0][:-len(rel_path)]
log.info('Installing the package from %s...', pkg_path)
subprocess.run(
[venv_python, '-m', 'pip', 'install', pkg_path],
check=True,
)
# Build.
try:
subprocess.run([venv_python, '-m', 'sphinx', *argv], check=True)
except subprocess.CalledProcessError:
raise SphinxError