def sphinx_content_preperation(app, conf):
# Download embedded git repositories and then run migrations before doing
# anything else.
with app.context() as asset_app:
asset_app.extend_queue(assets_tasks(conf))
with app.context() as migration_app:
migration_app.extend_queue(migration_tasks(conf))
# Copy all source to the ``build/<branch>/source`` directory.
with Timer('migrating source to build'):
with app.context(randomize=True) as source_app:
for (_, (build_config,
sconf)) in get_restricted_builder_jobs(conf):
source_app.extend_queue(source_tasks(build_config, sconf))
# load all generated content and create tasks.
with Timer('loading generated content'):
for (_, (build_config, sconf)) in get_restricted_builder_jobs(conf):
for content, func in build_config.system.content.task_generators:
app.add(Task(job=func, args=[build_config], target=True))
app.randomize = True
results = app.run()
app.reset()
for task_group in results:
app.extend_queue(task_group)
for ((edition, language, builder),
(build_config, sconf)) in get_restricted_builder_jobs(conf):
# these functions all return tasks
app.extend_queue(image_tasks(build_config, sconf))
for content_generator in (robots_txt_tasks, intersphinx_tasks,
table_tasks, hash_tasks):
app.extend_queue(content_generator(build_config))
dependency_refresh_app = app.add('app')
dependency_refresh_app.extend_queue(
refresh_dependency_tasks(build_config))
# once the source is prepared, we dump a dict with md5 hashes of all
# files, so we can do better dependency resolution the next time.
app.extend_queue(dump_file_hash_tasks(build_config))
# we transfer images to the latex directory directly because offset
# images are included using raw latex, and Sphinx doesn't know how
# to copy images in this case.
app.extend_queue(latex_image_transfer_tasks(build_config, sconf))
msg = 'added source tasks for ({0}, {1}, {2}) in {3}'
logger.debug(
msg.format(builder, language, edition,
build_config.paths.branch_source))
def build_content_generation_tasks(conf, app):
"""
:param Configuration conf: The current build configuration object.
:param BuildApp app: A :class:`~giza.core.app.BuildApp()` object.
Add tasks to the ``app`` for all tasks that modify the content in
``build/<branch>/source`` directory.
"""
app.randomize = True
with Timer("adding content tasks"):
for content, func in conf.system.content.task_generators:
t = app.add('task')
t.job = func
t.args = [conf]
t.target = True
results = app.run()
for content_generator_tasks in results:
app.extend_queue(content_generator_tasks)
robots_txt_tasks(conf, app)
intersphinx_tasks(conf, app)
includes_tasks(conf, app)
table_tasks(conf, app)
hash_tasks(conf, app)
api_tasks(conf, app)
redirect_tasks(conf, app)
image_tasks(conf, app)
def run_sphinx(builder, sconf, conf):
if safe_create_directory(sconf.fq_build_output):
logger.info('created directory "{1}" for sphinx builder {0}'.format(
builder, sconf.fq_build_output))
if 'language' in sconf and sconf.language is not None:
command('sphinx-intl build --language=' + sconf.language)
logger.info('compiled all PO files for translated build.')
logger.info('starting sphinx build {0}'.format(builder))
cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree
sphinx_cmd = cmd.format(
get_sphinx_args(sconf, conf),
os.path.join(conf.paths.projectroot,
conf.paths.branch_output), sconf.build_output,
os.path.join(conf.paths.projectroot, conf.paths.branch_source),
sconf.fq_build_output)
logger.debug(sphinx_cmd)
with Timer("running sphinx build for: {0}, {1}, {2}".format(
builder, sconf.language, sconf.edition)):
out = command(sphinx_cmd, capture=True, ignore=True)
logger.info('completed sphinx build {0}'.format(builder))
if True: # out.return_code == 0:
logger.info('successfully completed {0} sphinx build ({1})'.format(
builder, out.return_code))
finalizer_app = BuildApp(conf)
finalizer_app.pool = "thread"
finalizer_app.root_app = False
finalize_sphinx_build(sconf, conf, finalizer_app)
with Timer("finalize sphinx {0} build".format(builder)):
finalizer_app.run()
else:
logger.warning(
'the sphinx build {0} was not successful. not running finalize operation'
.format(builder))
output = '\n'.join([out.err, out.out])
return out.return_code, output
def main(args):
"""
Use Sphinx to generate build artifacts. Can generate artifacts for multiple
output types, content editions and translations.
"""
c = fetch_config(args)
app = BuildApp(c)
with Timer("full sphinx build process"):
return sphinx_publication(c, args, app)
def refresh_deps(conf):
with Timer('resolve dependency graph'):
# resolve a map of the source files to the files they depend on
# (i.e. the ones that they include).
graph = include_files(conf=conf)
# load, if possible, a mappping of all source files with hashes from the
# last build.
if not os.path.exists(conf.system.dependency_cache):
dep_map = None
else:
with open(conf.system.dependency_cache, 'r') as f:
try:
dep_cache = json.load(f)
dep_map = dep_cache['files']
except ValueError:
dep_map = None
m = 'no stored dependency information, will rebuild more things than necessary.'
logger.warning(m)
with Timer('dependency updates'):
_refresh_deps(graph, dep_map, conf)
def run_sphinx(builder, sconf, conf):
if safe_create_directory(sconf.fq_build_output):
m = 'created directory "{1}" for sphinx builder {0}'
logger.info(m.format(builder, sconf.fq_build_output))
if 'language' in sconf and sconf.language is not None:
cmd_str = 'sphinx-intl build --language=' + sconf.language
try:
subprocess.check_call(shlex.split(cmd_str))
logger.info('compiled all PO files for translated build.')
except subprocess.CalledProcessError as e:
logger.error('sphinx-intl encountered error: ' + str(e.returncode))
logger.info(cmd_str)
logger.info('starting sphinx build {0}'.format(builder))
cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree
sphinx_cmd = cmd.format(
get_sphinx_args(sconf, conf),
os.path.join(conf.paths.projectroot,
conf.paths.branch_output), sconf.build_output,
os.path.join(conf.paths.projectroot, conf.paths.branch_source),
sconf.fq_build_output)
logger.debug(sphinx_cmd)
m = "running sphinx build for: {0}, {1}, {2}"
with Timer(m.format(builder, sconf.language, sconf.edition)):
try:
output = subprocess.check_output(shlex.split(sphinx_cmd),
stderr=subprocess.STDOUT)
return_code = 0
except subprocess.CalledProcessError as e:
output = e.output
return_code = e.returncode
logger.info(sphinx_cmd)
try:
os.utime(sconf.fq_build_output, None)
except:
pass
m = 'completed {0} sphinx build for {1}.{2}.{3} ({4})'
logger.info(
m.format(builder, conf.project.name, conf.project.edition,
conf.git.branches.current, return_code))
return return_code, output
def main(args):
"""
Use Sphinx to generate build artifacts. Can generate artifacts for multiple
output types, content editions and translations.
"""
conf = fetch_config(args)
app = BuildApp.new(pool_type=conf.runstate.runner,
pool_size=conf.runstate.pool_size,
force=conf.runstate.force)
with Timer("full sphinx build process"):
# In general we try to avoid passing the "app" object between functions
# and mutating it at too many places in the stack (although in earlier
# versions this was the primary idiom). This call is a noted exception,
# and makes it possible to run portions of this process in separate
# targets.
sphinx_publication(conf, app)
def apiarg_tasks(conf):
with Timer('apiargs migrations'):
name_changes = migration_task(task='branch', conf=conf)
apiarg_sources = conf.system.content.apiargs.sources
a = ApiArgDataCache(apiarg_sources, conf)
if len(apiarg_sources) > 0 and not os.path.isdir(
conf.system.content.apiargs.output_dir):
safe_create_directory(conf.system.content.apiargs.output_dir)
tasks = []
# for dep_fn, table in a.file_iter():
# print dep_fn
logger.info(
'new apiargs not yet implemented, but there are {0} of them'.format(
str(len(conf.system.content.apiargs.sources))))
return []