def mine(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = 'thread'
gh = get_connection(conf)
pprint(mine_github_pulls(gh, app, conf))
def robots(args):
c = fetch_config(args)
app = BuildApp(c)
app.pool = "serial"
robots_txt_tasks(c, app)
app.run()
def create_branch(args):
"""
Takes a single branch name and (if necessary) creates a new branch. Then,
populates the ``build/<branch>`` directory for the new branch using either
the parent branch or ``master``. Safe to run multiple times (after a rebase)
to update the build cache from master.
Also calls :method:`~giza.operations.build_env.fix_build_environment()` to
tweak the new build output to update hashes and on-disk copies of the
environment to prevent unnecessary full-rebuilds from sphinx.
"""
conf = fetch_config(args)
g = GitRepo(conf.paths.projectroot)
branch = conf.runstate.git_branch
base_branch = g.current_branch()
if base_branch == branch:
base_branch = 'master'
logger.warning(
'seeding build data for branch "{0}" from "master"'.format(branch))
branch_builddir = os.path.join(conf.paths.projectroot, conf.paths.output,
branch)
base_builddir = os.path.join(conf.paths.projectroot, conf.paths.output,
base_branch)
if g.branch_exists(branch):
logger.info('checking out branch "{0}"'.format(branch))
else:
logger.info(
'creating and checking out a branch named "{0}"'.format(branch))
g.checkout_branch(branch)
cmd = "rsync -r --times --checksum {0}/ {1}".format(
base_builddir, branch_builddir)
logger.info('seeding build directory for "{0}" from "{1}"'.format(
branch, base_branch))
command(cmd)
logger.info('branch creation complete.')
# get a new config here for the new branch
conf = fetch_config(args)
builders = get_existing_builders(conf)
app = BuildApp(conf)
app.pool = 'process'
fix_build_env_tasks(builders, conf, app)
app.run()
def triage(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = 'thread'
j = JeerahClient(conf)
j.connect()
query_data = giza.jeerah.triage.query(j, app, conf)
pprint(giza.jeerah.triage.report(query_data, conf))
def triage(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = 'thread'
j = JeerahClient(conf)
j.connect()
query_data = giza.jeerah.triage.query(j, app, conf)
pprint(giza.jeerah.triage.report(query_data, conf))
def planning(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = 'thread'
j = JeerahClient(conf)
j.connect()
query_data = giza.jeerah.progress.query(j, app, conf)
pprint(giza.jeerah.planning.report(query_data, conf))
def progress(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = "thread"
j = JeerahClient(conf)
j.connect()
query_data = giza.jeerah.progress.query(j, app, conf)
pprint(giza.jeerah.progress.report(query_data, conf))
def actions(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = 'thread'
gh = get_connection(conf)
results = []
for pull in mine_github_pulls(gh, app, conf):
if pull['merge_safe'] is True:
results.append(pull)
pprint(results)
def sphinx_publication(c, args, app):
build_prep_tasks(c, app)
# this loop will produce an app for each language/edition/builder combination
build_source_copies = set()
sphinx_app = BuildApp(c)
sphinx_app.pool = app.pool
jobs = itertools.product(args.editions_to_build, args.languages_to_build, args.builder)
for edition, language, builder in jobs:
args.language = language
args.edition = edition
args.builder = builder
build_config = fetch_config(args)
prep_app = app.add('app')
prep_app.conf = build_config
primer_app = prep_app.add('app')
primer_migration_tasks(build_config, primer_app)
sconf = render_sconf(edition, builder, language, build_config)
if build_config.paths.branch_source not in build_source_copies:
build_source_copies.add(build_config.paths.branch_source)
source_tasks(build_config, sconf, prep_app)
source_app = prep_app.add('app')
build_content_generation_tasks(build_config, source_app)
refresh_dependency_tasks(build_config, prep_app)
sphinx_tasks(sconf, build_config, sphinx_app)
logger.info("adding builder job for {0} ({1}, {2})".format(builder, language, edition))
app.add(sphinx_app)
logger.info("sphinx build setup, running now.")
app.run()
logger.info("sphinx build complete.")
logger.info('builds finalized. sphinx output and errors to follow')
sphinx_output = '\n'.join([ o[1] for o in sphinx_app.results ])
ret_code = sum([ o[0] for o in sphinx_app.results ])
output_sphinx_stream(sphinx_output, c)
ret_code = 0
return ret_code
def run_sphinx(builder, sconf, conf):
if safe_create_directory(sconf.fq_build_output):
logger.info('created directory "{1}" for sphinx builder {0}'.format(
builder, sconf.fq_build_output))
if 'language' in sconf and sconf.language is not None:
command('sphinx-intl build --language=' + sconf.language)
logger.info('compiled all PO files for translated build.')
logger.info('starting sphinx build {0}'.format(builder))
cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree
sphinx_cmd = cmd.format(
get_sphinx_args(sconf, conf),
os.path.join(conf.paths.projectroot,
conf.paths.branch_output), sconf.build_output,
os.path.join(conf.paths.projectroot, conf.paths.branch_source),
sconf.fq_build_output)
logger.debug(sphinx_cmd)
with Timer("running sphinx build for: {0}, {1}, {2}".format(
builder, sconf.language, sconf.edition)):
out = command(sphinx_cmd, capture=True, ignore=True)
logger.info('completed sphinx build {0}'.format(builder))
if True: # out.return_code == 0:
logger.info('successfully completed {0} sphinx build ({1})'.format(
builder, out.return_code))
finalizer_app = BuildApp(conf)
finalizer_app.pool = "thread"
finalizer_app.root_app = False
finalize_sphinx_build(sconf, conf, finalizer_app)
with Timer("finalize sphinx {0} build".format(builder)):
finalizer_app.run()
else:
logger.warning(
'the sphinx build {0} was not successful. not running finalize operation'
.format(builder))
output = '\n'.join([out.err, out.out])
return out.return_code, output
def stats(args):
conf = fetch_config(args)
app = BuildApp(conf)
app.pool = 'thread'
gh = get_connection(conf)
users = set()
result = {'merge_safe': 0, 'total': 0 }
for pull in mine_github_pulls(gh, app, conf):
result['total'] += 1
if pull['merge_safe'] is True:
result['merge_safe'] += 1
users.add(pull['user'])
result['user_count'] = len(users)
result['users'] = list(users)
pprint(result)
def sphinx_publication(c, args, app):
"""
:arg Configuration c: A :class:`giza.config.main.Configuration()` object.
:arg RuntimeStateConfig args: A :class:`giza.config.runtime.RuntimeState()` object.
:arg BuildApp app: A :class:`giza.core.app.BuildApp()` object.
Adds all required tasks to build a Sphinx site. Specifically:
1. Iterates through the (language * builder * edition) combination and adds
tasks to generate the content in the
<build>/<branch>/source<-edition<-language>> directory. There is one
version of the <build>/<branch>/source directory for every
language/edition combination, but multiple builders can use the same
diretory as needed.
2. Add a task to run the ``sphinx-build`` task.
3. Run all tasks in proper order.
4. Process and print the output of ``sphinx-build``.
:return: The sum of all return codes from all ``sphinx-build`` tasks. All
non-zero statuses represent errors.
:rtype: int
"""
# sphinx-build tasks are separated into their own app.
sphinx_app = BuildApp(c)
sphinx_app.pool = app.pool
# this loop will produce an app for each language/edition/builder combination
build_source_copies = set()
for edition, language, builder in get_builder_jobs(c):
build_config, sconf = get_sphinx_build_configuration(
edition, language, builder, args)
# only do these tasks once per-language+edition combination
if build_config.paths.branch_source not in build_source_copies:
build_source_copies.add(build_config.paths.branch_source)
prep_app = app.add('app')
prep_app.conf = build_config
# this is where we add tasks to transfer the source into the
# ``build/<branch>/source`` directory.
source_tasks(build_config, sconf, prep_app)
# this function runs the entire prep_app compiled until now, so that
# the content generation tasks are created properly
# these operation groups each execute in isolation of each-other and should.
build_content_generation_tasks(build_config, prep_app.add('app'))
refresh_dependency_tasks(build_config, prep_app.add('app'))
# once the source is prepared, we dump a dict with md5 hashes of all
# files, so we can do better dependency resolution the next time.
dump_file_hash_tasks(build_config, prep_app)
# we transfer images to the latex directory directly because offset
# images are included using raw latex, and Sphinx doesn't know how
# to copy images in this case.
latex_image_transfer_tasks(build_config, sconf, prep_app)
msg = 'added source tasks for ({0}, {1}, {2}) in {3}'
logger.info(
msg.format(builder, language, edition,
build_config.paths.branch_source))
# Add sphinx tasks for this builder/language/edition combination
sphinx_tasks(sconf, build_config, sphinx_app)
logger.info("adding builder job for {0} ({1}, {2})".format(
builder, language, edition))
# Connect the special sphinx app to the main app.
app.add(sphinx_app)
logger.info("sphinx build configured, running the build now.")
app.run()
logger.info("sphinx build complete.")
logger.info('builds finalized. sphinx output and errors to follow')
# process the sphinx build. These oeprations allow us to de-duplicate
# messages between builds.
sphinx_output = '\n'.join([o[1] for o in sphinx_app.results])
output_sphinx_stream(sphinx_output, c)
# if entry points return this value, giza will inherit the sum of the Sphinx
# build return codes.
ret_code = sum([o[0] for o in sphinx_app.results])
return ret_code
