def mine(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' gh = get_connection(conf) pprint(mine_github_pulls(gh, app, conf))
def robots(args): c = fetch_config(args) app = BuildApp(c) app.pool = "serial" robots_txt_tasks(c, app) app.run()
def create_branch(args): """ Takes a single branch name and (if necessary) creates a new branch. Then, populates the ``build/<branch>`` directory for the new branch using either the parent branch or ``master``. Safe to run multiple times (after a rebase) to update the build cache from master. Also calls :method:`~giza.operations.build_env.fix_build_environment()` to tweak the new build output to update hashes and on-disk copies of the environment to prevent unnecessary full-rebuilds from sphinx. """ conf = fetch_config(args) g = GitRepo(conf.paths.projectroot) branch = conf.runstate.git_branch base_branch = g.current_branch() if base_branch == branch: base_branch = 'master' logger.warning( 'seeding build data for branch "{0}" from "master"'.format(branch)) branch_builddir = os.path.join(conf.paths.projectroot, conf.paths.output, branch) base_builddir = os.path.join(conf.paths.projectroot, conf.paths.output, base_branch) if g.branch_exists(branch): logger.info('checking out branch "{0}"'.format(branch)) else: logger.info( 'creating and checking out a branch named "{0}"'.format(branch)) g.checkout_branch(branch) cmd = "rsync -r --times --checksum {0}/ {1}".format( base_builddir, branch_builddir) logger.info('seeding build directory for "{0}" from "{1}"'.format( branch, base_branch)) command(cmd) logger.info('branch creation complete.') # get a new config here for the new branch conf = fetch_config(args) builders = get_existing_builders(conf) app = BuildApp(conf) app.pool = 'process' fix_build_env_tasks(builders, conf, app) app.run()
def triage(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' j = JeerahClient(conf) j.connect() query_data = giza.jeerah.triage.query(j, app, conf) pprint(giza.jeerah.triage.report(query_data, conf))
def planning(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' j = JeerahClient(conf) j.connect() query_data = giza.jeerah.progress.query(j, app, conf) pprint(giza.jeerah.planning.report(query_data, conf))
def progress(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = "thread" j = JeerahClient(conf) j.connect() query_data = giza.jeerah.progress.query(j, app, conf) pprint(giza.jeerah.progress.report(query_data, conf))
def actions(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' gh = get_connection(conf) results = [] for pull in mine_github_pulls(gh, app, conf): if pull['merge_safe'] is True: results.append(pull) pprint(results)
def sphinx_publication(c, args, app): build_prep_tasks(c, app) # this loop will produce an app for each language/edition/builder combination build_source_copies = set() sphinx_app = BuildApp(c) sphinx_app.pool = app.pool jobs = itertools.product(args.editions_to_build, args.languages_to_build, args.builder) for edition, language, builder in jobs: args.language = language args.edition = edition args.builder = builder build_config = fetch_config(args) prep_app = app.add('app') prep_app.conf = build_config primer_app = prep_app.add('app') primer_migration_tasks(build_config, primer_app) sconf = render_sconf(edition, builder, language, build_config) if build_config.paths.branch_source not in build_source_copies: build_source_copies.add(build_config.paths.branch_source) source_tasks(build_config, sconf, prep_app) source_app = prep_app.add('app') build_content_generation_tasks(build_config, source_app) refresh_dependency_tasks(build_config, prep_app) sphinx_tasks(sconf, build_config, sphinx_app) logger.info("adding builder job for {0} ({1}, {2})".format(builder, language, edition)) app.add(sphinx_app) logger.info("sphinx build setup, running now.") app.run() logger.info("sphinx build complete.") logger.info('builds finalized. sphinx output and errors to follow') sphinx_output = '\n'.join([ o[1] for o in sphinx_app.results ]) ret_code = sum([ o[0] for o in sphinx_app.results ]) output_sphinx_stream(sphinx_output, c) ret_code = 0 return ret_code
def run_sphinx(builder, sconf, conf): if safe_create_directory(sconf.fq_build_output): logger.info('created directory "{1}" for sphinx builder {0}'.format( builder, sconf.fq_build_output)) if 'language' in sconf and sconf.language is not None: command('sphinx-intl build --language=' + sconf.language) logger.info('compiled all PO files for translated build.') logger.info('starting sphinx build {0}'.format(builder)) cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree sphinx_cmd = cmd.format( get_sphinx_args(sconf, conf), os.path.join(conf.paths.projectroot, conf.paths.branch_output), sconf.build_output, os.path.join(conf.paths.projectroot, conf.paths.branch_source), sconf.fq_build_output) logger.debug(sphinx_cmd) with Timer("running sphinx build for: {0}, {1}, {2}".format( builder, sconf.language, sconf.edition)): out = command(sphinx_cmd, capture=True, ignore=True) logger.info('completed sphinx build {0}'.format(builder)) if True: # out.return_code == 0: logger.info('successfully completed {0} sphinx build ({1})'.format( builder, out.return_code)) finalizer_app = BuildApp(conf) finalizer_app.pool = "thread" finalizer_app.root_app = False finalize_sphinx_build(sconf, conf, finalizer_app) with Timer("finalize sphinx {0} build".format(builder)): finalizer_app.run() else: logger.warning( 'the sphinx build {0} was not successful. not running finalize operation' .format(builder)) output = '\n'.join([out.err, out.out]) return out.return_code, output
def stats(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' gh = get_connection(conf) users = set() result = {'merge_safe': 0, 'total': 0 } for pull in mine_github_pulls(gh, app, conf): result['total'] += 1 if pull['merge_safe'] is True: result['merge_safe'] += 1 users.add(pull['user']) result['user_count'] = len(users) result['users'] = list(users) pprint(result)
def sphinx_publication(c, args, app): """ :arg Configuration c: A :class:`giza.config.main.Configuration()` object. :arg RuntimeStateConfig args: A :class:`giza.config.runtime.RuntimeState()` object. :arg BuildApp app: A :class:`giza.core.app.BuildApp()` object. Adds all required tasks to build a Sphinx site. Specifically: 1. Iterates through the (language * builder * edition) combination and adds tasks to generate the content in the <build>/<branch>/source<-edition<-language>> directory. There is one version of the <build>/<branch>/source directory for every language/edition combination, but multiple builders can use the same diretory as needed. 2. Add a task to run the ``sphinx-build`` task. 3. Run all tasks in proper order. 4. Process and print the output of ``sphinx-build``. :return: The sum of all return codes from all ``sphinx-build`` tasks. All non-zero statuses represent errors. :rtype: int """ # sphinx-build tasks are separated into their own app. sphinx_app = BuildApp(c) sphinx_app.pool = app.pool # this loop will produce an app for each language/edition/builder combination build_source_copies = set() for edition, language, builder in get_builder_jobs(c): build_config, sconf = get_sphinx_build_configuration( edition, language, builder, args) # only do these tasks once per-language+edition combination if build_config.paths.branch_source not in build_source_copies: build_source_copies.add(build_config.paths.branch_source) prep_app = app.add('app') prep_app.conf = build_config # this is where we add tasks to transfer the source into the # ``build/<branch>/source`` directory. source_tasks(build_config, sconf, prep_app) # this function runs the entire prep_app compiled until now, so that # the content generation tasks are created properly # these operation groups each execute in isolation of each-other and should. build_content_generation_tasks(build_config, prep_app.add('app')) refresh_dependency_tasks(build_config, prep_app.add('app')) # once the source is prepared, we dump a dict with md5 hashes of all # files, so we can do better dependency resolution the next time. dump_file_hash_tasks(build_config, prep_app) # we transfer images to the latex directory directly because offset # images are included using raw latex, and Sphinx doesn't know how # to copy images in this case. latex_image_transfer_tasks(build_config, sconf, prep_app) msg = 'added source tasks for ({0}, {1}, {2}) in {3}' logger.info( msg.format(builder, language, edition, build_config.paths.branch_source)) # Add sphinx tasks for this builder/language/edition combination sphinx_tasks(sconf, build_config, sphinx_app) logger.info("adding builder job for {0} ({1}, {2})".format( builder, language, edition)) # Connect the special sphinx app to the main app. app.add(sphinx_app) logger.info("sphinx build configured, running the build now.") app.run() logger.info("sphinx build complete.") logger.info('builds finalized. sphinx output and errors to follow') # process the sphinx build. These oeprations allow us to de-duplicate # messages between builds. sphinx_output = '\n'.join([o[1] for o in sphinx_app.results]) output_sphinx_stream(sphinx_output, c) # if entry points return this value, giza will inherit the sum of the Sphinx # build return codes. ret_code = sum([o[0] for o in sphinx_app.results]) return ret_code