def test_finalizers_nested(self): t = self.Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.assertEqual(len(t.finalizers), 10) for task in t.finalizers: task.finalizers.extend( [Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10)]) self.assertEqual(len(task.finalizers), 10) task_result = t.run() self.assertEqual(task_result, 6) results = [task_result] results.extend(t.finalize()) self.assertEqual(len(results), 111) self.assertEqual(results[0], 6) self.assertEqual(results[1], 15) for result in results: self.assertIsInstance(result, numbers.Number) self.assertTrue(result >= 6) self.assertTrue(result <= 24)
def extract_tasks(conf): extracts = ExtractDataCache(conf.system.content.extracts.sources, conf) extracts.create_output_dir() tasks = [] for dep_fn, extract in extracts.content_iter(): t = Task(job=write_extract_file, args=(extract, extract.target), description="generating extract file: " + extract.target, target=extract.target, dependency=dep_fn) tasks.append(t) include_statement = get_include_statement(extract.target_project_path) for verb, adjc, files in [(prepend_to_file, 'prepend', extract.prepend), (append_to_file, 'append', extract.append)]: # have to run appends and prepends always, because the rsync that # populates build/<branch>/source should and does overwrite these # files on every source generation step. None in the dep list does this. for fn in files: msg = "{} extract include for '{}' to '{}'".format( adjc, extract.target, fn) t = Task(job=verb, args=(fn, include_statement), target=fn, dependency=[None, dep_fn], description=msg) tasks.append(t) logger.debug('added tasks for {0} extract generation tasks'.format( len(tasks))) return tasks
def test_finalizers_nested(self): t = Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.assertEqual(len(t.finalizers), 10) for task in t.finalizers: task.finalizers.extend([ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ]) self.assertEqual(len(task.finalizers), 10) self.app.add(t) results = self.app.run() self.assertEqual(len(results), 111) self.assertEqual(results[0], 6) self.assertEqual(results[1], 15) for result in results: self.assertIsInstance(result, numbers.Number) self.assertTrue(result >= 6) self.assertTrue(result <= 24)
def test_add_existing_task_object(self): self.assertEqual(self.app.queue, []) t = Task() self.app.add(t) self.assertIs(t, self.app.queue[0]) self.assertIsNot(t, Task()) self.assertIsNot(Task(), self.app.queue[0])
def test_single_runner_task(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) t = Task() t.job = sum t.description = 'test task' t.args = [[1, 2], 0] self.app.add(t) self.app.run(t) self.assertEqual(self.app.results[0], 3)
def option_tasks(conf): o = OptionDataCache(conf.system.content.options.sources, conf) o.create_output_dir() tasks = [] for dep_fn, option in o.content_iter(): program = option.program.replace(' ', '-') if option.has_field('filename'): option_name = option.filename else: option_name = option.name output_fn = os.path.join( conf.system.content.options.fn_prefix, ''.join( (option.directive, '-', program, '-', option_name + '.rst'))) t = Task(job=write_options, args=(option, output_fn, conf), description='generating option file "{0}" from "{1}"'.format( output_fn, dep_fn), target=output_fn, dependency=[dep_fn]) tasks.append(t) logger.debug('added tasks for {0} option generation tasks'.format( len(tasks))) return tasks
def setUp(self): self.c = Configuration() self.c.runstate = RuntimeStateConfig() self.task = Task() self.task.job = sum self.Task = Task self.task.conf = self.c
def test_conf_object_consistent_in_new_task(self): self.assertEqual(self.app.queue, []) t = Task() self.assertIsNone(t.conf) self.app.add(t) self.assertIsNotNone(t.conf) self.assertIs(self.c, self.app.queue[0].conf) self.assertIs(self.c, t.conf)
def test_finalizers_simple(self): t = Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.app.add(t) results = self.app.run() self.assertEqual(len(results), 11) self.assertEqual(results, [6, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def step_clean(conf): return [ Task(job=shutil.rmtree, args=[conf.system.content.steps.output_dir], target=True, dependency=[conf.system.content.steps.output_dir], description='removing {0}'.format( conf.system.content.steps.output_dir)) ]
def add(self, task=None, conf=None): """ Adds a new :class:`~giza.app.BuildApp()` or :class:`~giza.task.Task()` to the :class:`~giza.app.BuildApp()` object. :param string,Task,BuildApp task: Optional. If not specified, :meth:`~giza.app.BuildApp.add()` creates and returns a new :class:`~giza.task.Task()` object. You can pass the string ``task`` or the class :class:`~giza.task.Task` to explicitly create a new Task, or pass an existing :class:`~giza.task.Task()` instance to add that task to the :class:`~giza.app.BuildApp()` instance. You can also pass the string ``app`` or the :class:`~giza.app.BuildApp` class, to create and add new :class:`~giza.app.BuildApp()`: pass an existing :class:`~giza.app.BuildApp()` instance to add that that operation grouping to the queue. :returns: A reference to a :class:`~giza.app.BuildApp()` or :class:`~giza.task.Task()` object in the :class:`~giza.app.BuildApp()` :raises: :exc:`TypeError` if the ``task`` argument is invalid. """ if conf is not None: self.conf = conf if task is None or task in (Task, 'task'): t = Task() t.conf = self.conf t.force = self.force self.queue.append(t) return t elif task in (MapTask, 'map'): t = MapTask() t.conf = self.conf t.force = self.force self.queue.append(t) return t elif task in (BuildApp, 'app'): t = self.sub_app() self.queue.append(t) return t else: if isinstance(task, Task): task.force = self.force if task.conf is None: task.conf = self.conf self.queue.append(task) return task elif isinstance(task, BuildApp): task.root_app = False task.defualt_pool = self.default_pool task.force = self.force task.pool = self.pool self.queue.append(task) return task else: raise TypeError('invalid task type')
def sphinx_content_preperation(app, conf): # Download embedded git repositories and then run migrations before doing # anything else. with app.context() as asset_app: asset_app.extend_queue(assets_tasks(conf)) with app.context() as migration_app: migration_app.extend_queue(migration_tasks(conf)) # Copy all source to the ``build/<branch>/source`` directory. with Timer('migrating source to build'): with app.context(randomize=True) as source_app: for (_, (build_config, sconf)) in get_restricted_builder_jobs(conf): source_app.extend_queue(source_tasks(build_config, sconf)) # load all generated content and create tasks. with Timer('loading generated content'): for (_, (build_config, sconf)) in get_restricted_builder_jobs(conf): for content, func in build_config.system.content.task_generators: app.add(Task(job=func, args=[build_config], target=True)) app.randomize = True results = app.run() app.reset() for task_group in results: app.extend_queue(task_group) for ((edition, language, builder), (build_config, sconf)) in get_restricted_builder_jobs(conf): # these functions all return tasks app.extend_queue(image_tasks(build_config, sconf)) for content_generator in (robots_txt_tasks, intersphinx_tasks, table_tasks, hash_tasks): app.extend_queue(content_generator(build_config)) dependency_refresh_app = app.add('app') dependency_refresh_app.extend_queue( refresh_dependency_tasks(build_config)) # once the source is prepared, we dump a dict with md5 hashes of all # files, so we can do better dependency resolution the next time. app.extend_queue(dump_file_hash_tasks(build_config)) # we transfer images to the latex directory directly because offset # images are included using raw latex, and Sphinx doesn't know how # to copy images in this case. app.extend_queue(latex_image_transfer_tasks(build_config, sconf)) msg = 'added source tasks for ({0}, {1}, {2}) in {3}' logger.debug( msg.format(builder, language, edition, build_config.paths.branch_source))
def option_clean(conf): register_options(conf) tasks = [] for fn in conf.system.options.sources: t = Task(job=verbose_remove, args=[fn], target=fn, dependency=None, description='removing {0}'.format(fn)) tasks.append(t) return tasks
def glossary_tasks(conf): terms = GlossaryDataCache(conf.system.content.glossary.sources, conf) terms.create_output_dir() tasks = [] for fn, glossary_file in terms.file_iter(): tasks.append( Task(job=write_glossary, args=(glossary_file, glossary_file.target(fn)), description='generate glossary for: ' + fn, target=glossary_file.target(fn), dependency=fn)) logger.debug('add {0} glossary tasks'.format(len(tasks))) return tasks
def test_finalizers_simple(self): t = self.Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.assertEqual(len(t.finalizers), 10) self.assertEqual(t.run(), 6) self.assertEqual(len(t.finalizers), 10) finals = t.finalize() self.assertEqual(len(finals), 10) self.assertEqual(len(t.finalizers), 10) self.assertEqual(finals, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def release_tasks(conf): rel = ReleaseDataCache(conf.system.content.releases.sources, conf) rel.create_output_dir() tasks = [] for dep_fn, release in rel.content_iter(): t = Task(job=write_release_file, args=(release, release.target, conf), description='generating release spec file: ' + release.target, target=release.target, dependency=dep_fn) tasks.append(t) logger.debug('added tasks for {0} release generation tasks'.format( len(tasks))) return tasks
def step_tasks(conf): s = StepDataCache(conf.system.content.steps.sources, conf) s.create_output_dir() tasks = [] for fn, stepf in s.file_iter(): t = Task(job=write_steps, args=(stepf, stepf.target(fn), conf), description='generate a stepfile for ' + fn, target=stepf.target(fn), dependency=fn) tasks.append(t) logger.debug('added tasks for {0} step generation tasks'.format( len(tasks))) return tasks
def sphinx_tasks(sconf, conf): # Projects that use the append functionality in extracts or similar content # generators will rebuild this task every time. deps = [os.path.join(conf.paths.projectroot, 'conf.py')] deps.extend(conf.system.files.get_configs('sphinx_local')) deps.extend( expand_tree( os.path.join(conf.paths.projectroot, conf.paths.branch_source), 'txt')) return Task(job=run_sphinx, args=(sconf.builder, sconf, conf), target=os.path.join(conf.paths.projectroot, conf.paths.branch_output, sconf.builder), dependency=deps, description='building {0} with sphinx'.format(sconf.builder))
def example_tasks(conf): d = ExampleDataCache(conf.system.content.examples.sources, conf) d.create_output_dir() tasks = [] for fn, exmpf in d.file_iter(): out_fn = os.path.join( conf.system.content.examples.output_dir, conf.system.content.examples.get_basename(fn)) + '.rst' t = Task(job=write_full_example, args=(exmpf.collection, exmpf.examples, out_fn), description='generate an example for ' + fn, target=out_fn, dependency=fn) tasks.append(t) logger.debug('added tasks for {0} example generation tasks'.format( len(tasks))) return tasks
def apiarg_tasks(conf): a = ApiArgDataCache(conf.system.content.apiargs.sources, conf) a.create_output_dir() tasks = [] for dep_fn, apiargs in a.file_iter(): basename = conf.system.content.steps.get_basename(dep_fn)[2:] out_fn = os.path.join(conf.system.content.apiargs.output_dir, basename) + '.rst' t = Task(job=write_apiargs, args=(apiargs, out_fn), target=out_fn, dependency=dep_fn, description="write apiarg table for: " + dep_fn) tasks.append(t) logger.debug('added tasks for {0} apiarg table generation tasks'.format( len(tasks))) return tasks
def run_make_operations(targets, conf): """ :param list targets: A list of tuples in the form of ``(<action>, [option, option])`` that define build targets. :param Configuration conf: The top level configuration object. Parses the ``targets`` list and runs tasks defined, including all specified sphinx targets, all ``push`` deployment targets, and will create the ``env`` packages. Noteworthy behavior: - The order of options *except* for the action in the first option is not important. - If you run ``push`` target with the ``deploy`` option (i.e. ``push-deploy`` or ``push-<edition>-deploy``), ``giza`` will *not* run the ``publish`` Sphinx build. - This interface assumes that all deployment targets (defined in each project begin with ``push-`` or ``stage-``.) If you have a project with different deployment targets, you will need to call ``giza deploy`` directly. - The ``env`` cache targets take the same options as the Sphinx builders and package the environment for only those builders. If you specify ``env`` after a Sphinx target, ``giza`` will build the cache for only that package. """ sphinx_opts = {"languages": set(), "editions": set(), "builders": set()} push_opts = {"targets": set(), "type": None} packaging_opts = {} sphinx_builders = available_sphinx_builders() if 'push' in conf.system.files.data: deploy_configs = dict( (item['target'], item) for item in conf.system.files.data.push) else: deploy_configs = [] tasks = [] for action, options in targets: if action in sphinx_builders: tasks.append(sphinx_opts) add_sphinx_build_options(sphinx_opts, action, options, conf) elif action in ('stage', 'push'): tasks.append(push_opts) push_opts['type'] = action if 'deploy' not in options: sphinx_opts['builders'].add('publish') tasks.append(sphinx_opts) add_sphinx_build_options(sphinx_opts, action, options, conf) conf.runstate.fast = False if action in deploy_configs: push_opts['targets'].add(action) for build_option in options: deploy_target_name = '-'.join((action, build_option)) if build_option in deploy_configs: push_opts['targets'].add(build_option) elif deploy_target_name in deploy_configs: push_opts['targets'].add(deploy_target_name) elif action.startswith('env'): if len(packaging_opts) > 0: packaging_opts = copy.copy(sphinx_opts) tasks.append(packaging_opts) add_sphinx_build_options(packaging_opts, False, options, conf) else: logger.error( 'target: {0} not defined in the make interface'.format(action)) app = BuildApp.new(pool_type=conf.runstate.runner, force=conf.runstate.force, pool_size=conf.runstate.pool_size) if sphinx_opts in tasks: conf.runstate.languages_to_build = list(sphinx_opts['languages']) conf.runstate.editions_to_build = list(sphinx_opts['editions']) conf.runstate.builder = list(sphinx_opts['builders']) if 'publish' in conf.runstate.builder: conf.runstate.fast = False derive_command('sphinx', conf) sphinx_publication(conf, app) if push_opts in tasks: if len(push_opts['targets']) == 0: for lang, edition in itertools.product( conf.runstate.languages_to_build, conf.runstate.editions_to_build): push_target_name = [push_opts['type']] for opt in (edition, lang): if opt is not None: push_target_name.append(opt) push_target_name = '-'.join(push_target_name) push_opts['targets'].add(push_target_name) conf.runstate.push_targets = list(push_opts['targets']) deploy_tasks(conf, app) derive_command('deploy', conf) if packaging_opts in tasks: derive_command('env', conf) app.add( Task(job=env_package_worker, args=(conf.runstate, conf), target=True, dependency=None)) if len(app.queue) >= 1: app.run()
def toc_tasks(conf): tocs = TocDataCache(conf.system.content.toc.sources, conf) tocs.create_output_dir() tasks = [] for dep_fn, toc_data in tocs.file_iter(): deps = [dep_fn] if 'ref-toc-' in dep_fn: base_offset = 8 is_ref = True elif 'ref-spec-' in dep_fn: base_offset = 9 is_ref = True else: base_offset = 4 is_ref = False fn_basename = os.path.basename(dep_fn)[base_offset:].replace( 'yaml', 'rst') toc_items = toc_data.ordered_items() if toc_data.is_spec() is False: out_fn = os.path.join(conf.system.content.toc.output_dir, fn_basename) t = Task(job=write_toc_tree_output, args=(out_fn, toc_items, is_ref), target=out_fn, dependency=dep_fn, description='writing toctree to "{0}"'.format(out_fn)) tasks.append(t) else: deps.extend(toc_data.spec_deps()) if 'ref-toc' in dep_fn: out_fn = os.path.join(conf.system.content.toc.output_dir, 'table-' + fn_basename) reft = Task( job=write_toc_table, args=(out_fn, toc_items), target=out_fn, dependency=deps, description='write ref toc table to "{0}"'.format(out_fn)) tasks.append(reft) elif 'ref-spec' in dep_fn: out_fn = os.path.join(conf.system.content.toc.output_dir, 'table-spec-' + fn_basename) refspec = Task( job=write_toc_table, args=(out_fn, toc_items), target=out_fn, dependency=deps, description='write ref spec table to "{0}"'.format(out_fn)) tasks.append(refspec) else: out_fn = os.path.join(conf.system.content.toc.output_dir, 'dfn-list-' + fn_basename) dt = Task(job=write_dfn_list_output, args=(out_fn, toc_items), target=out_fn, dependency=deps, description='write definition list toc to "{0}"'.format( out_fn)) tasks.append(dt) logger.debug('added tasks for {0} toc generation tasks'.format(len(tasks))) return tasks
def changelog_tasks(conf): tasks = [] if 'jira' not in conf.system.files.data: logger.debug('changelog generation is not configured.') return [] dirname = os.path.join(conf.paths.projectroot, conf.paths.includes, 'changelogs') giza.tools.files.safe_create_directory(os.path.join(dirname, 'releases')) jira_config = os.path.join(conf.paths.projectroot, conf.paths.builddata, 'jira.yaml') major_versions = get_major_version_groupings(conf.system.files.data.jira.site.versions) # If no version listed in jira.yaml, just return; Should be same # as if jira is not configured in conf.system.files.data # Also, log explicit message stating that 0 changelog tasks added. if not major_versions: logger.warning('changelog version is not configured in jira.yaml.') logger.info('added {0} changelog tasks.'.format(len(tasks))) return [] # don't generate changelog content except on the most recent published # branch (i.e. master, typically.). if conf.git.branches.current != conf.git.branches.published[0]: logger.error('you must generate changelogs on the master branch and them backport them to another branch.') logger.info('added {0} changelog tasks'.format(len(tasks))) return tasks # only generate changelogs if there are credentials, even though we don't # really need an auth'ed connection, want to avoid making un-authed builds too long. if not os.path.exists(os.path.expanduser(conf.system.files.data.jira.site.credentials)): logger.warning('jira credentials are not configured for your user. not generating changelog tasks') logger.info('added {0} changelog tasks'.format(len(tasks))) return tasks # bump mtime of all existing files to avoid regenerating files that already committed files # exist. changelog_releases_dir = os.path.join(conf.paths.projectroot, conf.paths.includes, 'changelogs', 'releases') for fn in os.listdir(changelog_releases_dir): os.utime(os.path.join(changelog_releases_dir, fn), None) # add tasks for generating intermediate files for each major version. we do # this on all branches, and publishers need to backport the config changes. for version, releases in major_versions.items(): fn = os.path.join(dirname, version + '.rst') t = Task(job=giza.content.changelog.views.render_intermediate_files, args=(fn, version, releases, conf), target=fn, dependency=[jira_config]) tasks.append(t) # create a task for each version defined. should never regenerate existing files. for version in conf.system.files.data.jira.site.versions: fn = os.path.join(conf.paths.projectroot, conf.paths.includes, 'changelogs', 'releases', version + '.rst') t = Task(job=giza.content.changelog.views.get_changelog_content, args=(fn, version, conf), dependency=[jira_config], target=fn) tasks.append(t) logger.info('added {0} changelog tasks.'.format(len(tasks))) return tasks
def finalize_sphinx_build(sconf, conf): target = sconf.builder tasks = [] if target == 'html' and not conf.runstate.fast: t = Task(job=html_tarball, args=(sconf.name, sconf.build_output, conf), target=[get_tarball_name('html', conf), get_tarball_name('link-html', conf)], dependency=None, description="creating tarball for html archive") tasks.append(t) elif target == 'dirhtml' and not conf.runstate.fast: # We're experiencing some cases were giza seemingly randomly doesn't migrate. # Log this to help us figure out what's going on. logger.info('Going to migrate {} to {}'.format( sconf.fq_build_output, os.path.join(conf.paths.projectroot, conf.paths.public_site_output))) for job in (finalize_dirhtml_build, error_pages): t = Task(job=job, args=(sconf, conf), target=os.path.join(conf.paths.projectroot, conf.paths.public_site_output), dependency=None) tasks.append(t) elif target == 'epub': t = Task(job=finalize_epub_build, args=(target, conf), description='finalizing epub build', dependency=None, target=True) tasks.append(t) elif target == 'man': t = Task(job=man_tarball, args=(sconf.name, sconf.build_output, conf), target=[get_tarball_name('man', conf), get_tarball_name('link-man', conf)], dependency=None, description="creating tarball for manpages") tasks.extend(manpage_url_tasks(target, conf)) tasks.append(('final', t)) elif target == 'slides' and not conf.runstate.fast: tasks.extend(slide_tasks(sconf, conf)) elif target == 'json': json_tasks, transfer_op = json_output_tasks(conf) tasks.extend(json_tasks) tasks.append(('final', transfer_op)) # this is less than ideal elif target == 'singlehtml': tasks.extend(finalize_single_html_tasks(target, conf)) elif target == 'latex': tasks.extend(pdf_tasks(sconf, conf)) elif target == 'gettext': tasks.extend(gettext_tasks(conf)) elif target == 'linkcheck': msg_str = '{0}: See {1}/{0}/output.txt for output.' t = Task(job=printer, args=[msg_str.format(target, conf.paths.branch_output)], target=os.path.join(conf.paths.projectroot, conf.paths.branch_output, target, 'output.txt'), dependency=None) tasks.append(t) logger.debug('adding {0} finalizing tasks for {1} build'.format(len(tasks), target)) return tasks