def test_finalizers_nested(self): t = Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.assertEqual(len(t.finalizers), 10) for task in t.finalizers: task.finalizers.extend([ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ]) self.assertEqual(len(task.finalizers), 10) self.app.add(t) results = self.app.run() self.assertEqual(len(results), 111) self.assertEqual(results[0], 6) self.assertEqual(results[1], 15) for result in results: self.assertIsInstance(result, numbers.Number) self.assertTrue(result >= 6) self.assertTrue(result <= 24)
def test_add_existing_task_object(self): self.assertEqual(self.app.queue, []) t = Task() self.app.add(t) self.assertIs(t, self.app.queue[0]) self.assertIsNot(t, Task()) self.assertIsNot(Task(), self.app.queue[0])
def test_finalizers_nested(self): t = self.Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.assertEqual(len(t.finalizers), 10) for task in t.finalizers: task.finalizers.extend([ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ]) self.assertEqual(len(task.finalizers), 10) task_result = t.run() self.assertEqual(task_result, 6) results = [task_result] results.extend(t.finalize()) self.assertEqual(len(results), 111) self.assertEqual(results[0], 6) self.assertEqual(results[1], 15) for result in results: self.assertIsInstance(result, numbers.Number) self.assertTrue(result >= 6) self.assertTrue(result <= 24)
def extract_tasks(conf): extracts = ExtractDataCache(conf.system.content.extracts.sources, conf) extracts.create_output_dir() tasks = [] for dep_fn, extract in extracts.content_iter(): t = Task(job=write_extract_file, args=(extract, extract.target), description="generating extract file: " + extract.target, target=extract.target, dependency=dep_fn) tasks.append(t) include_statement = get_include_statement(extract.target_project_path) for verb, adjc, files in [(prepend_to_file, 'prepend', extract.prepend), (append_to_file, 'append', extract.append)]: # have to run appends and prepends always, because the rsync that # populates build/<branch>/source should and does overwrite these # files on every source generation step. None in the dep list does this. for fn in files: msg = "{0} extract include for '{0}' to '{1}'".format( adjc, extract.target, fn) t = Task(job=verb, args=(fn, include_statement), target=fn, dependency=[None, dep_fn], description=msg) tasks.append(t) logger.info("added tasks for {0} extract generation tasks".format( len(tasks))) return tasks
def test_single_runner_task(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) t = Task() t.job = sum t.description = 'test task' t.args = [[1, 2], 0] self.app.add(t) self.app.run(t) self.assertEqual(self.app.results[0], 3)
def test_finalizers_simple(self): t = Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.app.add(t) results = self.app.run() self.assertEqual(len(results), 11) self.assertEqual(results, [6, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def setUp(self): self.c = Configuration() self.c.runstate = RuntimeStateConfig() self.task = Task() self.task.job = sum self.Task = Task self.task.conf = self.c
def test_conf_object_consistent_in_new_task(self): self.assertEqual(self.app.queue, []) t = Task() self.assertIsNone(t.conf) self.app.add(t) self.assertIsNotNone(t.conf) self.assertIs(self.c, self.app.queue[0].conf) self.assertIs(self.c, t.conf)
def glossary_clean(conf): return [ Task(job=shutil.rmtree, args=[conf.system.content.glossary.output_dir], target=True, depdency=[conf.system.content.glossary.output_dir], descrption='removing {0}'.format( conf.system.content.glossary.output_dir)) ]
def example_clean(conf): return [ Task(job=shutil.rmtree, args=[conf.system.content.examples.output_dir], target=True, dependency=[conf.system.content.examples.output_dir], description='removing {0}'.format( conf.system.content.examples.output_dir)) ]
def add(self, task=None, conf=None): """ Adds a new :class:`~giza.app.BuildApp()` or :class:`~giza.task.Task()` to the :class:`~giza.app.BuildApp()` object. :param string,Task,BuildApp task: Optional. If not specified, :meth:`~giza.app.BuildApp.add()` creates and returns a new :class:`~giza.task.Task()` object. You can pass the string ``task`` or the class :class:`~giza.task.Task` to explicitly create a new Task, or pass an existing :class:`~giza.task.Task()` instance to add that task to the :class:`~giza.app.BuildApp()` instance. You can also pass the string ``app`` or the :class:`~giza.app.BuildApp` class, to create and add new :class:`~giza.app.BuildApp()`: pass an existing :class:`~giza.app.BuildApp()` instance to add that that operation grouping to the queue. :returns: A reference to a :class:`~giza.app.BuildApp()` or :class:`~giza.task.Task()` object in the :class:`~giza.app.BuildApp()` :raises: :exc:`TypeError` if the ``task`` argument is invalid. """ if conf is not None: self.conf = conf if task is None or task in (Task, 'task'): t = Task() t.conf = self.conf t.force = self.force self.queue.append(t) return t elif task in (MapTask, 'map'): t = MapTask() t.conf = self.conf t.force = self.force self.queue.append(t) return t elif task in (BuildApp, 'app'): t = self.sub_app() self.queue.append(t) return t else: if isinstance(task, Task): task.force = self.force if task.conf is None: task.conf = self.conf self.queue.append(task) return task elif isinstance(task, BuildApp): task.root_app = False task.defualt_pool = self.default_pool task.force = self.force task.pool = self.pool self.queue.append(task) return task else: raise TypeError('invalid task type')
def option_clean(conf): register_options(conf) tasks = [] for fn in conf.system.options.sources: t = Task(job=verbose_remove, args=[fn], target=fn, dependency=None, description='removing {0}'.format(fn)) tasks.append(t) return tasks
def sphinx_tasks(sconf, conf): # Projects that use the append functionality in extracts or similar content # generators will rebuild this task every time. deps = [os.path.join(conf.paths.projectroot, 'conf.py')] deps.extend(conf.system.files.get_configs('sphinx_local')) deps.extend(expand_tree(os.path.join(conf.paths.projectroot, conf.paths.branch_source), 'txt')) return Task(job=run_sphinx, args=(sconf.builder, sconf, conf), target=os.path.join(conf.paths.projectroot, conf.paths.branch_output, sconf.builder), dependency=deps, description='building {0} with sphinx'.format(sconf.builder))
def step_tasks(conf): s = StepDataCache(conf.system.content.steps.sources, conf) s.create_output_dir() tasks = [] for fn, stepf in s.file_iter(): t = Task(job=write_steps, args=(stepf, stepf.target(fn), conf), description='generate a stepfile for ' + fn, target=stepf.target(fn), dependency=fn) tasks.append(t) logger.info("added tasks for {0} step generation tasks".format(len(tasks))) return tasks
def glossary_tasks(conf): terms = GlossaryDataCache(conf.system.content.glossary.sources, conf) terms.create_output_dir() tasks = [] for fn, glossary_file in terms.file_iter(): tasks.append( Task(job=write_glossary, args=(glossary_file, glossary_file.target(fn)), description="generate glossary for: " + fn, target=glossary_file.target(fn), dependency=fn)) logger.info("add {0} glossary tasks".format(len(tasks))) return tasks
def release_tasks(conf): rel = ReleaseDataCache(conf.system.content.releases.sources, conf) rel.create_output_dir() tasks = [] for dep_fn, release in rel.content_iter(): t = Task(job=write_release_file, args=(release, release.target, conf), description='generating release spec file: ' + release.target, target=release.target, dependency=dep_fn) tasks.append(t) logger.info("added tasks for {0} release generation tasks".format( len(tasks))) return tasks
def example_tasks(conf): d = ExampleDataCache(conf.system.content.examples.sources, conf) d.create_output_dir() tasks = [] for fn, exmpf in d.file_iter(): out_fn = os.path.join( conf.system.content.examples.output_dir, conf.system.content.examples.get_basename(fn)) + '.rst' t = Task(job=write_full_example, args=(exmpf.collection, exmpf.examples, out_fn), description='generate an example for ' + fn, target=out_fn, dependency=fn) tasks.append(t) logger.info("added tasks for {0} example generation tasks".format( len(tasks))) return tasks
def test_finalizers_simple(self): t = self.Task(job=sum, args=((1, 2, 3), 0)) self.assertEqual(t.finalizers, []) t.finalizers = [ Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10) ] self.assertEqual(len(t.finalizers), 10) self.assertEqual(t.run(), 6) self.assertEqual(len(t.finalizers), 10) finals = t.finalize() self.assertEqual(len(finals), 10) self.assertEqual(len(t.finalizers), 10) self.assertEqual(finals, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def apiarg_tasks(conf): a = ApiArgDataCache(conf.system.content.apiargs.sources, conf) a.create_output_dir() tasks = [] for dep_fn, apiargs in a.file_iter(): basename = conf.system.content.steps.get_basename(dep_fn)[2:] out_fn = os.path.join(conf.system.content.apiargs.output_dir, basename) + '.rst' t = Task(job=write_apiargs, args=(apiargs, out_fn), target=out_fn, dependency=dep_fn, description="write apiarg table for: " + dep_fn) tasks.append(t) logger.info('added tasks for {0} apiarg table generation tasks'.format( len(tasks))) return tasks
def option_tasks(conf): o = OptionDataCache(conf.system.content.options.sources, conf) o.create_output_dir() tasks = [] for dep_fn, option in o.content_iter(): output_fn = os.path.join( conf.system.content.options.fn_prefix, ''.join( (option.directive, '-', option.program, '-', option.name + '.rst'))) t = Task(job=write_options, args=(option, output_fn, conf), description='generating option file "{0}" from "{1}"'.format( output_fn, dep_fn), target=output_fn, dependency=[dep_fn]) tasks.append(t) logger.info("added tasks for {0} option generation tasks".format( len(tasks))) return tasks
def toc_tasks(conf): tocs = TocDataCache(conf.system.content.toc.sources, conf) tocs.create_output_dir() tasks = [] for dep_fn, toc_data in tocs.file_iter(): deps = [dep_fn] if 'ref-toc-' in dep_fn: base_offset = 8 is_ref = True elif 'ref-spec-' in dep_fn: base_offset = 9 is_ref = True else: base_offset = 4 is_ref = False fn_basename = os.path.basename(dep_fn)[base_offset:].replace( 'yaml', 'rst') toc_items = toc_data.ordered_items() if toc_data.is_spec() is False: out_fn = os.path.join(conf.system.content.toc.output_dir, fn_basename) t = Task(job=write_toc_tree_output, args=(out_fn, toc_items, is_ref), target=out_fn, dependency=dep_fn, description="writing toctree to '{0}'".format(out_fn)) tasks.append(t) else: deps.extend(toc_data.spec_deps()) if 'ref-toc' in dep_fn: out_fn = os.path.join(conf.system.content.toc.output_dir, 'table-' + fn_basename) reft = Task( job=write_toc_table, args=(out_fn, toc_items), target=out_fn, dependency=deps, description="write ref toc table to '{0}'".format(out_fn)) tasks.append(reft) elif 'ref-spec' in dep_fn: out_fn = os.path.join(conf.system.content.toc.output_dir, 'table-spec-' + fn_basename) refspec = Task( job=write_toc_table, args=(out_fn, toc_items), target=out_fn, dependency=deps, description="write ref spec table to '{0}'".format(out_fn)) tasks.append(refspec) else: out_fn = os.path.join(conf.system.content.toc.output_dir, 'dfn-list-' + fn_basename) dt = Task(job=write_dfn_list_output, args=(out_fn, toc_items), target=out_fn, dependency=deps, description="write definition list toc to '{0}'".format( out_fn)) tasks.append(dt) logger.info('added tasks for {0} toc generation tasks'.format(len(tasks))) return tasks
def changelog_tasks(conf): tasks = [] if "jira" not in conf.system.files.data: logger.warning("changelog generation is not configured.") return [] dirname = os.path.join(conf.paths.projectroot, conf.paths.includes, "changelogs") giza.tools.files.safe_create_directory(os.path.join(dirname, "releases")) jira_config = os.path.join(conf.paths.projectroot, conf.paths.builddata, "jira.yaml") major_versions = get_major_version_groupings( conf.system.files.data.jira.site.versions) # If no version listed in jira.yaml, just return; Should be same # as if jira is not configured in conf.system.files.data # Also, log explicit message stating that 0 changelog tasks added. if not major_versions: logger.warning("changelog version is not configured in jira.yaml.") logger.info("added {0} changelog tasks.".format(len(tasks))) return [] # don't generate changelog content except on the most recent published # branch (i.e. master, typically.). if conf.git.branches.current != conf.git.branches.published[0]: logger.error( "you must generate changelogs on the master branch and them backport them to another branch." ) logger.info("added {0} changelog tasks".format(len(tasks))) return tasks # only generate changelogs if there are credentials, even though we don't # really need an auth'ed connection, want to avoid making un-authed builds too long. if not os.path.exists( os.path.expanduser(conf.system.files.data.jira.site.credentials)): logger.warning( "jira credentials are not configured for your user. not generating changelog tasks" ) logger.info("added {0} changelog tasks".format(len(tasks))) return tasks # bump mtime of all existing files to avoid regenerating files that already committed files # exist. changelog_releases_dir = os.path.join(conf.paths.projectroot, conf.paths.includes, "changelogs", "releases") for fn in os.listdir(changelog_releases_dir): os.utime(os.path.join(changelog_releases_dir, fn), None) # add tasks for generating intermediate files for each major version. we do # this on all branches, and publishers need to backport the config changes. for version, releases in major_versions.items(): fn = os.path.join(dirname, version + ".rst") t = Task(job=giza.content.changelog.views.render_intermediate_files, args=(fn, version, releases, conf), target=fn, dependency=[jira_config]) tasks.append(t) # create a task for each version defined. should never regenerate existing files. for version in conf.system.files.data.jira.site.versions: fn = os.path.join(conf.paths.projectroot, conf.paths.includes, "changelogs", "releases", version + ".rst") t = Task(job=giza.content.changelog.views.get_changelog_content, args=(fn, version, conf), dependency=[jira_config], target=fn) tasks.append(t) logger.info("added {0} changelog tasks.".format(len(tasks))) return tasks
def finalize_sphinx_build(sconf, conf): target = sconf.builder tasks = [] if target == 'html' and not conf.runstate.fast: t = Task(job=html_tarball, args=(sconf.name, sconf.build_output, conf), target=[ get_tarball_name('html', conf), get_tarball_name('link-html', conf) ], dependency=None, description="creating tarball for html archive") tasks.append(t) elif target == 'dirhtml' and not conf.runstate.fast: for job in (finalize_dirhtml_build, error_pages): t = Task(job=job, args=(sconf, conf), target=os.path.join(conf.paths.projectroot, conf.paths.public_site_output), dependency=None) tasks.append(t) if conf.system.branched is True and conf.git.branches.current == 'master': deps = conf.system.files.get_configs('integration') deps.append( os.path.join(conf.paths.projectroot, conf.paths.public_site_output)) t = Task(job=create_manual_symlink, args=[conf], target=[link[0] for link in get_public_links(conf)], dependency=deps, description='create symlinks') tasks.append(t) elif target == 'epub': t = Task(job=finalize_epub_build, args=(target, conf), description='finalizing epub build', dependency=None, target=True) tasks.append(t) elif target == 'man': t = Task(job=man_tarball, args=(sconf.name, sconf.build_output, conf), target=[ get_tarball_name('man', conf), get_tarball_name('link-man', conf) ], dependency=None, description="creating tarball for manpages") tasks.extend(manpage_url_tasks(target, conf)) tasks.append(('final', t)) elif target == 'slides' and not conf.runstate.fast: tasks.extend(slide_tasks(sconf, conf)) elif target == 'json': json_tasks, transfer_op = json_output_tasks(conf) tasks.extend(json_tasks) tasks.append(('final', transfer_op)) # this is less than ideal elif target == 'singlehtml': tasks.extend(finalize_single_html_tasks(target, conf)) elif target == 'latex': tasks.extend(pdf_tasks(sconf, conf)) elif target == 'gettext': tasks.extend(gettext_tasks(conf)) elif target == 'linkcheck': msg_str = '{0}: See {1}/{0}/output.txt for output.' t = Task(job=printer, args=[msg_str.format(target, conf.paths.branch_output)], target=os.path.join(conf.paths.projectroot, conf.paths.branch_output, target, 'output.txt'), dependency=None) tasks.append(t) logger.info('adding {0} finalizing tasks for {1} build'.format( len(tasks), target)) return tasks