def test_finalizers_nested(self):
t = Task(job=sum,
args=((1, 2, 3), 0))
self.assertEqual(t.finalizers, [])
t.finalizers = [
Task(job=sum,
args=((4, 5, 6, i), 0))
for i in range(10)
]
self.assertEqual(len(t.finalizers), 10)
for task in t.finalizers:
task.finalizers.extend([
Task(job=sum,
args=((4, 5, 6, i), 0))
for i in range(10)
])
self.assertEqual(len(task.finalizers), 10)
self.app.add(t)
results = self.app.run()
self.assertEqual(len(results), 111)
self.assertEqual(results[0], 6)
self.assertEqual(results[1], 15)
for result in results:
self.assertIsInstance(result, numbers.Number)
self.assertTrue(result >= 6)
self.assertTrue(result <= 24)
def test_add_existing_task_object(self):
self.assertEqual(self.app.queue, [])
t = Task()
self.app.add(t)
self.assertIs(t, self.app.queue[0])
self.assertIsNot(t, Task())
self.assertIsNot(Task(), self.app.queue[0])
def test_finalizers_nested(self):
t = self.Task(job=sum,
args=((1, 2, 3), 0))
self.assertEqual(t.finalizers, [])
t.finalizers = [
Task(job=sum,
args=((4, 5, 6, i), 0))
for i in range(10)
]
self.assertEqual(len(t.finalizers), 10)
for task in t.finalizers:
task.finalizers.extend([
Task(job=sum,
args=((4, 5, 6, i), 0))
for i in range(10)
])
self.assertEqual(len(task.finalizers), 10)
task_result = t.run()
self.assertEqual(task_result, 6)
results = [task_result]
results.extend(t.finalize())
self.assertEqual(len(results), 111)
self.assertEqual(results[0], 6)
self.assertEqual(results[1], 15)
for result in results:
self.assertIsInstance(result, numbers.Number)
self.assertTrue(result >= 6)
self.assertTrue(result <= 24)
def extract_tasks(conf):
extracts = ExtractDataCache(conf.system.content.extracts.sources, conf)
extracts.create_output_dir()
tasks = []
for dep_fn, extract in extracts.content_iter():
t = Task(job=write_extract_file,
args=(extract, extract.target),
description="generating extract file: " + extract.target,
target=extract.target,
dependency=dep_fn)
tasks.append(t)
include_statement = get_include_statement(extract.target_project_path)
for verb, adjc, files in [(prepend_to_file, 'prepend',
extract.prepend),
(append_to_file, 'append', extract.append)]:
# have to run appends and prepends always, because the rsync that
# populates build/<branch>/source should and does overwrite these
# files on every source generation step. None in the dep list does this.
for fn in files:
msg = "{0} extract include for '{0}' to '{1}'".format(
adjc, extract.target, fn)
t = Task(job=verb,
args=(fn, include_statement),
target=fn,
dependency=[None, dep_fn],
description=msg)
tasks.append(t)
logger.info("added tasks for {0} extract generation tasks".format(
len(tasks)))
return tasks
def test_single_runner_task(self):
self.assertEqual(self.app.queue, [])
self.assertEqual(self.app.results, [])
t = Task()
t.job = sum
t.description = 'test task'
t.args = [[1, 2], 0]
self.app.add(t)
self.app.run(t)
self.assertEqual(self.app.results[0], 3)
def test_single_runner_task(self):
self.assertEqual(self.app.queue, [])
self.assertEqual(self.app.results, [])
t = Task()
t.job = sum
t.description = 'test task'
t.args = [[1, 2], 0]
self.app.add(t)
self.app.run(t)
self.assertEqual(self.app.results[0], 3)
def test_finalizers_simple(self):
t = Task(job=sum, args=((1, 2, 3), 0))
self.assertEqual(t.finalizers, [])
t.finalizers = [
Task(job=sum, args=((4, 5, 6, i), 0)) for i in range(10)
]
self.app.add(t)
results = self.app.run()
self.assertEqual(len(results), 11)
self.assertEqual(results, [6, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def setUp(self):
self.c = Configuration()
self.c.runstate = RuntimeStateConfig()
self.task = Task()
self.task.job = sum
self.Task = Task
self.task.conf = self.c
def test_conf_object_consistent_in_new_task(self):
self.assertEqual(self.app.queue, [])
t = Task()
self.assertIsNone(t.conf)
self.app.add(t)
self.assertIsNotNone(t.conf)
self.assertIs(self.c, self.app.queue[0].conf)
self.assertIs(self.c, t.conf)
def test_finalizers_simple(self):
t = Task(job=sum,
args=((1, 2, 3), 0))
self.assertEqual(t.finalizers, [])
t.finalizers = [
Task(job=sum,
args=((4, 5, 6, i), 0))
for i in range(10)
]
self.app.add(t)
results = self.app.run()
self.assertEqual(len(results), 11)
self.assertEqual(results,
[6, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def glossary_clean(conf):
return [
Task(job=shutil.rmtree,
args=[conf.system.content.glossary.output_dir],
target=True,
depdency=[conf.system.content.glossary.output_dir],
descrption='removing {0}'.format(
conf.system.content.glossary.output_dir))
]
def example_clean(conf):
return [
Task(job=shutil.rmtree,
args=[conf.system.content.examples.output_dir],
target=True,
dependency=[conf.system.content.examples.output_dir],
description='removing {0}'.format(
conf.system.content.examples.output_dir))
]
def add(self, task=None, conf=None):
"""
Adds a new :class:`~giza.app.BuildApp()` or :class:`~giza.task.Task()`
to the :class:`~giza.app.BuildApp()` object.
:param string,Task,BuildApp task: Optional. If not specified,
:meth:`~giza.app.BuildApp.add()` creates and returns a new
:class:`~giza.task.Task()` object. You can pass the string ``task``
or the class :class:`~giza.task.Task` to explicitly create a new
Task, or pass an existing :class:`~giza.task.Task()` instance to add
that task to the :class:`~giza.app.BuildApp()` instance. You can
also pass the string ``app`` or the :class:`~giza.app.BuildApp`
class, to create and add new :class:`~giza.app.BuildApp()`: pass an
existing :class:`~giza.app.BuildApp()` instance to add that that
operation grouping to the queue.
:returns: A reference to a :class:`~giza.app.BuildApp()` or
:class:`~giza.task.Task()` object in the :class:`~giza.app.BuildApp()`
:raises: :exc:`TypeError` if the ``task`` argument is invalid.
"""
if conf is not None:
self.conf = conf
if task is None or task in (Task, 'task'):
t = Task()
t.conf = self.conf
t.force = self.force
self.queue.append(t)
return t
elif task in (MapTask, 'map'):
t = MapTask()
t.conf = self.conf
t.force = self.force
self.queue.append(t)
return t
elif task in (BuildApp, 'app'):
t = self.sub_app()
self.queue.append(t)
return t
else:
if isinstance(task, Task):
task.force = self.force
if task.conf is None:
task.conf = self.conf
self.queue.append(task)
return task
elif isinstance(task, BuildApp):
task.root_app = False
task.defualt_pool = self.default_pool
task.force = self.force
task.pool = self.pool
self.queue.append(task)
return task
else:
raise TypeError('invalid task type')
def add(self, task=None, conf=None):
"""
Adds a new :class:`~giza.app.BuildApp()` or :class:`~giza.task.Task()`
to the :class:`~giza.app.BuildApp()` object.
:param string,Task,BuildApp task: Optional. If not specified,
:meth:`~giza.app.BuildApp.add()` creates and returns a new
:class:`~giza.task.Task()` object. You can pass the string ``task``
or the class :class:`~giza.task.Task` to explicitly create a new
Task, or pass an existing :class:`~giza.task.Task()` instance to add
that task to the :class:`~giza.app.BuildApp()` instance. You can
also pass the string ``app`` or the :class:`~giza.app.BuildApp`
class, to create and add new :class:`~giza.app.BuildApp()`: pass an
existing :class:`~giza.app.BuildApp()` instance to add that that
operation grouping to the queue.
:returns: A reference to a :class:`~giza.app.BuildApp()` or
:class:`~giza.task.Task()` object in the :class:`~giza.app.BuildApp()`
:raises: :exc:`TypeError` if the ``task`` argument is invalid.
"""
if conf is not None:
self.conf = conf
if task is None or task in (Task, 'task'):
t = Task()
t.conf = self.conf
t.force = self.force
self.queue.append(t)
return t
elif task in (MapTask, 'map'):
t = MapTask()
t.conf = self.conf
t.force = self.force
self.queue.append(t)
return t
elif task in (BuildApp, 'app'):
t = self.sub_app()
self.queue.append(t)
return t
else:
if isinstance(task, Task):
task.force = self.force
if task.conf is None:
task.conf = self.conf
self.queue.append(task)
return task
elif isinstance(task, BuildApp):
task.root_app = False
task.defualt_pool = self.default_pool
task.force = self.force
task.pool = self.pool
self.queue.append(task)
return task
else:
raise TypeError('invalid task type')
def option_clean(conf):
register_options(conf)
tasks = []
for fn in conf.system.options.sources:
t = Task(job=verbose_remove,
args=[fn],
target=fn,
dependency=None,
description='removing {0}'.format(fn))
tasks.append(t)
return tasks
def sphinx_tasks(sconf, conf):
# Projects that use the append functionality in extracts or similar content
# generators will rebuild this task every time.
deps = [os.path.join(conf.paths.projectroot, 'conf.py')]
deps.extend(conf.system.files.get_configs('sphinx_local'))
deps.extend(expand_tree(os.path.join(conf.paths.projectroot, conf.paths.branch_source), 'txt'))
return Task(job=run_sphinx,
args=(sconf.builder, sconf, conf),
target=os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
sconf.builder),
dependency=deps,
description='building {0} with sphinx'.format(sconf.builder))
def step_tasks(conf):
s = StepDataCache(conf.system.content.steps.sources, conf)
s.create_output_dir()
tasks = []
for fn, stepf in s.file_iter():
t = Task(job=write_steps,
args=(stepf, stepf.target(fn), conf),
description='generate a stepfile for ' + fn,
target=stepf.target(fn),
dependency=fn)
tasks.append(t)
logger.info("added tasks for {0} step generation tasks".format(len(tasks)))
return tasks
def glossary_tasks(conf):
terms = GlossaryDataCache(conf.system.content.glossary.sources, conf)
terms.create_output_dir()
tasks = []
for fn, glossary_file in terms.file_iter():
tasks.append(
Task(job=write_glossary,
args=(glossary_file, glossary_file.target(fn)),
description="generate glossary for: " + fn,
target=glossary_file.target(fn),
dependency=fn))
logger.info("add {0} glossary tasks".format(len(tasks)))
return tasks
def release_tasks(conf):
rel = ReleaseDataCache(conf.system.content.releases.sources, conf)
rel.create_output_dir()
tasks = []
for dep_fn, release in rel.content_iter():
t = Task(job=write_release_file,
args=(release, release.target, conf),
description='generating release spec file: ' + release.target,
target=release.target,
dependency=dep_fn)
tasks.append(t)
logger.info("added tasks for {0} release generation tasks".format(
len(tasks)))
return tasks
def example_tasks(conf):
d = ExampleDataCache(conf.system.content.examples.sources, conf)
d.create_output_dir()
tasks = []
for fn, exmpf in d.file_iter():
out_fn = os.path.join(
conf.system.content.examples.output_dir,
conf.system.content.examples.get_basename(fn)) + '.rst'
t = Task(job=write_full_example,
args=(exmpf.collection, exmpf.examples, out_fn),
description='generate an example for ' + fn,
target=out_fn,
dependency=fn)
tasks.append(t)
logger.info("added tasks for {0} example generation tasks".format(
len(tasks)))
return tasks
def test_finalizers_simple(self):
t = self.Task(job=sum,
args=((1, 2, 3), 0))
self.assertEqual(t.finalizers, [])
t.finalizers = [
Task(job=sum,
args=((4, 5, 6, i), 0))
for i in range(10)
]
self.assertEqual(len(t.finalizers), 10)
self.assertEqual(t.run(), 6)
self.assertEqual(len(t.finalizers), 10)
finals = t.finalize()
self.assertEqual(len(finals), 10)
self.assertEqual(len(t.finalizers), 10)
self.assertEqual(finals,
[15, 16, 17, 18, 19, 20, 21, 22, 23, 24])
def apiarg_tasks(conf):
a = ApiArgDataCache(conf.system.content.apiargs.sources, conf)
a.create_output_dir()
tasks = []
for dep_fn, apiargs in a.file_iter():
basename = conf.system.content.steps.get_basename(dep_fn)[2:]
out_fn = os.path.join(conf.system.content.apiargs.output_dir,
basename) + '.rst'
t = Task(job=write_apiargs,
args=(apiargs, out_fn),
target=out_fn,
dependency=dep_fn,
description="write apiarg table for: " + dep_fn)
tasks.append(t)
logger.info('added tasks for {0} apiarg table generation tasks'.format(
len(tasks)))
return tasks
def option_tasks(conf):
o = OptionDataCache(conf.system.content.options.sources, conf)
o.create_output_dir()
tasks = []
for dep_fn, option in o.content_iter():
output_fn = os.path.join(
conf.system.content.options.fn_prefix, ''.join(
(option.directive, '-', option.program, '-',
option.name + '.rst')))
t = Task(job=write_options,
args=(option, output_fn, conf),
description='generating option file "{0}" from "{1}"'.format(
output_fn, dep_fn),
target=output_fn,
dependency=[dep_fn])
tasks.append(t)
logger.info("added tasks for {0} option generation tasks".format(
len(tasks)))
return tasks
def toc_tasks(conf):
tocs = TocDataCache(conf.system.content.toc.sources, conf)
tocs.create_output_dir()
tasks = []
for dep_fn, toc_data in tocs.file_iter():
deps = [dep_fn]
if 'ref-toc-' in dep_fn:
base_offset = 8
is_ref = True
elif 'ref-spec-' in dep_fn:
base_offset = 9
is_ref = True
else:
base_offset = 4
is_ref = False
fn_basename = os.path.basename(dep_fn)[base_offset:].replace(
'yaml', 'rst')
toc_items = toc_data.ordered_items()
if toc_data.is_spec() is False:
out_fn = os.path.join(conf.system.content.toc.output_dir,
fn_basename)
t = Task(job=write_toc_tree_output,
args=(out_fn, toc_items, is_ref),
target=out_fn,
dependency=dep_fn,
description="writing toctree to '{0}'".format(out_fn))
tasks.append(t)
else:
deps.extend(toc_data.spec_deps())
if 'ref-toc' in dep_fn:
out_fn = os.path.join(conf.system.content.toc.output_dir,
'table-' + fn_basename)
reft = Task(
job=write_toc_table,
args=(out_fn, toc_items),
target=out_fn,
dependency=deps,
description="write ref toc table to '{0}'".format(out_fn))
tasks.append(reft)
elif 'ref-spec' in dep_fn:
out_fn = os.path.join(conf.system.content.toc.output_dir,
'table-spec-' + fn_basename)
refspec = Task(
job=write_toc_table,
args=(out_fn, toc_items),
target=out_fn,
dependency=deps,
description="write ref spec table to '{0}'".format(out_fn))
tasks.append(refspec)
else:
out_fn = os.path.join(conf.system.content.toc.output_dir,
'dfn-list-' + fn_basename)
dt = Task(job=write_dfn_list_output,
args=(out_fn, toc_items),
target=out_fn,
dependency=deps,
description="write definition list toc to '{0}'".format(
out_fn))
tasks.append(dt)
logger.info('added tasks for {0} toc generation tasks'.format(len(tasks)))
return tasks
def changelog_tasks(conf):
tasks = []
if "jira" not in conf.system.files.data:
logger.warning("changelog generation is not configured.")
return []
dirname = os.path.join(conf.paths.projectroot, conf.paths.includes,
"changelogs")
giza.tools.files.safe_create_directory(os.path.join(dirname, "releases"))
jira_config = os.path.join(conf.paths.projectroot, conf.paths.builddata,
"jira.yaml")
major_versions = get_major_version_groupings(
conf.system.files.data.jira.site.versions)
# If no version listed in jira.yaml, just return; Should be same
# as if jira is not configured in conf.system.files.data
# Also, log explicit message stating that 0 changelog tasks added.
if not major_versions:
logger.warning("changelog version is not configured in jira.yaml.")
logger.info("added {0} changelog tasks.".format(len(tasks)))
return []
# don't generate changelog content except on the most recent published
# branch (i.e. master, typically.).
if conf.git.branches.current != conf.git.branches.published[0]:
logger.error(
"you must generate changelogs on the master branch and them backport them to another branch."
)
logger.info("added {0} changelog tasks".format(len(tasks)))
return tasks
# only generate changelogs if there are credentials, even though we don't
# really need an auth'ed connection, want to avoid making un-authed builds too long.
if not os.path.exists(
os.path.expanduser(conf.system.files.data.jira.site.credentials)):
logger.warning(
"jira credentials are not configured for your user. not generating changelog tasks"
)
logger.info("added {0} changelog tasks".format(len(tasks)))
return tasks
# bump mtime of all existing files to avoid regenerating files that already committed files
# exist.
changelog_releases_dir = os.path.join(conf.paths.projectroot,
conf.paths.includes, "changelogs",
"releases")
for fn in os.listdir(changelog_releases_dir):
os.utime(os.path.join(changelog_releases_dir, fn), None)
# add tasks for generating intermediate files for each major version. we do
# this on all branches, and publishers need to backport the config changes.
for version, releases in major_versions.items():
fn = os.path.join(dirname, version + ".rst")
t = Task(job=giza.content.changelog.views.render_intermediate_files,
args=(fn, version, releases, conf),
target=fn,
dependency=[jira_config])
tasks.append(t)
# create a task for each version defined. should never regenerate existing files.
for version in conf.system.files.data.jira.site.versions:
fn = os.path.join(conf.paths.projectroot, conf.paths.includes,
"changelogs", "releases", version + ".rst")
t = Task(job=giza.content.changelog.views.get_changelog_content,
args=(fn, version, conf),
dependency=[jira_config],
target=fn)
tasks.append(t)
logger.info("added {0} changelog tasks.".format(len(tasks)))
return tasks
def finalize_sphinx_build(sconf, conf):
target = sconf.builder
tasks = []
if target == 'html' and not conf.runstate.fast:
t = Task(job=html_tarball,
args=(sconf.name, sconf.build_output, conf),
target=[
get_tarball_name('html', conf),
get_tarball_name('link-html', conf)
],
dependency=None,
description="creating tarball for html archive")
tasks.append(t)
elif target == 'dirhtml' and not conf.runstate.fast:
for job in (finalize_dirhtml_build, error_pages):
t = Task(job=job,
args=(sconf, conf),
target=os.path.join(conf.paths.projectroot,
conf.paths.public_site_output),
dependency=None)
tasks.append(t)
if conf.system.branched is True and conf.git.branches.current == 'master':
deps = conf.system.files.get_configs('integration')
deps.append(
os.path.join(conf.paths.projectroot,
conf.paths.public_site_output))
t = Task(job=create_manual_symlink,
args=[conf],
target=[link[0] for link in get_public_links(conf)],
dependency=deps,
description='create symlinks')
tasks.append(t)
elif target == 'epub':
t = Task(job=finalize_epub_build,
args=(target, conf),
description='finalizing epub build',
dependency=None,
target=True)
tasks.append(t)
elif target == 'man':
t = Task(job=man_tarball,
args=(sconf.name, sconf.build_output, conf),
target=[
get_tarball_name('man', conf),
get_tarball_name('link-man', conf)
],
dependency=None,
description="creating tarball for manpages")
tasks.extend(manpage_url_tasks(target, conf))
tasks.append(('final', t))
elif target == 'slides' and not conf.runstate.fast:
tasks.extend(slide_tasks(sconf, conf))
elif target == 'json':
json_tasks, transfer_op = json_output_tasks(conf)
tasks.extend(json_tasks)
tasks.append(('final', transfer_op)) # this is less than ideal
elif target == 'singlehtml':
tasks.extend(finalize_single_html_tasks(target, conf))
elif target == 'latex':
tasks.extend(pdf_tasks(sconf, conf))
elif target == 'gettext':
tasks.extend(gettext_tasks(conf))
elif target == 'linkcheck':
msg_str = '{0}: See {1}/{0}/output.txt for output.'
t = Task(job=printer,
args=[msg_str.format(target, conf.paths.branch_output)],
target=os.path.join(conf.paths.projectroot,
conf.paths.branch_output, target,
'output.txt'),
dependency=None)
tasks.append(t)
logger.info('adding {0} finalizing tasks for {1} build'.format(
len(tasks), target))
return tasks
