def gettext_tasks(conf):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
tasks = []
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
t = libgiza.task.Task(job=copy_if_needed,
args=(source, target, None),
target=target,
dependency=source,
description="migrating po file {0} if needed".format(fn))
tasks.append(t)
return tasks
def gettext_tasks(conf):
locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale,
'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None,
conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder_name)
path_offset = len(branch_output) + 1
tasks = []
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
t = libgiza.task.Task(
job=copy_if_needed,
args=(source, target, None),
target=target,
dependency=source,
description="migrating po file {0} if needed".format(fn))
tasks.append(t)
logger.info(
"if you added files to the corpus since your last gettext build,"
"please run the gettext build a second time for complete finalization."
)
return tasks
def main(args):
"""
Removes build artifacts from ``build/`` directory.
"""
c = fetch_config(args)
app = BuildApp(c)
to_remove = set()
if c.runstate.git_branch is not None:
to_remove.add(os.path.join(c.paths.projectroot, c.paths.branch_output))
if c.runstate.builder != []:
for edition, language, builder in get_builder_jobs(c):
builder_path = resolve_builder_path(builder, edition, language, c)
builder_path = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder_path)
to_remove.add(builder_path)
dirpath, base = os.path.split(builder_path)
to_remove.add(os.path.join(dirpath, 'doctrees-' + base))
m = 'remove artifacts associated with the {0} builder in {1} ({2}, {3})'
logger.debug(
m.format(builder, c.git.branches.current, edition, language))
if c.runstate.days_to_save is not None:
published_branches = [
'docs-tools', 'archive', 'public', 'primer', c.git.branches.current
]
published_branches.extend(c.git.branches.published)
for build in os.listdir(
os.path.join(c.paths.projectroot, c.paths.output)):
build = os.path.join(c.paths.projectroot, c.paths.output, build)
branch = os.path.split(build)[1]
if branch in published_branches:
continue
elif not os.path.isdir(build):
continue
elif os.stat(build).st_mtime > c.runstate.days_to_save:
to_remove.add(build)
to_remove.add(
os.path.join(c.paths.projectroot, c.paths.output, 'public',
branch))
logger.debug(
'removed stale artifacts: "{0}" and "build/public/{0}"'.
format(branch))
for fn in to_remove:
t = app.add()
t.job = rm_rf
t.args = fn
m = 'removing artifact: {0}'.format(fn)
t.description = m
logger.critical(m)
app.run()
def gettext_tasks(conf):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
tasks = []
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
t = giza.libgiza.task.Task(job=copy_if_needed,
args=(source, target, None),
target=target,
dependency=source,
description="migrating po file {0} if needed".format(fn))
tasks.append(t)
logger.info("if you added files to the corpus since your last gettext build,"
"please run the gettext build a second time for complete finalization.")
return tasks
def main(args):
"""
Removes build artifacts from ``build/`` directory.
"""
c = fetch_config(args)
app = BuildApp.new(pool_type=c.runstate.runner,
pool_size=c.runstate.pool_size,
force=c.runstate.force)
to_remove = set()
if c.runstate.git_branch is not None:
to_remove.add(os.path.join(c.paths.projectroot, c.paths.branch_output))
if c.runstate.builder != []:
for edition, language, builder in get_builder_jobs(c):
builder_path = resolve_builder_path(builder, edition, language, c)
builder_path = os.path.join(c.paths.projectroot, c.paths.branch_output, builder_path)
to_remove.add(builder_path)
dirpath, base = os.path.split(builder_path)
to_remove.add(os.path.join(dirpath, 'doctrees-' + base))
m = 'remove artifacts associated with the {0} builder in {1} ({2}, {3})'
logger.debug(m.format(builder, c.git.branches.current, edition, language))
if c.runstate.days_to_save is not None:
published_branches = ['docs-tools', 'archive', 'public', 'primer', c.git.branches.current]
published_branches.extend(c.git.branches.published)
for build in os.listdir(os.path.join(c.paths.projectroot, c.paths.output)):
build = os.path.join(c.paths.projectroot, c.paths.output, build)
branch = os.path.split(build)[1]
if branch in published_branches:
continue
elif not os.path.isdir(build):
continue
elif os.stat(build).st_mtime > c.runstate.days_to_save:
to_remove.add(build)
to_remove.add(os.path.join(c.paths.projectroot, c.paths.output, 'public', branch))
logger.debug('removed stale artifacts: "{0}" and "build/public/{0}"'.format(branch))
for fn in to_remove:
if os.path.isdir(fn):
job = shutil.rmtree
else:
job = os.remove
t = app.add('task')
t.job = job
t.args = fn
m = 'removing artifact: {0}'.format(fn)
t.description = m
logger.critical(m)
app.run()
def package_build_env(builders, editions, languages, conf):
arc_fn = hyph_concat('cache', conf.project.name, conf.git.branches.current,
datetime.datetime.utcnow().strftime('%s'),
conf.git.commit[:8]) + ".tar.gz"
archive_path = os.path.join(conf.paths.buildarchive, arc_fn)
safe_create_directory(conf.paths.buildarchive)
existing_archives = os.listdir(conf.paths.buildarchive)
for arc in existing_archives:
if conf.git.commit[:8] in arc:
logger.warning(
'archive "{0}" exists for current git hash, not recreating'.
format(archive_path))
return
logger.debug("no archive for commit '{0}' continuing".format(
conf.git.commit))
with cd(conf.paths.projectroot):
files_to_archive = set()
for edition, language, builder in itertools.product(
editions, languages, builders):
rconf, sconf = get_sphinx_build_configuration(
edition, language, builder, copy.deepcopy(conf.runstate))
builder_dirname = resolve_builder_path(builder, edition, language,
rconf)
files_to_archive.add(rconf.paths.branch_source)
files_to_archive.add(
os.path.join(rconf.paths.branch_output, builder_dirname))
files_to_archive.add(
os.path.join(rconf.paths.branch_output,
hyph_concat('doctrees', builder_dirname)))
files_to_archive.add(rconf.system.dependency_cache_fn)
files_to_archive = list(files_to_archive)
logger.info('prepped build cache archive. writing file now.')
for fn in files_to_archive:
if not os.path.exists(fn):
raise FileNotFoundError(fn)
try:
with tarfile.open(archive_path, 'w:gz') as t:
for fn in files_to_archive:
t.add(fn, exclude=is_git_dir)
logger.info("created build-cache archive: " + archive_path)
except Exception as e:
os.remove(archive_path)
logger.critical("failed to create archive: " + archive_path)
logger.error(e)
def gettext_tasks(conf, app):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
task = app.add('task')
task.target = fn
task.job = copy_if_needed
task.args = [ fn, os.path.join(locale_dirs, fn[path_offset:]), None]
task.description = "migrating po file {0} if needed".format(fn)
def gettext_tasks(conf, app):
locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale,
'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None,
conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder_name)
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
task = app.add('task')
task.target = target
task.dependency = source
task.job = copy_if_needed
task.args = [source, target, None]
task.description = "migrating po file {0} if needed".format(fn)
def image_tasks(conf, sconf):
tasks = []
deps = conf.system.files.get_configs('images')
deps.append(os.path.abspath(__file__))
if 'images' not in conf.system.files.data:
return []
giza.tools.files.safe_create_directory(
os.path.join(conf.paths.projectroot, conf.paths.branch_images))
for image in conf.system.files.data.images:
if not os.path.isfile(image.source_core):
logger.error('"{0}" does not exist'.format(image.source_core))
continue
description = "generating rst include file {0} for {1}".format(
image.rst_file, image.source_core)
t = libgiza.task.Task(
job=giza.content.images.views.generate_image_pages,
args=(image, conf), # as kwargs
target=image.rst_file,
dependency=deps,
description=description)
tasks.append(t)
if conf.runstate.fast is True:
continue
for output in image.outputs:
description = 'generating image file {0} from {1}'.format(
output.output, image.source_core)
t = libgiza.task.Task(job=generate_image_inkscape,
args=(output.build_type, output.dpi,
output.width, output.output,
image.source_file),
target=output.output,
dependency=image.source_core,
description=description)
tasks.append(t)
if output.type == 'target':
image_output = os.path.join(
conf.paths.projectroot, conf.paths.branch_output,
resolve_builder_path(sconf.builder, conf.project.edition,
None, conf), '_images',
''.join(
[image.name, '-', output.tag, '.', output.build_type]))
description = 'copying fullsize image file {0} from {1}'.format(
image_output, output.output)
t.add_finalizer(
libgiza.task.Task(job=giza.tools.files.copy_if_needed,
args=(output.output, image_output),
description=description,
target=image_output,
dependency=None))
logger.info('registered {0} image generation tasks'.format(len(tasks)))
return tasks
def image_tasks(conf, sconf):
tasks = []
deps = conf.system.files.get_configs('images')
deps.append(os.path.abspath(__file__))
if 'images' not in conf.system.files.data:
return []
giza.tools.files.safe_create_directory(os.path.join(conf.paths.projectroot,
conf.paths.branch_images))
for image in conf.system.files.data.images:
if not os.path.isfile(image.source_core):
logger.error('"{0}" does not exist'.format(image.source_core))
continue
description = "generating rst include file {0} for {1}".format(image.rst_file,
image.source_core)
t = libgiza.task.Task(job=giza.content.images.views.generate_image_pages,
args=(image, conf), # as kwargs
target=image.rst_file,
dependency=deps,
description=description)
tasks.append(t)
if conf.runstate.fast is True:
continue
for output in image.outputs:
description = 'generating image file {0} from {1}'.format(output.output,
image.source_core)
t = libgiza.task.Task(job=generate_image_inkscape,
args=(output.build_type, output.dpi, output.width,
output.output, image.source_file),
target=output.output,
dependency=image.source_core,
description=description)
tasks.append(t)
if output.type == 'target':
image_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
resolve_builder_path(sconf.builder,
conf.project.edition,
None, conf),
'_images',
''.join([image.name, '-', output.tag,
'.', output.build_type]))
description = 'copying fullsize image file {0} from {1}'.format(image_output,
output.output)
t.add_finalizer(libgiza.task.Task(job=giza.tools.files.copy_if_needed,
args=(output.output, image_output),
description=description,
target=image_output,
dependency=None))
logger.info('registered {0} image generation tasks'.format(len(tasks)))
return tasks
