def gettext_tasks(conf): locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale, 'pot') builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf) branch_output = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_name) path_offset = len(branch_output) + 1 tasks = [] for fn in expand_tree(branch_output, None): target = os.path.join(locale_dirs, fn[path_offset:]) source = fn t = libgiza.task.Task(job=copy_if_needed, args=(source, target, None), target=target, dependency=source, description="migrating po file {0} if needed".format(fn)) tasks.append(t) return tasks
def gettext_tasks(conf): locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale, 'pot') builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf) branch_output = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_name) path_offset = len(branch_output) + 1 tasks = [] for fn in expand_tree(branch_output, None): target = os.path.join(locale_dirs, fn[path_offset:]) source = fn t = libgiza.task.Task( job=copy_if_needed, args=(source, target, None), target=target, dependency=source, description="migrating po file {0} if needed".format(fn)) tasks.append(t) logger.info( "if you added files to the corpus since your last gettext build," "please run the gettext build a second time for complete finalization." ) return tasks
def main(args): """ Removes build artifacts from ``build/`` directory. """ c = fetch_config(args) app = BuildApp(c) to_remove = set() if c.runstate.git_branch is not None: to_remove.add(os.path.join(c.paths.projectroot, c.paths.branch_output)) if c.runstate.builder != []: for edition, language, builder in get_builder_jobs(c): builder_path = resolve_builder_path(builder, edition, language, c) builder_path = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_path) to_remove.add(builder_path) dirpath, base = os.path.split(builder_path) to_remove.add(os.path.join(dirpath, 'doctrees-' + base)) m = 'remove artifacts associated with the {0} builder in {1} ({2}, {3})' logger.debug( m.format(builder, c.git.branches.current, edition, language)) if c.runstate.days_to_save is not None: published_branches = [ 'docs-tools', 'archive', 'public', 'primer', c.git.branches.current ] published_branches.extend(c.git.branches.published) for build in os.listdir( os.path.join(c.paths.projectroot, c.paths.output)): build = os.path.join(c.paths.projectroot, c.paths.output, build) branch = os.path.split(build)[1] if branch in published_branches: continue elif not os.path.isdir(build): continue elif os.stat(build).st_mtime > c.runstate.days_to_save: to_remove.add(build) to_remove.add( os.path.join(c.paths.projectroot, c.paths.output, 'public', branch)) logger.debug( 'removed stale artifacts: "{0}" and "build/public/{0}"'. format(branch)) for fn in to_remove: t = app.add() t.job = rm_rf t.args = fn m = 'removing artifact: {0}'.format(fn) t.description = m logger.critical(m) app.run()
def gettext_tasks(conf): locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale, 'pot') builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf) branch_output = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_name) path_offset = len(branch_output) + 1 tasks = [] for fn in expand_tree(branch_output, None): target = os.path.join(locale_dirs, fn[path_offset:]) source = fn t = giza.libgiza.task.Task(job=copy_if_needed, args=(source, target, None), target=target, dependency=source, description="migrating po file {0} if needed".format(fn)) tasks.append(t) logger.info("if you added files to the corpus since your last gettext build," "please run the gettext build a second time for complete finalization.") return tasks
def main(args): """ Removes build artifacts from ``build/`` directory. """ c = fetch_config(args) app = BuildApp.new(pool_type=c.runstate.runner, pool_size=c.runstate.pool_size, force=c.runstate.force) to_remove = set() if c.runstate.git_branch is not None: to_remove.add(os.path.join(c.paths.projectroot, c.paths.branch_output)) if c.runstate.builder != []: for edition, language, builder in get_builder_jobs(c): builder_path = resolve_builder_path(builder, edition, language, c) builder_path = os.path.join(c.paths.projectroot, c.paths.branch_output, builder_path) to_remove.add(builder_path) dirpath, base = os.path.split(builder_path) to_remove.add(os.path.join(dirpath, 'doctrees-' + base)) m = 'remove artifacts associated with the {0} builder in {1} ({2}, {3})' logger.debug(m.format(builder, c.git.branches.current, edition, language)) if c.runstate.days_to_save is not None: published_branches = ['docs-tools', 'archive', 'public', 'primer', c.git.branches.current] published_branches.extend(c.git.branches.published) for build in os.listdir(os.path.join(c.paths.projectroot, c.paths.output)): build = os.path.join(c.paths.projectroot, c.paths.output, build) branch = os.path.split(build)[1] if branch in published_branches: continue elif not os.path.isdir(build): continue elif os.stat(build).st_mtime > c.runstate.days_to_save: to_remove.add(build) to_remove.add(os.path.join(c.paths.projectroot, c.paths.output, 'public', branch)) logger.debug('removed stale artifacts: "{0}" and "build/public/{0}"'.format(branch)) for fn in to_remove: if os.path.isdir(fn): job = shutil.rmtree else: job = os.remove t = app.add('task') t.job = job t.args = fn m = 'removing artifact: {0}'.format(fn) t.description = m logger.critical(m) app.run()
def package_build_env(builders, editions, languages, conf): arc_fn = hyph_concat('cache', conf.project.name, conf.git.branches.current, datetime.datetime.utcnow().strftime('%s'), conf.git.commit[:8]) + ".tar.gz" archive_path = os.path.join(conf.paths.buildarchive, arc_fn) safe_create_directory(conf.paths.buildarchive) existing_archives = os.listdir(conf.paths.buildarchive) for arc in existing_archives: if conf.git.commit[:8] in arc: logger.warning( 'archive "{0}" exists for current git hash, not recreating'. format(archive_path)) return logger.debug("no archive for commit '{0}' continuing".format( conf.git.commit)) with cd(conf.paths.projectroot): files_to_archive = set() for edition, language, builder in itertools.product( editions, languages, builders): rconf, sconf = get_sphinx_build_configuration( edition, language, builder, copy.deepcopy(conf.runstate)) builder_dirname = resolve_builder_path(builder, edition, language, rconf) files_to_archive.add(rconf.paths.branch_source) files_to_archive.add( os.path.join(rconf.paths.branch_output, builder_dirname)) files_to_archive.add( os.path.join(rconf.paths.branch_output, hyph_concat('doctrees', builder_dirname))) files_to_archive.add(rconf.system.dependency_cache_fn) files_to_archive = list(files_to_archive) logger.info('prepped build cache archive. writing file now.') for fn in files_to_archive: if not os.path.exists(fn): raise FileNotFoundError(fn) try: with tarfile.open(archive_path, 'w:gz') as t: for fn in files_to_archive: t.add(fn, exclude=is_git_dir) logger.info("created build-cache archive: " + archive_path) except Exception as e: os.remove(archive_path) logger.critical("failed to create archive: " + archive_path) logger.error(e)
def gettext_tasks(conf, app): locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale, 'pot') builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf) branch_output = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_name) path_offset = len(branch_output) + 1 for fn in expand_tree(branch_output, None): task = app.add('task') task.target = fn task.job = copy_if_needed task.args = [ fn, os.path.join(locale_dirs, fn[path_offset:]), None] task.description = "migrating po file {0} if needed".format(fn)
def gettext_tasks(conf, app): locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale, 'pot') builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf) branch_output = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_name) path_offset = len(branch_output) + 1 for fn in expand_tree(branch_output, None): target = os.path.join(locale_dirs, fn[path_offset:]) source = fn task = app.add('task') task.target = target task.dependency = source task.job = copy_if_needed task.args = [source, target, None] task.description = "migrating po file {0} if needed".format(fn)
def image_tasks(conf, sconf): tasks = [] deps = conf.system.files.get_configs('images') deps.append(os.path.abspath(__file__)) if 'images' not in conf.system.files.data: return [] giza.tools.files.safe_create_directory( os.path.join(conf.paths.projectroot, conf.paths.branch_images)) for image in conf.system.files.data.images: if not os.path.isfile(image.source_core): logger.error('"{0}" does not exist'.format(image.source_core)) continue description = "generating rst include file {0} for {1}".format( image.rst_file, image.source_core) t = libgiza.task.Task( job=giza.content.images.views.generate_image_pages, args=(image, conf), # as kwargs target=image.rst_file, dependency=deps, description=description) tasks.append(t) if conf.runstate.fast is True: continue for output in image.outputs: description = 'generating image file {0} from {1}'.format( output.output, image.source_core) t = libgiza.task.Task(job=generate_image_inkscape, args=(output.build_type, output.dpi, output.width, output.output, image.source_file), target=output.output, dependency=image.source_core, description=description) tasks.append(t) if output.type == 'target': image_output = os.path.join( conf.paths.projectroot, conf.paths.branch_output, resolve_builder_path(sconf.builder, conf.project.edition, None, conf), '_images', ''.join( [image.name, '-', output.tag, '.', output.build_type])) description = 'copying fullsize image file {0} from {1}'.format( image_output, output.output) t.add_finalizer( libgiza.task.Task(job=giza.tools.files.copy_if_needed, args=(output.output, image_output), description=description, target=image_output, dependency=None)) logger.info('registered {0} image generation tasks'.format(len(tasks))) return tasks
def image_tasks(conf, sconf): tasks = [] deps = conf.system.files.get_configs('images') deps.append(os.path.abspath(__file__)) if 'images' not in conf.system.files.data: return [] giza.tools.files.safe_create_directory(os.path.join(conf.paths.projectroot, conf.paths.branch_images)) for image in conf.system.files.data.images: if not os.path.isfile(image.source_core): logger.error('"{0}" does not exist'.format(image.source_core)) continue description = "generating rst include file {0} for {1}".format(image.rst_file, image.source_core) t = libgiza.task.Task(job=giza.content.images.views.generate_image_pages, args=(image, conf), # as kwargs target=image.rst_file, dependency=deps, description=description) tasks.append(t) if conf.runstate.fast is True: continue for output in image.outputs: description = 'generating image file {0} from {1}'.format(output.output, image.source_core) t = libgiza.task.Task(job=generate_image_inkscape, args=(output.build_type, output.dpi, output.width, output.output, image.source_file), target=output.output, dependency=image.source_core, description=description) tasks.append(t) if output.type == 'target': image_output = os.path.join(conf.paths.projectroot, conf.paths.branch_output, resolve_builder_path(sconf.builder, conf.project.edition, None, conf), '_images', ''.join([image.name, '-', output.tag, '.', output.build_type])) description = 'copying fullsize image file {0} from {1}'.format(image_output, output.output) t.add_finalizer(libgiza.task.Task(job=giza.tools.files.copy_if_needed, args=(output.output, image_output), description=description, target=image_output, dependency=None)) logger.info('registered {0} image generation tasks'.format(len(tasks))) return tasks