def dump_file_hashes(conf):
output = conf.system.dependency_cache
o = { 'conf': conf.dict(),
'time': datetime.datetime.utcnow().strftime("%s"),
'files': { }
}
files = expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
output_dir = os.path.dirname(output)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output, 'w') as f:
json.dump(o, f)
logger.info('wrote dependency cache to: {0}'.format(output))
def example_tasks(conf, app):
include_dir = os.path.join(conf.paths.projectroot, conf.paths.includes)
fn_prefix = os.path.join(include_dir, 'example')
example_sources = [ fn for fn in
expand_tree(include_dir, 'yaml')
if fn.startswith(fn_prefix) ]
d = ExampleDataCache(example_sources, conf)
if not os.path.isdir(fn_prefix):
os.makedirs(fn_prefix)
for fn in d.cache.keys():
exmpf = d.cache[fn]
basename = fn[len(fn_prefix)+1:-5]
out_fn = os.path.join(conf.paths.projectroot,
conf.paths.branch_source,
'includes', 'examples', basename) + '.rst'
t = app.add('task')
t.target = out_fn
t.dependency = fn
t.job = write_full_example
t.args = (exmpf.collection, exmpf.examples, out_fn)
t.description = 'generate an example for ' + basename
logger.debug('added all tasks for example generation')
def finalize_single_html_jobs(builder, conf):
pjoin = os.path.join
single_html_dir = get_single_html_dir(conf)
if not os.path.exists(single_html_dir):
os.makedirs(single_html_dir)
try:
manual_single_html(input_file=pjoin(conf.paths.branch_output, builder,
'contents.html'),
output_file=pjoin(single_html_dir, 'index.html'))
except (IOError, OSError):
manual_single_html(input_file=pjoin(conf.paths.branch_output, builder,
'index.html'),
output_file=pjoin(single_html_dir, 'index.html'))
copy_if_needed(source_file=pjoin(conf.paths.branch_output, builder,
'objects.inv'),
target_file=pjoin(single_html_dir, 'objects.inv'))
single_path = pjoin(single_html_dir, '_static')
for fn in expand_tree(pjoin(conf.paths.branch_output, builder, '_static'),
None):
yield {
'job': copy_if_needed,
'args': [fn, pjoin(single_path, os.path.basename(fn))],
'target': None,
'dependency': None
}
def gettext_tasks(conf):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
tasks = []
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
t = libgiza.task.Task(job=copy_if_needed,
args=(source, target, None),
target=target,
dependency=source,
description="migrating po file {0} if needed".format(fn))
tasks.append(t)
return tasks
def example_tasks(conf, app):
include_dir = os.path.join(conf.paths.projectroot, conf.paths.includes)
fn_prefix = os.path.join(include_dir, 'example')
example_sources = [ fn for fn in
expand_tree(include_dir, 'yaml')
if fn.startswith(fn_prefix) ]
d = ExampleDataCache(example_sources, conf)
if not os.path.isdir(fn_prefix):
os.makedirs(fn_prefix)
for fn in d.cache.keys():
exmpf = d.cache[fn]
basename = fn[len(fn_prefix)+1:-5]
out_fn = os.path.join(conf.paths.projectroot,
conf.paths.branch_source,
'includes', 'examples', basename) + '.rst'
t = app.add('task')
t.target = out_fn
t.dependency = fn
t.job = write_full_example
t.args = (exmpf.collection, exmpf.examples, out_fn)
t.description = 'generate an example for ' + basename
logger.debug('added all tasks for example generation')
def finalize_single_html_jobs(builder, conf):
pjoin = os.path.join
single_html_dir = get_single_html_dir(conf)
if not os.path.exists(single_html_dir):
os.makedirs(single_html_dir)
try:
manual_single_html(input_file=pjoin(conf.paths.branch_output,
builder, 'contents.html'),
output_file=pjoin(single_html_dir, 'index.html'))
except (IOError, OSError):
manual_single_html(input_file=pjoin(conf.paths.branch_output,
builder, 'index.html'),
output_file=pjoin(single_html_dir, 'index.html'))
copy_if_needed(source_file=pjoin(conf.paths.branch_output,
builder, 'objects.inv'),
target_file=pjoin(single_html_dir, 'objects.inv'))
single_path = pjoin(single_html_dir, '_static')
for fn in expand_tree(pjoin(conf.paths.branch_output,
builder, '_static'), None):
yield {
'job': copy_if_needed,
'args': [fn, pjoin(single_path, os.path.basename(fn))],
'target': None,
'dependency': None
}
def finalize_single_html_tasks(builder, conf):
single_html_dir = get_single_html_dir(conf)
# create directory when registering tasks.
safe_create_directory(single_html_dir)
safe_create_directory(os.path.join(single_html_dir, '_static'))
if 'edition' in conf.project and conf.project.edition != conf.project.name:
artifact_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
'-'.join((builder, conf.project.edition)))
else:
artifact_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder)
tasks = [giza.libgiza.task.Task(job=finalize_single_html,
args=(single_html_dir, artifact_dir, conf),
target=True,
dependency=None,
description="migrating singlehtml")]
for fn in expand_tree(os.path.join(artifact_dir, '_static'), None):
target_fn = os.path.join(single_html_dir, '_static', os.path.basename(fn))
tasks.append(giza.libgiza.task.Task(job=copy_if_needed,
args=(fn, target_fn),
target=target_fn,
dependency=fn,
description="moving static files to the singlehtml build"))
return tasks
def gettext_tasks(conf):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
tasks = []
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
t = giza.libgiza.task.Task(job=copy_if_needed,
args=(source, target, None),
target=target,
dependency=source,
description="migrating po file {0} if needed".format(fn))
tasks.append(t)
logger.info("if you added files to the corpus since your last gettext build,"
"please run the gettext build a second time for complete finalization.")
return tasks
def gettext_tasks(conf):
locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale,
'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None,
conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder_name)
path_offset = len(branch_output) + 1
tasks = []
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
t = libgiza.task.Task(
job=copy_if_needed,
args=(source, target, None),
target=target,
dependency=source,
description="migrating po file {0} if needed".format(fn))
tasks.append(t)
logger.info(
"if you added files to the corpus since your last gettext build,"
"please run the gettext build a second time for complete finalization."
)
return tasks
def toc_tasks(conf, app):
paths = conf.paths
for fn in expand_tree(paths.includes, 'yaml'):
if not (fn.startswith(os.path.join(paths.includes, 'toc')) or
fn.startswith(os.path.join(paths.includes, 'ref-toc')) or
fn.startswith(os.path.join(paths.includes, 'ref-spec'))):
continue
elif len(fn) >= 24:
task = app.add('task')
base_name = _get_toc_base_name(fn)
target = []
fmt = fn[20:24]
if fmt != 'spec':
fmt = fn[16:19]
task.dependency = os.path.join(paths.projectroot, fn)
task.job = _generate_toc_tree
task.args = [fn, fmt, base_name, paths, conf]
task.description = 'generating {0} from {1}'.format(fmt, fn)
if fmt != 'spec':
target.append(_get_toc_output_name(base_name, 'toc', paths))
is_ref_spec = fn.startswith(os.path.join(os.path.dirname(fn), 'ref-spec'))
if not is_ref_spec and (fmt == 'toc' or fmt == 'spec'):
target.append(_get_toc_output_name(base_name, 'dfn-list', paths))
elif fmt == 'ref' or is_ref_spec:
target.append(_get_toc_output_name(base_name, 'table', paths))
task.target = target
logger.debug('added task for generating toc from {0}'.format(fn))
def manpage_url_tasks(builder, conf):
project_source = os.path.join(conf.paths.projectroot,
conf.paths.source)
top_level_items = set()
for fs_obj in os.listdir(project_source):
if fs_obj.startswith('.static') or fs_obj == 'index.txt':
continue
if os.path.isdir(os.path.join(project_source, fs_obj)):
top_level_items.add(fs_obj)
if fs_obj.endswith('.txt'):
top_level_items.add(fs_obj[:-4])
top_level_items = '/' + r'[^\s]*|/'.join(top_level_items) + r'[^\s]*'
re_string = r'(\\fB({0})\\fP)'.format(top_level_items).replace(r'-', r'\-')
subst = conf.project.url + '/' + conf.project.tag + r'\2'
regex_obj = (re.compile(re_string), subst)
tasks = []
for manpage in expand_tree(os.path.join(conf.paths.projectroot,
conf.paths.output,
conf.git.branches.current,
builder), ['1', '5']):
description = 'processing urls in manpage file: {0}'.format(manpage)
tasks.append(libgiza.task.Task(job=manpage_url,
args=(regex_obj, manpage),
target=manpage,
dependency=None,
description=description))
return tasks
def task(task, conf):
if task == 'source':
legacy_tables = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.source,
'reference'), 'yaml')
dirname = os.path.join(conf.paths.projectroot, conf.paths.includes,
'apiargs')
safe_create_directory(dirname)
offset = len(os.path.join(conf.paths.projectroot, conf.paths.source))
elif task == 'branch':
legacy_tables = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.branch_source,
'reference'), 'yaml')
safe_create_directory(conf.system.content.apiargs.output_dir)
offset = len(
os.path.join(conf.paths.projectroot, conf.paths.branch_source))
else:
logger.critical('cannot perform apiarg migration for: ' + str(task))
return
new_apiarg = []
new_fns = []
for fn in legacy_tables:
new_data, new_fn = migrate_legacy_apiarg(task, fn, conf)
if new_fn in new_fns:
logger.error("duplicate: {0}, from: {1}".format(
os.path.basename(new_fn), os.path.basename(fn)))
else:
new_fns.append(new_fn[offset:])
new_apiarg.append((new_fn, new_data))
for fn, data in new_apiarg:
write_yaml(data, fn)
# for fn in legacy_tables:
# os.remove(fn)
new_sources = conf.system.content.apiargs.sources
if len(new_sources) != len(legacy_tables) and len(legacy_tables) != len(
new_fns):
logger.critical('problem in apiargs table migration.')
else:
logger.info('legacy apiargs tables migrated successfully.')
legacy_tables = [fn[offset:] for fn in legacy_tables]
return zip(legacy_tables, new_fns)
def sources(self):
files = expand_tree(self.dir, 'yaml')
sources = set()
for prefix, fn in itertools.product(self.prefixes, files):
if fn.startswith(self.output_dir[:-1]):
sources.add(fn)
return list(sources)
def json_output_tasks(conf):
regexes = [
(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''),
(re.compile(r'“'), '"'),
(re.compile(r'”'), '"'),
(re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"),
(re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''),
(re.compile(r'>'), '>'),
(re.compile(r'<'), '<')
]
outputs = []
tasks = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
if 'edition' in conf.project and conf.project.edition != conf.project.name:
path = os.path.join(conf.paths.branch_output,
'json-' + conf.project.edition,
os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
else:
path = os.path.join(conf.paths.branch_output,
'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = path + '.fjson'
jsonf = path + '.json'
task = libgiza.task.Task(job=process_json_file,
args=(fjson, jsonf, regexes, conf),
target=jsonf,
dependency=fjson,
description="processing json file".format(json))
tasks.append(task)
outputs.append(jsonf)
list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
tasks.append(libgiza.task.Task(job=generate_list_file,
args=(outputs, list_file, conf),
target=list_file,
dependency=None,
description="generating list of json files"))
transfer = libgiza.task.Task(job=json_output,
args=[conf],
target=True,
dependency=None,
description='transfer json output to public directory')
return tasks, transfer
def example_clean(conf, app):
fn_prefix = os.path.join(include_dir, "example")
example_sources = [fn for fn in expand_tree(include_dir, "yaml") if fn.startswith(fn_prefix)]
for fn in example_sources:
basename = fn[len(fn_prefix) + 1 : -5]
out_fn = (
os.path.join(conf.paths.projectroot, conf.paths.branch_source, "includes", "examples", basename) + ".rst"
)
def json_output_tasks(conf, app):
regexes = [
(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''),
(re.compile(r'“'), '"'),
(re.compile(r'”'), '"'),
(re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"),
(re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''),
(re.compile(r'>'), '>'),
(re.compile(r'<'), '<')
]
outputs = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
if 'edition' in conf.project and conf.project.edition != conf.project.name:
path = os.path.join(conf.paths.branch_output,
'json-' + conf.project.edition,
os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
else:
path = os.path.join(conf.paths.branch_output,
'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = dot_concat(path, 'fjson')
json = dot_concat(path, 'json')
task = app.add('task')
task.target = json
task.dependency = fjson
task.job = process_json_file
task.description = "processing json file".format(json)
task.args = [fjson, json, regexes, conf]
outputs.append(json)
list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
list_task = app.add('task')
list_task.target = list_file
list_task.job = generate_list_file
list_task.args = [outputs, list_file, conf]
output = app.add('app')
out_task = output.add('task')
out_task.job = json_output
out_task.args = [conf]
out_task.description = 'transfer json output to public directory'
def json_output_tasks(conf, app):
regexes = [
(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''),
(re.compile(r'“'), '"'),
(re.compile(r'”'), '"'),
(re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"),
(re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''),
(re.compile(r'>'), '>'),
(re.compile(r'<'), '<')
]
outputs = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
if 'edition' in conf.project and conf.project.edition != conf.project.name:
path = os.path.join(conf.paths.branch_output,
'json-' + conf.project.edition,
os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
else:
path = os.path.join(conf.paths.branch_output,
'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = dot_concat(path, 'fjson')
json = dot_concat(path, 'json')
task = app.add('task')
task.target = json
task.dependency = fjson
task.job = process_json_file
task.description = "processing json file".format(json)
task.args = [fjson, json, regexes, conf]
outputs.append(json)
list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
list_task = app.add('task')
list_task.target = list_file
list_task.job = generate_list_file
list_task.args = [outputs, list_file, conf]
output = app.add('app')
out_task = output.add('task')
out_task.job = json_output
out_task.args = [conf]
out_task.description = 'transfer json output to public directory'
def include_files_unused(conf, inc_files=None):
inc_files = [ fn[6:] for fn in expand_tree(os.path.join(conf.paths.includes), None) ]
mapping = include_files(conf=conf)
results = []
for fn in inc_files:
if fn.endswith('yaml') or fn.endswith('~'):
continue
if fn not in mapping.keys():
results.append(fn)
return results
def example_clean(conf, app):
fn_prefix = os.path.join(include_dir, 'example')
example_sources = [ fn for fn in
expand_tree(include_dir, 'yaml')
if fn.startswith(fn_prefix) ]
for fn in example_sources:
basename = fn[len(fn_prefix)+1:-5]
out_fn = os.path.join(conf.paths.projectroot,
conf.paths.branch_source,
'includes', 'examples', basename) + '.rst'
def steps_tasks(conf, app):
paths = conf.paths
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), 'yaml'):
if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
out_fn = _get_steps_output_fn(fn, paths)
task = app.add('task')
task.dependency = fn
task.target = out_fn
task.job = render_step_file
task.args = [fn, out_fn, conf]
logger.debug('added task to generate step: {0}'.format(out_fn))
def include_files_unused(conf, inc_files=None):
inc_files = [
fn[6:] for fn in expand_tree(os.path.join(conf.paths.includes), None)
]
keys = set(include_files(conf=conf).keys())
results = []
for fn in inc_files:
if fn.endswith('yaml') or fn.endswith('~'):
continue
if fn not in keys:
results.append(fn)
return results
def sphinx_tasks(sconf, conf):
# Projects that use the append functionality in extracts or similar content
# generators will rebuild this task every time.
deps = [os.path.join(conf.paths.projectroot, 'conf.py')]
deps.extend(conf.system.files.get_configs('sphinx_local'))
deps.extend(expand_tree(os.path.join(conf.paths.projectroot, conf.paths.branch_source), 'txt'))
return Task(job=run_sphinx,
args=(sconf.builder, sconf, conf),
target=os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
sconf.builder),
dependency=deps,
description='building {0} with sphinx'.format(sconf.builder))
def sphinx_tasks(sconf, conf):
# Projects that use the append functionality in extracts or similar content
# generators will rebuild this task every time.
deps = [os.path.join(conf.paths.projectroot, 'conf.py')]
deps.extend(conf.system.files.get_configs('sphinx_local'))
deps.extend(expand_tree(os.path.join(conf.paths.projectroot, conf.paths.branch_source), 'txt'))
return Task(job=run_sphinx,
args=(sconf.builder, sconf, conf),
target=os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
sconf.builder),
dependency=deps,
description='building {0} with sphinx'.format(sconf.builder))
def steps_tasks(conf, app):
paths = conf.paths
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes),
'yaml'):
if fn.startswith(
os.path.join(paths.projectroot, paths.includes, 'step')):
out_fn = _get_steps_output_fn(fn, paths)
task = app.add('task')
task.dependency = fn
task.target = out_fn
task.job = render_step_file
task.args = [fn, out_fn, conf]
logger.debug('added task to generate step: {0}'.format(out_fn))
def finalize_single_html_tasks(builder, conf, app):
single_html_dir = get_single_html_dir(conf)
safe_create_directory(single_html_dir)
found_src = False
for base_path in (builder, hyph_concat(builder, conf.project.edition)):
if found_src is True:
break
for fn in [
os.path.join(base_path, f)
for f in ('contents.html', 'index.html')
]:
src_fn = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, fn)
if os.path.exists(src_fn):
manual_single_html(input_file=src_fn,
output_file=os.path.join(
single_html_dir, 'index.html'))
copy_if_needed(
source_file=os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
base_path, 'objects.inv'),
target_file=os.path.join(single_html_dir, 'objects.inv'))
found_src = True
break
if found_src is not True:
raise FileNotFoundError('singlehtml source file')
single_path = os.path.join(single_html_dir, '_static')
for fn in expand_tree(os.path.join(os.path.dirname(src_fn), '_static'),
None):
target_fn = os.path.join(single_path, os.path.basename(fn))
task = app.add('task')
task.job = copy_if_needed
task.target = target_fn
task.dependency = fn
task.args = [fn, target_fn]
task.description = "migrating static files to the HTML build"
def generated_includes(conf):
toc_spec_files = []
step_files = []
for fn in expand_tree(os.path.join(conf.paths.includes),
input_extension='yaml'):
base = os.path.basename(fn)
if base.startswith('toc-spec'):
toc_spec_files.append(fn)
elif base.startswith('ref-spec'):
toc_spec_files.append(fn)
elif base.startswith('steps'):
step_files.append(fn)
elif base.startswith('example'):
# example files, for the purpose of this have the same structure as
# steps, so we can just use that:
step_files.append(fn)
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
mapping = {}
for spec_file in toc_spec_files:
if os.path.exists(spec_file):
data = ingest_yaml_doc(spec_file)
else:
continue
deps = [os.path.join(path_prefix, i) for i in data['sources']]
mapping[spec_file[maskl:]] = deps
for step_def in step_files:
data = ingest_yaml_list(step_def)
deps = []
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [os.path.join(path_prefix, i) for i in deps]
mapping[step_def[maskl:]] = deps
return mapping
def get_file_list(path, input_extension):
''' This function wraps around expand tree to return a list of only 1 file
if the user gives a path to a file and not a directory. Otherwise it has
the same functionality
:param string path: path to the file
:param list input_extension: a list (or a single) of extensions that is acceptable
'''
if os.path.isfile(path):
if input_extension is not None:
if isinstance(input_extension, list):
if os.path.splitext(path)[1][1:] not in input_extension:
return []
else:
if not path.endswith(input_extension):
return []
return [path]
else:
return expand_tree(path, input_extension)
def directory_expansion(source_path, page, conf):
new_page = {'sources': expand_tree(source_path, None)}
del page['source']
if 'source_dir' in page:
new_page['source_dir'] = page['source_dir']
pages = []
for p in convert_multi_source(new_page):
p.update(page)
p['target'] = os.path.join(conf.paths.projectroot, conf.paths.source,
p['target'],
p['source'][len(source_path) + 1:])
pages.append(p)
return pages
def get_file_list(path, input_extension):
''' This function wraps around expand tree to return a list of only 1 file
if the user gives a path to a file and not a directory. Otherwise it has
the same functionality
:param string path: path to the file
:param list input_extension: a list (or a single) of extensions that is acceptable
'''
if os.path.isfile(path):
if input_extension is not None:
if isinstance(input_extension, list):
if os.path.splitext(path)[1][1:] not in input_extension:
return []
else:
if not path.endswith(input_extension):
return []
return [path]
else:
return expand_tree(path, input_extension)
def gettext_tasks(conf, app):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
task = app.add('task')
task.target = fn
task.job = copy_if_needed
task.args = [ fn, os.path.join(locale_dirs, fn[path_offset:]), None]
task.description = "migrating po file {0} if needed".format(fn)
def generated_includes(conf):
toc_spec_files = []
step_files = []
for fn in expand_tree(os.path.join(conf.paths.includes), input_extension='yaml'):
base = os.path.basename(fn)
if base.startswith('toc-spec'):
toc_spec_files.append(fn)
elif base.startswith('ref-spec'):
toc_spec_files.append(fn)
elif base.startswith('steps'):
step_files.append(fn)
elif base.startswith('example'):
# example files, for the purpose of this have the same structure as
# steps, so we can just use that:
step_files.append(fn)
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
mapping = {}
for spec_file in toc_spec_files:
if os.path.exists(spec_file):
data = ingest_yaml_doc(spec_file)
else:
continue
deps = [ os.path.join(path_prefix, i ) for i in data['sources']]
mapping[spec_file[maskl:]] = deps
for step_def in step_files:
data = ingest_yaml_list(step_def)
deps = []
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [ os.path.join(path_prefix, i ) for i in deps ]
mapping[step_def[maskl:]] = deps
return mapping
def directory_expansion(source_path, page, conf):
new_page = { 'sources': expand_tree(source_path, None)}
del page['source']
if 'source_dir' in page:
new_page['source_dir'] = page['source_dir']
pages = []
for p in convert_multi_source(new_page):
p.update(page)
p['target'] = os.path.join(conf.paths.projectroot,
conf.paths.source,
p['target'],
p['source'][len(source_path)+1:])
pages.append(p)
return pages
def dump_file_hashes(conf):
output = conf.system.dependency_cache
o = {'time': datetime.datetime.utcnow().strftime("%s"),
'files': {}}
files = expand_tree(os.path.join(conf.paths.projectroot, conf.paths.branch_source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
safe_create_directory(os.path.dirname(output))
with open(output, 'w') as f:
json.dump(o, f)
logger.debug('wrote dependency cache to: {0}'.format(output))
def dump_file_hashes(conf):
output = conf.system.dependency_cache
o = {'time': datetime.datetime.utcnow().strftime("%s"), 'files': {}}
files = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.branch_source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
safe_create_directory(os.path.dirname(output))
with open(output, 'w') as f:
json.dump(o, f)
logger.info('wrote dependency cache to: {0}'.format(output))
def finalize_single_html_tasks(builder, conf, app):
single_html_dir = get_single_html_dir(conf)
if not os.path.exists(single_html_dir):
os.makedirs(single_html_dir)
found_src = False
for base_path in (builder, hyph_concat(builder, conf.project.edition)):
if found_src is True:
break
for fn in [ os.path.join(base_path, f) for f in ('contents.html', 'index.html') ]:
src_fn = os.path.join(conf.paths.projectroot, conf.paths.branch_output, fn)
if os.path.exists(src_fn):
manual_single_html(input_file=src_fn,
output_file=os.path.join(single_html_dir, 'index.html'))
copy_if_needed(source_file=os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
base_path, 'objects.inv'),
target_file=os.path.join(single_html_dir, 'objects.inv'))
found_src = True
break
if found_src is not True:
raise FileNotFoundError('singlehtml source file')
single_path = os.path.join(single_html_dir, '_static')
for fn in expand_tree(os.path.join(os.path.dirname(src_fn), '_static'), None):
target_fn = os.path.join(single_path, os.path.basename(fn))
task = app.add('task')
task.job = copy_if_needed
task.target = target_fn
task.dependency = fn
task.args = [fn, target_fn]
task.description = "migrating static files to the HTML build"
def gettext_tasks(conf, app):
locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale,
'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None,
conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder_name)
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
target = os.path.join(locale_dirs, fn[path_offset:])
source = fn
task = app.add('task')
task.target = target
task.dependency = source
task.job = copy_if_needed
task.args = [source, target, None]
task.description = "migrating po file {0} if needed".format(fn)
def generated_includes(conf):
step_files = []
mapping = {}
content_prefixes = []
for _, prefixes in conf.system.content.content_prefixes:
content_prefixes.extend(prefixes)
for fn in expand_tree(os.path.join(conf.paths.includes),
input_extension='yaml'):
base = os.path.basename(fn)
# example/toc-specs files, for the purpose of this have the same
# structure as steps, so we can just use that
for prefix in content_prefixes:
if base.startswith(prefix):
step_files.append(fn)
break
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
for step_def in step_files:
deps = []
with open(step_def, 'r') as f:
data = yaml.safe_load_all(f)
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [os.path.join(path_prefix, i) for i in deps]
mapping[step_def[maskl:]] = deps
return mapping
def toc_tasks(conf, app):
paths = conf.paths
for fn in expand_tree(paths.includes, 'yaml'):
if not (fn.startswith(os.path.join(paths.includes, 'toc'))
or fn.startswith(os.path.join(paths.includes, 'ref-toc'))
or fn.startswith(os.path.join(paths.includes, 'ref-spec'))):
continue
elif len(fn) >= 24:
task = app.add('task')
base_name = _get_toc_base_name(fn)
target = []
fmt = fn[20:24]
if fmt != 'spec':
fmt = fn[16:19]
task.dependency = os.path.join(paths.projectroot, fn)
task.job = _generate_toc_tree
task.args = [fn, fmt, base_name, paths, conf]
task.description = 'generating {0} from {1}'.format(fmt, fn)
if fmt != 'spec':
target.append(_get_toc_output_name(base_name, 'toc', paths))
is_ref_spec = fn.startswith(
os.path.join(os.path.dirname(fn), 'ref-spec'))
if not is_ref_spec and (fmt == 'toc' or fmt == 'spec'):
target.append(
_get_toc_output_name(base_name, 'dfn-list', paths))
elif fmt == 'ref' or is_ref_spec:
target.append(_get_toc_output_name(base_name, 'table', paths))
task.target = target
logger.debug('added task for generating toc from {0}'.format(fn))
def generated_includes(conf):
step_files = []
mapping = {}
content_prefixes = []
for _, prefixes in conf.system.content.content_prefixes:
content_prefixes.extend(prefixes)
for fn in expand_tree(os.path.join(conf.paths.includes), input_extension='yaml'):
base = os.path.basename(fn)
# example/toc-specs files, for the purpose of this have the same
# structure as steps, so we can just use that
for prefix in content_prefixes:
if base.startswith(prefix):
step_files.append(fn)
break
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
for step_def in step_files:
deps = []
with open(step_def, 'r') as f:
data = yaml.safe_load_all(f)
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [os.path.join(path_prefix, i) for i in deps]
mapping[step_def[maskl:]] = deps
return mapping
def finalize_single_html_tasks(builder, conf):
single_html_dir = get_single_html_dir(conf)
# create directory when registering tasks.
safe_create_directory(single_html_dir)
safe_create_directory(os.path.join(single_html_dir, '_static'))
if 'edition' in conf.project and conf.project.edition != conf.project.name:
artifact_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, '-'.join(
(builder, conf.project.edition)))
else:
artifact_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder)
tasks = [
libgiza.task.Task(job=finalize_single_html,
args=(single_html_dir, artifact_dir, conf),
target=True,
dependency=None,
description="migrating singlehtml")
]
for fn in expand_tree(os.path.join(artifact_dir, '_static'), None):
target_fn = os.path.join(single_html_dir, '_static',
os.path.basename(fn))
tasks.append(
libgiza.task.Task(
job=copy_if_needed,
args=(fn, target_fn),
target=target_fn,
dependency=fn,
description="moving static files to the singlehtml build"))
return tasks
def get_migration_specifications(conf):
return [ fn for fn in expand_tree(os.path.join(conf.paths.projectroot,
conf.paths.builddata))
if conf.project.name in os.path.basename(fn) and 'migrations' in fn ]
def include_file_data(conf):
inc_path = os.path.join(conf.paths.includes)
include_file_list = expand_tree(path=inc_path, input_extension=None)
include_graph = include_files(conf=conf)
recursive_use = included_recusively(conf, include_graph)
generated = generated_includes(conf)
omni = {}
for idx, fn in enumerate(include_file_list):
incf = fn[len(conf.paths.source):]
if fn.endswith('~'):
continue
for prefix in suppressed_page_prefixes:
if incf.startswith(prefix):
break
else:
omni[incf] = {
'id': idx,
'name': os.path.splitext(incf)[0],
'path': incf,
}
if incf in generated:
omni[incf]['generated'] = True
else:
omni[incf]['generated'] = False
if incf in recursive_use:
omni[incf]['recursive'] = True
else:
omni[incf]['recursive'] = False
if incf in include_graph:
omni[incf]['num_clients'] = len(include_graph[incf])
omni[incf]['clients'] = []
for cl in include_graph[incf]:
cl, ext = os.path.splitext(cl)
if ext == 'yaml':
continue
if (cl.startswith('/includes/generated/overview') or
cl.startswith('/includes/manpage-')):
continue
omni[incf]['clients'].append(cl)
if len(omni[incf]['clients']) == 0:
omni[incf]['yaml_only'] = True
else:
omni[incf]['yaml_only'] = False
else:
omni[incf]['clients'] = dict()
omni[incf]['num_clients'] = 0
with open(fn, 'r') as f:
omni[incf]['content'] = [ ln.rstrip() for ln in f.readlines() ]
return omni
def option_sources(conf):
output_path = get_option_path(conf)
for fn in expand_tree(get_base_path(conf), 'yaml'):
if fn.startswith(output_path):
yield fn
def main():
files = expand_tree('source/includes/', 'yaml')
spec_fns = []
toc_fns = []
for fn in files:
if 'ref-spec' in fn:
spec_fns.append(fn)
elif 'toc' not in fn:
continue
elif 'spec-new' in fn:
continue
elif 'spec' in fn:
spec_fns.append(fn)
else:
toc_fns.append(fn)
logger.info('have {0} spec files'.format(len(spec_fns)))
logger.info('have {0} toc files'.format(len(toc_fns)))
specs = {}
tocs = {}
for fn in spec_fns:
try:
with open(fn, 'r') as f:
specs[fn] = [ d for d in yaml.safe_load_all(f) if d is not None][0]
except:
continue
for fn in toc_fns:
with open(fn, 'r') as f:
tocs[fn] = [ d for d in yaml.safe_load_all(f) if d is not None]
logger.info('have {0} spec files'.format(len(specs)))
logger.info('have {0} toc files'.format(len(tocs)))
new_specs = {}
for fn in specs:
new_spec = []
spec = specs[fn]
new_specs[fn] = new_spec
for entry in spec['files']:
new_entry = {}
new_spec.append(new_entry)
if 'level' in entry:
new_entry['level'] = entry['level']
else:
new_entry['level'] = 1
if isinstance(entry, basestring):
entry = { 'file': entry }
if 'file' in entry:
new_entry['source'] = {
'file': get_enclosing_file(tocs, entry['file']),
'ref': entry['file']
}
else:
new_entry['text_only'] = True
new_entry['description'] = entry['text']
new_entry['ref'] = "".join( [random.choice(string.letters) for i in xrange(15)] )
with open(fn, 'w') as f:
yaml.safe_dump_all(new_spec, f, default_flow_style=False)
logger.info('wrote: ' + fn)
def step_sources(paths):
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), ' yaml'):
if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
yield fn
def table_sources(conf):
for source in expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.branch_includes),
'yaml'):
if os.path.basename(source).startswith('table'):
yield source
def api_sources(conf):
return expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.branch_source,
'reference'), 'yaml')
def include_file_data(conf):
inc_path = os.path.join(conf.paths.includes)
include_file_list = expand_tree(path=inc_path, input_extension=None)
include_graph = include_files(conf=conf)
recursive_use = included_recusively(conf, include_graph)
generated = generated_includes(conf)
omni = {}
for idx, fn in enumerate(include_file_list):
incf = fn[len(conf.paths.source):]
if fn.endswith('~'):
continue
for prefix in suppressed_page_prefixes:
if incf.startswith(prefix):
break
else:
omni[incf] = {
'id': idx,
'name': os.path.splitext(incf)[0],
'path': incf,
}
if incf in generated:
omni[incf]['generated'] = True
else:
omni[incf]['generated'] = False
if incf in recursive_use:
omni[incf]['recursive'] = True
else:
omni[incf]['recursive'] = False
if incf in include_graph:
omni[incf]['num_clients'] = len(include_graph[incf])
omni[incf]['clients'] = []
for cl in include_graph[incf]:
cl, ext = os.path.splitext(cl)
if ext == 'yaml':
continue
if (cl.startswith('/includes/generated/overview') or
cl.startswith('/includes/manpage-')):
continue
omni[incf]['clients'].append(cl)
if len(omni[incf]['clients']) == 0:
omni[incf]['yaml_only'] = True
else:
omni[incf]['yaml_only'] = False
else:
omni[incf]['clients'] = dict()
omni[incf]['num_clients'] = 0
with open(fn, 'r') as f:
omni[incf]['content'] = [ ln.rstrip() for ln in f.readlines() ]
return omni
def get_migration_specifications(conf):
return [
fn for fn in expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.builddata))
if conf.project.name in os.path.basename(fn) and 'migrations' in fn
]
def main():
files = expand_tree('source/includes/', 'yaml')
spec_fns = []
toc_fns = []
for fn in files:
if 'ref-spec' in fn:
spec_fns.append(fn)
elif 'toc' not in fn:
continue
elif 'spec-new' in fn:
continue
elif 'spec' in fn:
spec_fns.append(fn)
else:
toc_fns.append(fn)
logger.info('have {0} spec files'.format(len(spec_fns)))
logger.info('have {0} toc files'.format(len(toc_fns)))
specs = {}
tocs = {}
for fn in spec_fns:
try:
with open(fn, 'r') as f:
specs[fn] = [
d for d in yaml.safe_load_all(f) if d is not None
][0]
except:
continue
for fn in toc_fns:
with open(fn, 'r') as f:
tocs[fn] = [d for d in yaml.safe_load_all(f) if d is not None]
logger.info('have {0} spec files'.format(len(specs)))
logger.info('have {0} toc files'.format(len(tocs)))
new_specs = {}
for fn in specs:
new_spec = []
spec = specs[fn]
new_specs[fn] = new_spec
for entry in spec['files']:
new_entry = {}
new_spec.append(new_entry)
if 'level' in entry:
new_entry['level'] = entry['level']
else:
new_entry['level'] = 1
if isinstance(entry, basestring):
entry = {'file': entry}
if 'file' in entry:
new_entry['source'] = {
'file': get_enclosing_file(tocs, entry['file']),
'ref': entry['file']
}
else:
new_entry['text_only'] = True
new_entry['description'] = entry['text']
new_entry['ref'] = "".join(
[random.choice(string.letters) for i in xrange(15)])
with open(fn, 'w') as f:
yaml.safe_dump_all(new_spec, f, default_flow_style=False)
logger.info('wrote: ' + fn)
def table_sources(conf):
for source in expand_tree(os.path.join(conf.paths.projectroot, conf.paths.includes), 'yaml'):
if os.path.basename(source).startswith('table'):
yield source
def api_sources(conf):
return expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source, 'reference'), 'yaml')
def step_sources(paths):
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes),
' yaml'):
if fn.startswith(
os.path.join(paths.projectroot, paths.includes, 'step')):
yield fn
