def example_tasks(conf, app):
include_dir = os.path.join(conf.paths.projectroot, conf.paths.includes)
fn_prefix = os.path.join(include_dir, 'example')
example_sources = [ fn for fn in
expand_tree(include_dir, 'yaml')
if fn.startswith(fn_prefix) ]
d = ExampleDataCache(example_sources, conf)
if not os.path.isdir(fn_prefix):
os.makedirs(fn_prefix)
for fn in d.cache.keys():
exmpf = d.cache[fn]
basename = fn[len(fn_prefix)+1:-5]
out_fn = os.path.join(conf.paths.projectroot,
conf.paths.branch_source,
'includes', 'examples', basename) + '.rst'
t = app.add('task')
t.target = out_fn
t.dependency = fn
t.job = write_full_example
t.args = (exmpf.collection, exmpf.examples, out_fn)
t.description = 'generate an example for ' + basename
logger.debug('added all tasks for example generation')
def manpage_url_tasks(builder, conf, app):
project_source = os.path.join(conf.paths.projectroot,
conf.paths.source)
top_level_items = set()
for fs_obj in os.listdir(project_source):
if fs_obj.startswith('.static') or fs_obj == 'index.txt':
continue
if os.path.isdir(os.path.join(project_source, fs_obj)):
top_level_items.add(fs_obj)
if fs_obj.endswith('.txt'):
top_level_items.add(fs_obj[:-4])
top_level_items = '/'+ r'[^\s]*|/'.join(top_level_items) + r'[^\s]*'
re_string = r'(\\fB({0})\\fP)'.format(top_level_items).replace(r'-', r'\-')
subst = conf.project.url + '/' + conf.project.tag + r'\2'
regex_obj = (re.compile(re_string), subst)
for manpage in expand_tree(os.path.join(conf.paths.projectroot,
conf.paths.output,
conf.git.branches.current,
builder), ['1', '5']):
task = app.add('task')
task.target = manpage
task.job = manpage_url
task.args = [regex_obj, manpage]
task.description = 'processing urls in manpage file: {0}'.format(manpage)
def dump_file_hashes(conf):
output = conf.system.dependency_cache
o = { 'conf': conf.dict(),
'time': datetime.datetime.utcnow().strftime("%s"),
'files': { }
}
files = expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
output_dir = os.path.dirname(output)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output, 'w') as f:
json.dump(o, f)
logger.info('wrote dependency cache to: {0}'.format(output))
def toc_tasks(conf, app):
paths = conf.paths
for fn in expand_tree(paths.includes, 'yaml'):
if not (fn.startswith(os.path.join(paths.includes, 'toc')) or
fn.startswith(os.path.join(paths.includes, 'ref-toc')) or
fn.startswith(os.path.join(paths.includes, 'ref-spec'))):
continue
elif len(fn) >= 24:
task = app.add('task')
base_name = _get_toc_base_name(fn)
target = []
fmt = fn[20:24]
if fmt != 'spec':
fmt = fn[16:19]
task.dependency = os.path.join(paths.projectroot, fn)
task.job = _generate_toc_tree
task.args = [fn, fmt, base_name, paths, conf]
task.description = 'generating {0} from {1}'.format(fmt, fn)
if fmt != 'spec':
target.append(_get_toc_output_name(base_name, 'toc', paths))
is_ref_spec = fn.startswith(os.path.join(os.path.dirname(fn), 'ref-spec'))
if not is_ref_spec and (fmt == 'toc' or fmt == 'spec'):
target.append(_get_toc_output_name(base_name, 'dfn-list', paths))
elif fmt == 'ref' or is_ref_spec:
target.append(_get_toc_output_name(base_name, 'table', paths))
task.target = target
logger.debug('added task for generating toc from {0}'.format(fn))
def finalize_single_html_jobs(builder, conf):
pjoin = os.path.join
single_html_dir = get_single_html_dir(conf)
if not os.path.exists(single_html_dir):
os.makedirs(single_html_dir)
try:
manual_single_html(input_file=pjoin(conf.paths.branch_output,
builder, 'contents.html'),
output_file=pjoin(single_html_dir, 'index.html'))
except (IOError, OSError):
manual_single_html(input_file=pjoin(conf.paths.branch_output,
builder, 'index.html'),
output_file=pjoin(single_html_dir, 'index.html'))
copy_if_needed(source_file=pjoin(conf.paths.branch_output,
builder, 'objects.inv'),
target_file=pjoin(single_html_dir, 'objects.inv'))
single_path = pjoin(single_html_dir, '_static')
for fn in expand_tree(pjoin(conf.paths.branch_output,
builder, '_static'), None):
yield {
'job': copy_if_needed,
'args': [fn, pjoin(single_path, os.path.basename(fn))],
'target': None,
'dependency': None
}
def include_files_unused(conf, inc_files=None):
inc_files = [ fn[6:] for fn in expand_tree(os.path.join(conf.paths.includes), None) ]
mapping = include_files(conf=conf)
results = []
for fn in inc_files:
if fn.endswith('yaml') or fn.endswith('~'):
continue
if fn not in mapping.keys():
results.append(fn)
return results
def steps_tasks(conf, app):
paths = conf.paths
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), 'yaml'):
if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
out_fn = _get_steps_output_fn(fn, paths)
task = app.add('task')
task.dependency = fn
task.target = out_fn
task.job = render_step_file
task.args = [fn, out_fn, conf]
logger.debug('added task to generate step: {0}'.format(out_fn))
def json_output_tasks(conf, app):
regexes = [
(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''),
(re.compile(r'“'), '"'),
(re.compile(r'”'), '"'),
(re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"),
(re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''),
(re.compile(r'>'), '>'),
(re.compile(r'<'), '<')
]
outputs = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
path = os.path.join(conf.paths.branch_output,
'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = dot_concat(path, 'fjson')
json = dot_concat(path, 'json')
if conf.project.name == 'mms':
if not os.path.exists(fjson):
continue
task = app.add('task')
task.target = json
task.dependency = fjson
task.job = process_json_file
task.description = "processing json file".format(json)
task.args = [fjson, json, regexes, conf]
outputs.append(json)
list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
list_task = app.add('task')
list_task.target = list_file
list_task.job = generate_list_file
list_task.args = [outputs, list_file, conf]
output = app.add('app')
out_task = output.add('task')
out_task.job = json_output
out_task.args = [conf]
out_task.description = 'transfer json output to public directory'
def gettext_tasks(conf, app):
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
builder_name)
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
task = app.add('task')
task.target = fn
task.job = copy_if_needed
task.args = [ fn, os.path.join(locale_dirs, fn[path_offset:]), None]
task.description = "migrating po file {0} if needed".format(fn)
def get_file_list(path, input_extension):
''' This function wraps around expand tree to return a list of only 1 file
if the user gives a path to a file and not a directory. Otherwise it has
the same functionality
:param string path: path to the file
:param list input_extension: a list (or a single) of extensions that is acceptable
'''
if os.path.isfile(path):
if input_extension is not None:
if isinstance(input_extension, list):
if os.path.splitext(path)[1][1:] not in input_extension:
return []
else:
if not path.endswith(input_extension):
return []
return [path]
else:
return expand_tree(path, input_extension)
def generated_includes(conf):
toc_spec_files = []
step_files = []
for fn in expand_tree(os.path.join(conf.paths.includes), input_extension='yaml'):
base = os.path.basename(fn)
if base.startswith('toc-spec'):
toc_spec_files.append(fn)
elif base.startswith('ref-spec'):
toc_spec_files.append(fn)
elif base.startswith('steps'):
step_files.append(fn)
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
mapping = {}
for spec_file in toc_spec_files:
if os.path.exists(spec_file):
data = ingest_yaml_doc(spec_file)
else:
continue
deps = [ os.path.join(path_prefix, i ) for i in data['sources']]
mapping[spec_file[maskl:]] = deps
for step_def in step_files:
data = ingest_yaml_list(step_def)
deps = []
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [ os.path.join(path_prefix, i ) for i in deps ]
mapping[step_def[maskl:]] = deps
return mapping
def get_migration_specifications(conf):
return [ fn for fn in expand_tree(os.path.join(conf.paths.projectroot,
conf.paths.builddata))
if 'primer' in fn and 'migrations' in fn ]
def api_sources(conf):
return expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source, 'reference'), 'yaml')
def option_sources(conf):
output_path = get_option_path(conf)
for fn in expand_tree(get_base_path(conf), 'yaml'):
if fn.startswith(output_path):
yield fn
def step_sources(paths):
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), ' yaml'):
if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
yield fn
def get_migration_specifications(conf):
return [ fn for fn in expand_tree(os.path.join(conf.paths.projectroot,
conf.paths.builddata))
if conf.project.name in os.path.basename(fn) and 'migrations' in fn ]