def manpage_jobs(conf=None):
conf = lazy_conf(conf)
options_compat_re = [(re.compile(r'\.\. option:: --'), r'.. setting:: '),
(re.compile(r'setting:: (\w+) .*'), r'setting:: \1'),
(re.compile(r':option:`--'), r':setting:`')]
jobs = [
(os.path.join(conf.paths.includes, "manpage-options-auth.rst"),
os.path.join(conf.paths.includes, 'manpage-options-auth-mongo.rst'),
(re.compile('fact-authentication-source-tool'),
'fact-authentication-source-mongo')),
(os.path.join(conf.paths.includes, 'manpage-options-ssl.rst'),
os.path.join(conf.paths.includes,
'manpage-options-ssl-settings.rst'), options_compat_re),
(os.path.join(conf.paths.includes, 'manpage-options-audit.rst'),
os.path.join(conf.paths.includes,
'manpage-options-audit-settings.rst'), options_compat_re)
]
for input_fn, output_fn, regex in jobs:
if os.path.exists(input_fn):
yield {
'target':
output_fn,
'dependency':
input_fn,
'job':
process_page,
'args': [input_fn, output_fn, regex, 'manpage'],
'description':
"generating manpage {0} from {1}".format(output_fn, input_fn)
}
def main():
ui = user_input()
conf = lazy_conf()
lines = []
for doc in ingest_yaml(ui.data):
if doc['type'] == 'redirect':
lines.append(
generate_redirects(process_redirect(doc, conf=conf),
match=ui.match,
conf=conf))
if doc['type'] == 'draft':
print(
generate_redirects(process_redirect(doc, conf=conf),
match=ui.match,
conf=conf))
if lines:
dirname = os.path.dirname(ui.filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(ui.filename, 'w') as f:
for line in lines:
f.write(line)
print('[redirect]: regenerated ' + ui.filename + ' file.')
def generate_redirects(redirect, match=False, conf=None):
conf = lazy_conf(conf)
multi = determine_is_multi(redirect['outputs'])
if 'external' in redirect:
o = ''
for output in redirect['outputs']:
o += generate_external_rule(redirect, output, conf)
o += '\n'
elif multi and match is True:
_base = ''
for path in redirect['outputs']:
_base += path + '|'
base = _base[:-1]
o = generate_match_rule(redirect, base, conf)
o += '\n'
elif multi is True and match is False:
o = ''
for output in redirect['outputs']:
o += generate_simple_rule(redirect, output, conf)
o += '\n'
elif multi is False:
o = generate_simple_rule(redirect, conf=conf)
o += '\n'
o = o.replace('http:/', 'http://')
o = o.replace('//', '/')
return o
def generate_match_rule(redir, base, conf=None):
conf = lazy_conf(conf)
o = 'RedirectMatch {0} /({1}){2} {3}/$1{4}'
return o.format(redir['code'], base, redir['from'], conf.project.url,
redir['to'])
def process_redirect(redirect, conf=None):
conf = lazy_conf(conf)
if 'all' in redirect['outputs']:
redirect['outputs'].remove('all')
for branch in conf.git.branches.published:
redirect['outputs'].append(branch)
for output in redirect['outputs']:
if isinstance(output, dict):
continue
elif output.startswith('after-'):
idx = conf.git.branches.published.index(output.split('-', 1)[1])
redirect['outputs'].remove(output)
redirect['outputs'].extend(conf.git.branches.published[:idx])
elif output.startswith('before-'):
idx = conf.git.branches.published.index(output.split('-', 1)[1])
redirect['outputs'].remove(output)
redirect['outputs'].extend(conf.git.branches.published[idx:])
if redirect['code'] in [301, 302, 303]:
redirect['code'] = str(redirect['code'])
else:
raise Exception(
str(redirect['code']) + ' is not a supported redirect code')
return redirect
def generate_match_rule(redir, base, conf=None):
conf = lazy_conf(conf)
o = 'RedirectMatch {0} /({1}){2} {3}/$1{4}'
return o.format(redir['code'], base, redir['redirect-path'],
conf.project.url, redir['url-base'])
def include_files(files=None, conf=None):
if files is not None:
return files
else:
conf = lazy_conf(conf)
source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
grep = command(
'grep -R ".. include:: /" {0} || exit 0'.format(source_dir),
capture=True).out
rx = re.compile(source_dir + r'(.*):.*\.\. include:: (.*)')
s = [
m.groups() for m in [rx.match(d) for d in grep.split('\n')]
if m is not None
]
def tuple_sort(k):
return k[1]
s.sort(key=tuple_sort)
files = dict()
for i in groupby(s, itemgetter(1)):
files[i[0]] = set()
for src in i[1]:
if not src[0].endswith('~'):
files[i[0]].add(src[0])
files[i[0]] = list(files[i[0]])
files.update(generated_includes(conf))
return files
def process_redirect(redirect, conf=None):
conf = lazy_conf(conf)
if 'all' in redirect['outputs']:
redirect['outputs'].remove('all')
for branch in conf.git.branches.published:
redirect['outputs'].append(branch)
for output in redirect['outputs']:
if isinstance(output, dict):
continue
elif output.startswith('after-'):
idx = conf.git.branches.published.index(output.split('-', 1)[1])
redirect['outputs'].remove(output)
redirect['outputs'].extend(conf.git.branches.published[:idx])
elif output.startswith('before-'):
idx = conf.git.branches.published.index(output.split('-', 1)[1])
redirect['outputs'].remove(output)
redirect['outputs'].extend(conf.git.branches.published[idx:])
if redirect['code'] in [ 301, 302, 303 ]:
redirect['code'] = str(redirect['code'])
else:
raise Exception(str(redirect['code']) + ' is not a supported redirect code')
return redirect
def generate_redirects(redirect, match=False, conf=None):
conf = lazy_conf(conf)
multi = determine_is_multi(redirect['outputs'])
if 'external' in redirect:
o = ''
for output in redirect['outputs']:
o += generate_external_rule(redirect, output, conf)
o += '\n'
elif multi and match is True:
_base = ''
for path in redirect['outputs']:
_base += path + '|'
base = _base[:-1]
o = generate_match_rule(redirect, base, conf)
o += '\n'
elif multi is True and match is False:
o = ''
for output in redirect['outputs']:
o += generate_simple_rule(redirect, output, conf)
o += '\n'
elif multi is False:
o = generate_simple_rule(redirect, conf=conf)
o += '\n'
return o
def dump_file_hashes(conf=None):
conf = lazy_conf(conf)
output = conf.system.dependency_cache
o = { 'conf': conf,
'time': datetime.datetime.utcnow().strftime("%s"),
'files': { }
}
files = expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
output_dir = os.path.dirname(output)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output, 'w') as f:
json.dump(o, f)
logger.info('wrote dependency cache to: {0}'.format(output))
def manpage_jobs(conf=None):
conf = lazy_conf(conf)
options_compat_re = [ (re.compile(r'\.\. option:: --'), r'.. setting:: ' ),
(re.compile(r'setting:: (\w+) .*'), r'setting:: \1'),
(re.compile(r':option:`--'), r':setting:`') ]
jobs = [
(
os.path.join(conf.paths.includes, "manpage-options-auth.rst"),
os.path.join(conf.paths.includes, 'manpage-options-auth-mongo.rst'),
( re.compile('fact-authentication-source-tool'),
'fact-authentication-source-mongo' )
),
(
os.path.join(conf.paths.includes, 'manpage-options-ssl.rst'),
os.path.join(conf.paths.includes, 'manpage-options-ssl-settings.rst'),
options_compat_re
),
(
os.path.join(conf.paths.includes, 'manpage-options-audit.rst'),
os.path.join(conf.paths.includes, 'manpage-options-audit-settings.rst'),
options_compat_re
)
]
for input_fn, output_fn, regex in jobs:
if os.path.exists(input_fn):
yield {
'target': output_fn,
'dependency': input_fn,
'job': process_page,
'args': [ input_fn, output_fn, regex, 'manpage' ],
}
def include_files(files=None, conf=None):
if files is not None:
return files
else:
conf = lazy_conf(conf)
source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
grep = command('grep -R ".. include:: /" {0} || exit 0'.format(source_dir), capture=True).out
rx = re.compile(source_dir + r'(.*):.*\.\. include:: (.*)')
s = [ m.groups()
for m in [ rx.match(d)
for d in grep.split('\n') ]
if m is not None
]
def tuple_sort(k):
return k[1]
s.sort(key=tuple_sort)
files = dict()
for i in groupby(s, itemgetter(1) ):
files[i[0]] = set()
for src in i[1]:
if not src[0].endswith('~'):
files[i[0]].add(src[0])
files[i[0]] = list(files[i[0]])
files.update(generated_includes(conf))
return files
def dump_file_hashes(conf=None):
conf = lazy_conf(conf)
output = conf.system.dependency_cache
o = {
'conf': conf,
'time': datetime.datetime.utcnow().strftime("%s"),
'files': {}
}
files = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
output_dir = os.path.dirname(output)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output, 'w') as f:
json.dump(o, f)
logger.info('wrote dependency cache to: {0}'.format(output))
def refresh_dependencies(conf=None):
conf = lazy_conf(conf)
results = runner(refresh_dependency_jobs(conf),
pool=4,
parallel='process',
force=False)
return sum(results)
def post_process_jobs(source_fn=None, tasks=None, conf=None):
"""
input documents should be:
{
'transform': {
'regex': str,
'replace': str
}
'type': <str>
'file': <str|list>
}
``transform`` can be either a document or a list of documents.
"""
if tasks is None:
conf = lazy_conf(conf)
if source_fn is None:
source_fn = os.path.join(conf.paths.project.root,
conf.paths.builddata, 'processing.yaml')
tasks = ingest_yaml(source_fn)
elif not isinstance(tasks, collections.Iterable):
raise ProcessingError(
'[ERROR]: cannot parse post processing specification.')
def rjob(fn, regex, type):
return {
'target': fn,
'dependency': None,
'job': process_page,
'args': dict(fn=fn, output_fn=fn, regex=regex, builder=type)
}
for job in tasks:
if not isinstance(job, dict):
raise ProcessingError(
'[ERROR]: invalid replacement specification.')
elif not 'file' in job and not 'transform' in job:
raise ProcessingError(
'[ERROR]: replacement specification incomplete.')
if 'type' not in job:
job['type'] = 'processor'
if isinstance(job['transform'], list):
regex = [(re.compile(rs['regex']), rs['replace'])
for rs in job['transform']]
else:
regex = (re.compile(job['transform']['regex']),
job['transform']['replace'])
if isinstance(job['file'], list):
for fn in job['file']:
yield rjob(fn, regex, job['type'])
else:
yield rjob(job['file'], regex, job['type'])
def release_jobs(conf=None):
conf = lazy_conf(conf)
data_file = os.path.join(conf.paths.builddata, 'releases') + '.yaml'
if 'release' in conf.version:
release_version = conf.version.release
else:
release_version = conf.version.published[0]
if not os.path.exists(data_file):
return
rel_data = ingest_yaml(os.path.join(conf.paths.builddata, 'releases') + '.yaml')
deps = [ os.path.join(conf.paths.projectroot, conf.system.conf_file) ]
for rel in rel_data['source-files']:
target = os.path.join(conf.paths.projectroot,
conf.paths.includes,
'install-curl-release-{0}.rst'.format(rel))
yield {
'target': target,
'dependency': deps,
'job': _generate_release_core,
'args': [ rel, target, release_version ]
}
target = os.path.join(conf.paths.projectroot,
conf.paths.includes,
'install-untar-release-{0}.rst'.format(rel))
yield {
'target': target,
'dependency': deps,
'job': _generate_untar_core,
'args': [ rel, target, release_version ]
}
target = os.path.join(conf.paths.projectroot,
conf.paths.includes,
'install-copy-release-{0}.rst'.format(rel))
yield {
'target': target,
'dependency': deps,
'job': _generate_copy_core,
'args': [ rel, target, release_version ]
}
for rel in rel_data['subscription-build']:
target = 'source/includes/install-curl-release-ent-{0}.rst'.format(rel['system'])
yield {
'target': target,
'dependency': deps,
'job': _generate_release_ent,
'args': [ rel, target, release_version ]
}
def image_jobs(conf=None):
conf = lazy_conf(None)
paths = conf.paths
meta_file = os.path.join(paths.images, 'metadata') + '.yaml'
if not os.path.exists(meta_file):
raise StopIteration
images_meta = ingest_yaml_list(meta_file)
if images_meta is None:
raise StopIteration
for image in images_meta:
image['dir'] = paths.images
source_base = os.path.join(image['dir'], image['name'])
source_file = dot_concat(source_base, 'svg')
rst_file = dot_concat(source_base, 'rst')
image['conf'] = conf
yield {
'target':
rst_file,
'dependency': [
meta_file,
os.path.join(paths.buildsystem, 'utils', 'rstcloth',
'images.py')
],
'job':
generate_image_pages,
'args':
image
}
for output in image['output']:
if 'tag' in output:
tag = '-' + output['tag']
else:
tag = ''
target_img = source_base + tag + '.png'
inkscape_cmd = '{cmd} -z -d {dpi} -w {width} -y 0.0 -e >/dev/null {target} {source}'
yield {
'target':
target_img,
'dependency': [source_file, meta_file],
'job':
_generate_images,
'args': [
inkscape_cmd, output['dpi'], output['width'], target_img,
source_file
],
}
def generate_external_rule(redir, base=None, conf=None):
conf = lazy_conf(conf)
if base is None:
base = redir['outputs'][0]
o = 'Redirect {0} /{1}{2} {3}/{4}{5}'
return o.format(redir['code'], base, redir['redirect-path'],
conf.project.url, redir['external'], redir['url-base'])
def __init__(self, steps, conf=None):
if not isinstance(steps, Steps):
raise TypeError
else:
self.steps = steps
self.conf = lazy_conf(conf)
self.current_step = 1
self.rst = RstCloth()
self.hook()
def generate_external_rule(redir, base=None, conf=None):
conf = lazy_conf(conf)
if base is None:
base = redir['outputs'][0]
o = 'Redirect {0} /{1}{2} {3}/{4}{5}'
return o.format(redir['code'], base, redir['redirect-path'],
conf.project.url, redir['external'], redir['url-base'])
def __init__(self, steps, conf=None):
if not isinstance(steps, Steps):
raise TypeError
else:
self.steps = steps
self.conf = lazy_conf(conf)
self.current_step = 1
self.rst = RstCloth()
self.hook()
def post_process_jobs(source_fn=None, tasks=None, conf=None):
"""
input documents should be:
{
'transform': {
'regex': str,
'replace': str
}
'type': <str>
'file': <str|list>
}
``transform`` can be either a document or a list of documents.
"""
if tasks is None:
conf = lazy_conf(conf)
if source_fn is None:
source_fn = os.path.join(conf.paths.project.root,
conf.paths.builddata,
'processing.yaml')
tasks = ingest_yaml(source_fn)
elif not isinstance(tasks, collections.Iterable):
raise ProcessingError('[ERROR]: cannot parse post processing specification.')
def rjob(fn, regex, type):
return {
'target': fn,
'dependency': None,
'job': process_page,
'args': dict(fn=fn, output_fn=fn, regex=regex, builder=type)
}
for job in tasks:
if not isinstance(job, dict):
raise ProcessingError('[ERROR]: invalid replacement specification.')
elif not 'file' in job and not 'transform' in job:
raise ProcessingError('[ERROR]: replacement specification incomplete.')
if 'type' not in job:
job['type'] = 'processor'
if isinstance(job['transform'], list):
regex = [ (re.compile(rs['regex']), rs['replace'])
for rs in job['transform'] ]
else:
regex = (re.compile(job['transform']['regex']), job['transform']['replace'])
if isinstance(job['file'], list):
for fn in job['file']:
yield rjob(fn, regex, job['type'])
else:
yield rjob(job['file'], regex, job['type'])
def process_redirect(redirect, conf=None):
"""Given a redirection document, returns a properly formatted string for an
Apache htaccess redirect statement."""
conf = lazy_conf(conf)
redirect = redirect_schema_migration(redirect)
if 'all' in redirect['outputs']:
redirect['outputs'].remove('all')
redirect['outputs'].extend(conf.git.branches.published)
for output in redirect['outputs']:
if isinstance(output, dict):
source, target = output.items()[0]
if isinstance(target, dict):
left, right = target.items()[0]
if source.startswith('after-'):
redirect['outputs'].remove(output)
idx = conf.git.branches.published.index(source.split('-', 1)[1])
for out in conf.git.branches.published[:idx]:
redirect['outputs'].append({ slash_concat(left, out): slash_concat(right,out) })
elif source.startswith('before-'):
redirect['outputs'].remove(output)
idx = conf.git.branches.published.index(source.split('-', 1)[1])
for out in conf.git.branches.published[idx:]:
redirect['outputs'].append({ slash_concat(left, out): slash_concat(right,out) })
else:
logger.error("{0} is invalid source for redirect: {1}".format(source, redirect))
else:
continue
elif output.startswith('after-'):
idx = conf.git.branches.published.index(output.split('-', 1)[1])
redirect['outputs'].remove(output)
redirect['outputs'].extend(conf.git.branches.published[:idx])
elif output.startswith('before-'):
idx = conf.git.branches.published.index(output.split('-', 1)[1])
redirect['outputs'].remove(output)
redirect['outputs'].extend(conf.git.branches.published[idx:])
if redirect['code'] in [ 301, 302, 303 ]:
redirect['code'] = str(redirect['code'])
else:
msg = str(redirect['code']) + ' is not a supported redirect code'
logger.critical(msg)
raise Exception(msg)
return redirect
def api_jobs(conf=None):
conf = lazy_conf(conf)
for source in expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source, 'reference'), 'yaml'):
target = dot_concat(os.path.splitext(source)[0], 'rst')
yield {
'target': target,
'dependency': source,
'job': _generate_api_param,
'args': [source, target, conf]
}
def post_process_jobs(source_fn=None, tasks=None, conf=None):
"""
input documents should be:
{
'transform': {
'regex': str,
'replace': str
}
'type': <str>
'file': <str|list>
}
``transform`` can be either a document or a list of documents.
"""
if tasks is None:
conf = lazy_conf(conf)
if source_fn is None:
source_fn = os.path.join(conf.paths.project.root, conf.paths.builddata, "processing.yaml")
tasks = ingest_yaml(source_fn)
elif not isinstance(tasks, collections.Iterable):
raise ProcessingError("[ERROR]: cannot parse post processing specification.")
def rjob(fn, regex, type):
return {
"target": fn,
"dependency": None,
"job": process_page,
"args": dict(fn=fn, output_fn=fn, regex=regex, builder=type),
}
for job in tasks:
if not isinstance(job, dict):
raise ProcessingError("[ERROR]: invalid replacement specification.")
elif not "file" in job and not "transform" in job:
raise ProcessingError("[ERROR]: replacement specification incomplete.")
if "type" not in job:
job["type"] = "processor"
if isinstance(job["transform"], list):
regex = [(re.compile(rs["regex"]), rs["replace"]) for rs in job["transform"]]
else:
regex = (re.compile(job["transform"]["regex"]), job["transform"]["replace"])
if isinstance(job["file"], list):
for fn in job["file"]:
yield rjob(fn, regex, job["type"])
else:
yield rjob(job["file"], regex, job["type"])
def image_jobs(conf=None):
conf = lazy_conf(None)
paths = conf.paths
meta_file = os.path.join(paths.images, 'metadata') + '.yaml'
if not os.path.exists(meta_file):
raise StopIteration
images_meta = ingest_yaml_list(meta_file)
if images_meta is None:
raise StopIteration
for image in images_meta:
image['dir'] = paths.images
source_base = os.path.join(image['dir'], image['name'])
source_file = dot_concat(source_base, 'svg')
rst_file = dot_concat(source_base, 'rst')
image['conf'] = conf
yield {
'target': rst_file,
'dependency': [ meta_file, os.path.join(paths.buildsystem, 'utils', 'rstcloth', 'images.py') ],
'job': generate_image_pages,
'args': image,
'description': "generating rst include file {0} for {1}".format(rst_file, source_file)
}
for output in image['output']:
if 'tag' in output:
tag = '-' + output['tag']
else:
tag = ''
target_img = source_base + tag + '.png'
inkscape_cmd = '{cmd} -z -d {dpi} -w {width} -y 0.0 -e >/dev/null {target} {source}'
yield {
'target': target_img,
'dependency': [ source_file, meta_file ],
'job': _generate_images,
'args': [
inkscape_cmd,
output['dpi'],
output['width'],
target_img,
source_file
],
'description': 'generating image file {0} from {1}'.format(target_img, source_file)
}
def table_jobs(conf=None):
conf = lazy_conf(conf)
for source in expand_tree(os.path.join(conf.paths.projectroot, conf.paths.includes), 'yaml'):
if os.path.basename(source).startswith('table'):
target = _get_table_output_name(source)
list_target = _get_list_table_output_name(source)
yield {
'target': [ target, list_target ],
'dependency': source,
'job': _generate_tables,
'args': [ source, target, list_target ]
}
def include_files_unused(inc_files=None, conf=None):
conf = lazy_conf(conf)
inc_files = [ fn[6:] for fn in expand_tree(os.path.join(conf.paths.includes), None) ]
mapping = include_files(conf)
results = []
for fn in inc_files:
if fn.endswith('yaml') or fn.endswith('~'):
continue
if fn not in mapping.keys():
results.append(fn)
return results
def generate_external_rule(redir, base=None, conf=None):
conf = lazy_conf(conf)
if base is None:
base = redir['outputs'][0]
if redir['external'].startswith('http'):
o = 'Redirect {0} /{1}{2} {3}{4}'
return o.format(redir['code'], base, redir['from'],
redir['external'], redir['to'])
else:
o = 'Redirect {0} /{1}{2} {3}{4}{5}'
return o.format(redir['code'], base, redir['from'],
conf.project.url, redir['external'], redir['to'])
def __init__(self, filename, conf=None, sort=False):
self.spec = self._process_spec(filename, sort)
self.conf = lazy_conf(conf)
if "ref-toc" in filename:
self._is_ref = True
else:
self._is_ref = False
self.table = None
self.contents = None
self.dfn = None
self.final = False
def generate_external_rule(redir, base=None, conf=None):
conf = lazy_conf(conf)
if base is None:
base = redir['outputs'][0]
if redir['external'].startswith('http'):
o = 'Redirect {0} /{1}{2} {3}{4}'
return o.format(redir['code'], base, redir['from'], redir['external'],
redir['to'])
else:
o = 'Redirect {0} /{1}{2} {3}{4}{5}'
return o.format(redir['code'], base, redir['from'], conf.project.url,
redir['external'], redir['to'])
def __init__(self, filename, conf=None, sort=False):
self.spec = self._process_spec(filename, sort)
self.conf = lazy_conf(conf)
if "ref-toc" in filename:
self._is_ref = True
else:
self._is_ref = False
self.table = None
self.contents = None
self.dfn = None
self.final = False
def generate_list_file(outputs, path, conf=None):
dirname = os.path.dirname(path)
conf = lazy_conf(conf)
url = '/'.join([ conf.project.url, conf.project.basepath, 'json' ])
if not os.path.exists(dirname):
os.mkdir(dirname)
with open(path, 'w') as f:
for fn in outputs:
f.write( '/'.join([ url, fn.split('/', 3)[3:][0]]) )
f.write('\n')
logger.info('rebuilt inventory of json output.')
def generate_list_file(outputs, path, conf=None):
dirname = os.path.dirname(path)
conf = lazy_conf(conf)
url = '/'.join([conf.project.url, conf.project.basepath, 'json'])
if not os.path.exists(dirname):
os.mkdir(dirname)
with open(path, 'w') as f:
for fn in outputs:
f.write('/'.join([url, fn.split('/', 3)[3:][0]]))
f.write('\n')
print('[json]: rebuilt inventory of json output.')
def include_files_unused(inc_files=None, conf=None):
conf = lazy_conf(conf)
inc_files = [
fn[6:] for fn in expand_tree(os.path.join(conf.paths.includes), None)
]
mapping = include_files(conf)
results = []
for fn in inc_files:
if fn.endswith('yaml') or fn.endswith('~'):
continue
if fn not in mapping.keys():
results.append(fn)
return results
def gettext_jobs(conf=None):
conf = lazy_conf(conf)
locale_dirs = os.path.join(conf.paths.projectroot, conf.paths.locale,
'pot')
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, 'gettext')
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
yield {
'job': copy_if_needed,
'args': [fn, os.path.join(locale_dirs, fn[path_offset:]), None],
'description': "migrating po file {0} if needed".format(fn)
}
def gettext_jobs(conf=None):
conf = lazy_conf(conf)
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
'gettext')
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
yield {
'job': copy_if_needed,
'args': [ fn, os.path.join(locale_dirs, fn[path_offset:]), None]
}
def generate_simple_rule(redir, base=None, conf=None):
conf = lazy_conf(conf)
if base is None:
base = redir['outputs'][0]
if isinstance(base, dict):
left, right = base.items()[0]
o = 'Redirect {0} /{1}{2} {3}/{4}{5}'
return o.format(redir['code'], left, redir['from'], conf.project.url,
right, redir['to'])
else:
o = 'Redirect {0} /{1}{2} {3}/{1}{4}'
return o.format(redir['code'], base, redir['from'], conf.project.url,
redir['to'])
def json_output(conf=None):
conf = lazy_conf(conf)
list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
public_list_file = os.path.join(conf.paths.public_site_output,
'json', '.file_list')
cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'
json_dst = os.path.join(conf.paths.public_site_output, 'json')
if not os.path.exists(json_dst):
os.makedirs(json_dst)
command(cmd.format(src=os.path.join(conf.paths.branch_output, 'json') + '/',
dst=json_dst))
copy_if_needed(list_file, public_list_file)
print('[json]: deployed json files to local staging.')
def gettext_jobs(conf=None):
conf = lazy_conf(conf)
locale_dirs = os.path.join(conf.paths.projectroot,
conf.paths.locale, 'pot')
branch_output = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
'gettext')
path_offset = len(branch_output) + 1
for fn in expand_tree(branch_output, None):
yield {
'job': copy_if_needed,
'args': [ fn, os.path.join(locale_dirs, fn[path_offset:]), None],
'description': "migrating po file {0} if needed".format(fn)
}
def generate_simple_rule(redir, base=None, conf=None):
conf = lazy_conf(conf)
if base is None:
base = redir['outputs'][0]
if isinstance(base, dict):
left, right = base.items()[0]
o = 'Redirect {0} /{1}{2} {3}/{4}{5}'
return o.format(redir['code'], left, redir['redirect-path'],
conf.project.url, right, redir['url-base'])
else:
o = 'Redirect {0} /{1}{2} {3}/{1}{4}'
return o.format(redir['code'], base, redir['redirect-path'],
conf.project.url, redir['url-base'])
def json_output_jobs(conf=None):
conf = lazy_conf(conf)
regexes = [
(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''),
(re.compile(r'“'), '"'),
(re.compile(r'”'), '"'),
(re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"),
(re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''),
(re.compile(r'>'), '>'),
(re.compile(r'<'), '<')
]
outputs = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
path = os.path.join(conf.paths.branch_output,
'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = dot_concat(path, 'fjson')
json = dot_concat(path, 'json')
if conf.project.name == 'mms':
if not os.path.exists(fjson):
continue
yield { 'target': json,
'dependency': fjson,
'job': process_json_file,
'description': "processing json file".format(json),
'args': (fjson, json, regexes, conf) }
outputs.append(json)
list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
yield { 'target': list_file,
'dependency': None,
'description': 'generating json index list {0}'.format(list_file),
'job': generate_list_file,
'args': (outputs, list_file, conf) }
def json_output(conf=None):
conf = lazy_conf(conf)
list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
public_list_file = os.path.join(conf.paths.public_site_output, 'json',
'.file_list')
cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'
json_dst = os.path.join(conf.paths.public_site_output, 'json')
if not os.path.exists(json_dst):
os.makedirs(json_dst)
command(
cmd.format(src=os.path.join(conf.paths.branch_output, 'json') + '/',
dst=json_dst))
copy_if_needed(list_file, public_list_file)
print('[json]: deployed json files to local staging.')
def error_pages(conf=None):
conf = lazy_conf(conf)
error_conf = os.path.join(conf.paths.builddata, 'errors.yaml')
if not os.path.exists(error_conf):
return None
else:
error_pages = ingest_yaml_list(error_conf)
sub = (re.compile(r'\.\./\.\./'), conf.project.url + r'/' + conf.project.tag + r'/')
for error in error_pages:
page = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, 'dirhtml',
'meta', error, 'index.html')
munge_page(fn=page, regex=sub, tag='error-pages')
print('[error-pages]: rendered {0} error pages'.format(len(error_pages)))
def json_output_jobs(conf=None):
conf = lazy_conf(conf)
regexes = [
(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''),
(re.compile(r'“'), '"'),
(re.compile(r'”'), '"'),
(re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"),
(re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''),
(re.compile(r'>'), '>'),
(re.compile(r'<'), '<')
]
outputs = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
path = os.path.join(conf.paths.branch_output,
'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = dot_concat(path, 'fjson')
json = dot_concat(path, 'json')
if conf.project.name == 'mms':
if not os.path.exists(fjson):
continue
yield dict(target=json,
dependency=fjson,
job=process_json_file,
args=(fjson, json, regexes, conf))
outputs.append(json)
list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
yield dict(target=list_file,
dependency=None,
job=generate_list_file,
args=(outputs, list_file, conf))
def output_sphinx_stream(out, conf=None):
if conf is None:
conf = lazy_conf(conf)
out = [o for o in out.split('\n') if o != '']
full_path = os.path.join(conf.paths.projectroot, conf.paths.branch_output)
regx = re.compile(
r'(.*):[0-9]+: WARNING: duplicate object description of ".*", other instance in (.*)'
)
printable = []
for idx, l in enumerate(out):
if is_msg_worthy(l) is not True:
printable.append(None)
continue
f1 = regx.match(l)
if f1 is not None:
g = f1.groups()
if g[1].endswith(g[0]):
printable.append(None)
continue
l = path_normalization(l, full_path, conf)
if l.startswith('InputError: [Errno 2] No such file or directory'):
l = path_normalization(
l.split(' ')[-1].strip()[1:-2], full_path, conf)
printable[idx - 1] += ' ' + l
l = None
printable.append(l)
printable = list(set(printable))
printable.sort()
print_build_messages(printable)
def main():
ui = user_input()
conf = lazy_conf()
lines = []
for doc in ingest_yaml(ui.data):
if doc['type'] == 'redirect':
lines.append(generate_redirects(process_redirect(doc, conf=conf), match=ui.match, conf=conf))
if doc['type'] == 'redirect-draft':
print(generate_redirects(process_redirect(doc, conf=conf), match=ui.match, conf=conf))
if lines:
dirname = os.path.dirname(ui.filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(ui.filename, 'w') as f:
for line in lines:
f.write(line)
print('[redirect]: regenerated ' + ui.filename + ' file.' )
def makefile_meta():
# because this is typically called by bootstrap.py, projectroot is the
# doctools directory.
conf = lazy_conf(None)
# re/generate the makefile.meta
script_path = os.path.join(conf.paths.projectroot, conf.paths.buildsystem,
'makecloth', 'meta.py')
makefn_path = os.path.join(conf.paths.projectroot, conf.paths.output,
'makefile.meta')
cmd = 'python {0} {1}'.format(script_path, makefn_path).split()
try:
subprocess.check_call(cmd)
except:
subprocess.check_call('python {0} {1}'.format(
os.path.join('build', 'docs-tools', 'makecloth', 'meta.py'),
os.path.join('build', 'makefile.meta')))
finally:
return
def generated_includes(conf=None):
conf = lazy_conf(conf)
toc_spec_files = []
step_files = []
for fn in expand_tree(os.path.join(conf.paths.includes),
input_extension='yaml'):
base = os.path.basename(fn)
if base.startswith('toc-spec'):
toc_spec_files.append(fn)
elif base.startswith('ref-spec'):
toc_spec_files.append(fn)
elif base.startswith('steps'):
step_files.append(fn)
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
mapping = {}
for spec_file in toc_spec_files:
data = ingest_yaml_doc(spec_file)
deps = [os.path.join(path_prefix, i) for i in data['sources']]
mapping[spec_file[maskl:]] = deps
for step_def in step_files:
data = ingest_yaml_list(step_def)
deps = []
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [os.path.join(path_prefix, i) for i in deps]
mapping[step_def[maskl:]] = deps
return mapping
def generated_includes(conf=None):
conf = lazy_conf(conf)
toc_spec_files = []
step_files = []
for fn in expand_tree(os.path.join(conf.paths.includes), input_extension='yaml'):
base = os.path.basename(fn)
if base.startswith('toc-spec'):
toc_spec_files.append(fn)
elif base.startswith('ref-spec'):
toc_spec_files.append(fn)
elif base.startswith('steps'):
step_files.append(fn)
maskl = len(conf.paths.source)
path_prefix = conf.paths.includes[len(conf.paths.source):]
mapping = {}
for spec_file in toc_spec_files:
data = ingest_yaml_doc(spec_file)
deps = [ os.path.join(path_prefix, i ) for i in data['sources']]
mapping[spec_file[maskl:]] = deps
for step_def in step_files:
data = ingest_yaml_list(step_def)
deps = []
for step in data:
if 'source' in step:
deps.append(step['source']['file'])
if len(deps) != 0:
deps = [ os.path.join(path_prefix, i ) for i in deps ]
mapping[step_def[maskl:]] = deps
return mapping
def external_jobs(conf=None):
conf = lazy_conf(conf)
ext_mod_path = os.path.join(conf.paths.projectroot, 'local')
if not os.path.exists(ext_mod_path):
raise StopIteration
external_mods = []
for mod in expand_tree(ext_mod_path, 'py'):
path, name = os.path.split(mod)
name, _ = os.path.splitext(name)
file, filename, data = imp.find_module(name, [path])
imp.load_module(name, file, mod, data)
external_mods.append(name)
for name in external_mods:
mod = sys.modules[name]
if 'jobs' in dir(mod) and 'stage' in dir(mod) and mod.stage.startswith('pre'):
for task in mod.jobs(conf):
yield task
def changed_includes(conf=None):
from pygit2 import Repository, GIT_STATUS_CURRENT, GIT_STATUS_IGNORED
conf = lazy_conf(conf)
repo_path = conf.paths.projectroot
r = Repository(repo_path)
changed = []
for path, flag in r.status().items():
if flag not in [ GIT_STATUS_CURRENT, GIT_STATUS_IGNORED ]:
if path.startswith('source/'):
if path.endswith('.txt'):
changed.append(path[6:])
source_path = os.path.join(conf.paths.source, conf.paths.output, conf.git.branches.current, 'json')
changed_report = []
for report in _generate_report(None):
if report['source'][len(source_path):] in changed:
changed_report.append(report)
return changed_report
def bootstrap(**kwargs):
"""
The bootstrap file calls this function. Use this as a site for future
extension.
"""
try:
build_tools_path = kwargs['build_tools_path']
conf_path = kwargs['conf_path']
symlink(name=os.path.join(build_tools_path, 'bin', 'utils'),
target=os.path.join(os.path.abspath(build_tools_path),
'utils'))
import utils.bootstrap
utils.bootstrap.fabric(build_tools_path, conf_path)
utils.bootstrap.config(build_tools_path, conf_path)
utils.bootstrap.utils(build_tools_path, conf_path)
except KeyError:
print(
'[bootstrap] [warning]: your bootstrap.py is probably out of date. '
'Please update as soon as possible.')
import utils.bootstrap
from utils.config import lazy_conf
conf = lazy_conf(None)
utils.bootstrap.pin_tools(conf)
utils.bootstrap.makefile_meta(conf)
if 'primer' in os.path.split(os.getcwd()):
utils.bootstrap.primer()
print('[bootstrap]: initialized fabfiles and dependencies.')
def changed_includes(conf=None):
from pygit2 import Repository, GIT_STATUS_CURRENT, GIT_STATUS_IGNORED
conf = lazy_conf(conf)
repo_path = conf.paths.projectroot
r = Repository(repo_path)
changed = []
for path, flag in r.status().items():
if flag not in [GIT_STATUS_CURRENT, GIT_STATUS_IGNORED]:
if path.startswith('source/'):
if path.endswith('.txt'):
changed.append(path[6:])
source_path = os.path.join(conf.paths.source, conf.paths.output,
conf.git.branches.current, 'json')
changed_report = []
for report in _generate_report(None):
if report['source'][len(source_path):] in changed:
changed_report.append(report)
return changed_report
def external_jobs(conf=None):
conf = lazy_conf(conf)
ext_mod_path = os.path.join(conf.paths.projectroot, 'local')
if not os.path.exists(ext_mod_path):
raise StopIteration
external_mods = []
for mod in expand_tree(ext_mod_path, 'py'):
path, name = os.path.split(mod)
name, _ = os.path.splitext(name)
file, filename, data = imp.find_module(name, [path])
imp.load_module(name, file, mod, data)
external_mods.append(name)
for name in external_mods:
mod = sys.modules[name]
if 'jobs' in dir(mod) and 'stage' in dir(mod) and mod.stage.startswith(
'pre'):
for task in mod.jobs(conf):
yield task
def json_output_jobs(conf=None):
conf = lazy_conf(conf)
regexes = [(re.compile(r'<a class=\"headerlink\"'), '<a'),
(re.compile(r'<[^>]*>'), ''), (re.compile(r'“'), '"'),
(re.compile(r'”'), '"'), (re.compile(r'‘'), "'"),
(re.compile(r'’'), "'"), (re.compile(r'&#\d{4};'), ''),
(re.compile(r' '), ''), (re.compile(r'>'), '>'),
(re.compile(r'<'), '<')]
outputs = []
for fn in expand_tree('source', 'txt'):
# path = build/<branch>/json/<filename>
path = os.path.join(conf.paths.branch_output, 'json',
os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
fjson = dot_concat(path, 'fjson')
json = dot_concat(path, 'json')
if conf.project.name == 'mms':
if not os.path.exists(fjson):
continue
yield dict(target=json,
dependency=fjson,
job=process_json_file,
args=(fjson, json, regexes, conf))
outputs.append(json)
list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
yield dict(target=list_file,
dependency=None,
job=generate_list_file,
args=(outputs, list_file, conf))
