def generate_meta():
m.section_break('branch/release meta', block='rel')
m.var('manual-branch', MANUAL_BRANCH, block='rel')
m.var('current-branch', str(utils.get_branch()), block='rel')
m.var('last-commit', str(utils.get_commit()), block='rel')
m.var('current-if-not-manual', str(get_manual_path()), block='rel')
paths = render_paths(True)
m.section_break('file system paths', block='paths')
m.var('output', paths['output'], block='paths')
m.var('public-output', paths['public'], block='paths')
m.var('branch-output', paths['branch-output'], block='paths')
m.var('rst-include', paths['includes'], block='paths')
m.var('branch-source', paths['branch-source'], block='paths')
m.var('public-branch-output', paths['branch-staging'], block='paths')
generated_makefiles = []
m.newline()
for target in dynamic_makefiles():
file = '/'.join([paths['output'], "makefile." + target])
cloth = '/'.join([paths['tools'], "makecloth", target + '.py'])
generated_makefiles.append(file)
m.raw(['-include ' + paths['output'] + '/makefile.' + target])
m.target(target=file, dependency=cloth, block='makefiles')
m.job(' '.join(["$(PYTHONBIN)", cloth, file]))
m.newline()
m.newline()
m.target('.PHONY', generated_makefiles)
def generate_meta():
m.section_break('branch/release meta', block='rel')
m.var('manual-branch', MANUAL_BRANCH, block='rel')
m.var('current-branch', str(utils.get_branch()), block='rel')
m.var('last-commit', str(utils.get_commit()), block='rel')
m.var('current-if-not-manual', str(get_manual_path()), block='rel')
paths = render_paths(True)
m.section_break('file system paths', block='paths')
m.var('output', paths['output'], block='paths')
m.var('public-output', paths['public'], block='paths')
m.var('branch-output', paths['branch-output'], block='paths')
m.var('rst-include', paths['includes'], block='paths')
m.var('branch-source', paths['branch-source'], block='paths')
m.var('public-branch-output', paths['branch-staging'], block='paths')
generated_makefiles = []
m.newline()
for target in dynamic_makefiles():
file ='/'.join([paths['output'], "makefile." + target])
cloth = '/'.join([paths['tools'], "makecloth", target + '.py'])
generated_makefiles.append(file)
m.raw(['-include ' + paths['output'] + '/makefile.' + target])
m.target(target=file, dependency=cloth, block='makefiles')
m.job(' '.join(["$(PYTHONBIN)", cloth, file]))
m.newline()
m.newline()
m.target('.PHONY', generated_makefiles)
def toc_jobs():
paths = render_paths('obj')
for fn in expand_tree(paths.includes, 'yaml'):
if fn.startswith(os.path.join(paths.includes, 'table')):
pass
elif len(fn) >= 24:
base_name = _get_toc_base_name(fn)
fmt = fn[20:24]
if fmt != 'spec':
fmt = fn[16:19]
o = {
'dependency': fn,
'job': _generate_toc_tree,
'target': [],
'args': [fn, fmt, base_name, paths]
}
if fmt != 'spec':
o['target'].append(_get_toc_output_name(base_name, 'toc', paths))
is_ref_spec = fn.startswith(os.path.join(os.path.dirname(fn), 'ref-spec'))
if not is_ref_spec and (fmt == 'toc' or fmt == 'spec'):
o['target'].append(_get_toc_output_name(base_name, 'dfn-list', paths))
elif fmt == 'ref' or is_ref_spec:
o['target'].append(_get_toc_output_name(base_name, 'table', paths))
yield o
def steps_jobs():
paths = render_paths('obj')
for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), 'yaml'):
if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
out_fn = _get_steps_output_fn(fn, paths)
yield { 'dependency': fn,
'target': out_fn,
'job': render_step_file,
'args': [fn, out_fn] }
def api_jobs():
paths = render_paths('obj')
for source in expand_tree(os.path.join(paths.source, 'reference'), 'yaml'):
target = dot_concat(os.path.splitext(source)[0], 'rst')
yield {
'target': target,
'dependency': source,
'job': _generate_api_param,
'args': [source, target]
}
def generate_json_output_meta():
m = MakefileCloth()
m.section_break('json output coordination.')
paths = render_paths('dict')
if get_conf().git.remote.upstream.endswith('ecosystem'):
public_json_output = os.path.join(paths['public'], 'json')
else:
public_json_output = os.path.join(paths['branch-staging'], 'json')
build_json_output = os.path.join(paths['branch-output'], 'json')
branch_json_list_file = os.path.join(paths['branch-output'],
'json-file-list')
public_json_list_file = os.path.join(public_json_output, '.file_list')
m.section_break('meta')
m.target('json-output', ['json'])
m.job('fab process.json_output')
rsync_cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {0}/ {1}'
m.job(rsync_cmd.format(build_json_output, public_json_output))
m.msg('[json]: migrated all .json files to staging.')
m.msg('[json]: processed all json files.')
m.section_break('list file')
m.comment(
'the meta build system generates "{0}" when it generates this file'.
format(branch_json_list_file))
fab_cmd = 'fab process.input:{0} process.output:{1} process.copy_if_needed:json'
m.target('json-file-list', public_json_list_file)
m.target(public_json_list_file, 'json-output')
m.job(fab_cmd.format(branch_json_list_file, public_json_list_file))
m.msg('[json]: rebuilt inventory of json output.')
m.target(build_json_output, 'json')
m.target(
'.PHONY',
['clean-json-output', 'clean-json', 'json-output', 'json-file-list'])
m.target('clean-json-output', 'clean-json')
m.job(' '.join([
'rm -rf ', public_json_list_file, branch_json_list_file,
public_json_output
]))
m.msg('[json]: removed all processed json.')
return m
def table_jobs():
paths = render_paths('obj')
for source in expand_tree(paths.includes, 'yaml'):
if os.path.basename(source).startswith('table'):
target = _get_table_output_name(source)
list_target = _get_list_table_output_name(source)
yield {
'target': [ target, list_target ],
'dependency': source,
'job': _generate_tables,
'args': [ source, target, list_target ]
}
def generate_meta():
m.section_break('branch/release meta', block='rel')
m.var('manual-branch', docs_meta.MANUAL_BRANCH, block='rel')
m.var('current-branch', str(docs_meta.get_branch()), block='rel')
m.var('last-commit', str(docs_meta.get_commit()), block='rel')
m.var('current-if-not-manual', str(docs_meta.get_manual_path()), block='rel')
paths = docs_meta.render_paths(True)
m.section_break('file system paths', block='paths')
m.var('output', paths['output'], block='paths')
m.var('public-output', paths['public'], block='paths')
m.var('branch-output', paths['branch-output'], block='paths')
m.var('rst-include', paths['includes'], block='paths')
m.var('branch-source', paths['branch-source'], block='paths')
m.var('public-branch-output', paths['branch-staging'], block='paths')
def generate_meta():
m.section_break("branch/release meta", block="rel")
m.var("manual-branch", MANUAL_BRANCH, block="rel")
m.var("current-branch", str(utils.get_branch()), block="rel")
m.var("last-commit", str(utils.get_commit()), block="rel")
m.var("current-if-not-manual", str(get_manual_path()), block="rel")
paths = render_paths(True)
m.section_break("file system paths", block="paths")
m.var("output", paths["output"], block="paths")
m.var("public-output", paths["public"], block="paths")
m.var("branch-output", paths["branch-output"], block="paths")
m.var("rst-include", paths["includes"], block="paths")
m.var("branch-source", paths["branch-source"], block="paths")
m.var("public-branch-output", paths["branch-staging"], block="paths")
def image_jobs():
paths = render_paths('obj')
meta_file = os.path.join(paths.images, 'metadata') + '.yaml'
if not os.path.exists(meta_file):
return
images_meta = ingest_yaml_list(meta_file)
for image in images_meta:
image['dir'] = paths.images
source_base = os.path.join(image['dir'], image['name'])
source_file = source_base + '.svg'
rst_file = source_base + '.rst'
yield {
'target': rst_file,
'dependency': [ meta_file, os.path.join(paths.buildsystem, 'rstcloth', 'images.py') ],
'job': generate_image_pages,
'args': image
}
for output in image['output']:
if 'tag' in output:
tag = '-' + output['tag']
else:
tag = ''
target_img = source_base + tag + '.png'
inkscape_cmd = '{cmd} -z -d {dpi} -w {width} -y 0.0 -e >/dev/null {target} {source}'
yield {
'target': target_img,
'dependency': source_file,
'job': _generate_images,
'args': [
inkscape_cmd,
output['dpi'],
output['width'],
target_img,
source_file
],
}
def generate_json_output_meta():
m = MakefileCloth()
m.section_break('json output coordination.')
paths = render_paths('dict')
if get_conf().git.remote.upstream.endswith('ecosystem'):
public_json_output = os.path.join(paths['public'], 'json')
else:
public_json_output = os.path.join(paths['branch-staging'], 'json')
build_json_output = os.path.join(paths['branch-output'], 'json')
branch_json_list_file = os.path.join(paths['branch-output'], 'json-file-list')
public_json_list_file = os.path.join(public_json_output, '.file_list')
m.section_break('meta')
m.target('json-output', ['json'])
m.job('fab process.json_output')
rsync_cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {0}/ {1}'
m.job(rsync_cmd.format(build_json_output, public_json_output))
m.msg('[json]: migrated all .json files to staging.')
m.msg('[json]: processed all json files.')
m.section_break('list file')
m.comment('the meta build system generates "{0}" when it generates this file'.format(branch_json_list_file))
fab_cmd = 'fab process.input:{0} process.output:{1} process.copy_if_needed:json'
m.target('json-file-list', public_json_list_file)
m.target(public_json_list_file, 'json-output')
m.job(fab_cmd.format(branch_json_list_file , public_json_list_file))
m.msg('[json]: rebuilt inventory of json output.')
m.target(build_json_output, 'json')
m.target('.PHONY', ['clean-json-output', 'clean-json', 'json-output', 'json-file-list'])
m.target('clean-json-output', 'clean-json')
m.job(' '.join(['rm -rf ', public_json_list_file, branch_json_list_file, public_json_output]))
m.msg('[json]: removed all processed json.')
return m
def sitemap(config_path=None):
paths = render_paths('obj')
sys.path.append(os.path.join(paths.projectroot, paths.buildsystem, 'bin'))
import sitemap_gen
if config_path is None:
config_path = os.path.join(paths.projectroot, 'conf-sitemap.xml')
if not os.path.exists(config_path):
puts('[ERROR] [sitemap]: configuration file {0} does not exist. Returning early'.fomrat(config_path))
return False
sitemap = sitemap_gen.CreateSitemapFromFile(configpath=config_path,
suppress_notify=True)
if sitemap is None:
puts('[ERROR] [sitemap]: failed to generate the sitemap due to encountered errors.')
return False
sitemap.Generate()
puts('[sitemap]: generated sitemap according to the config file {0}'.format(config_path))
return True
from fabric.api import cd, local, task, env, hide, settings
from fabric.utils import puts
from multiprocessing import cpu_count
import pkg_resources
import docs_meta
import datetime
paths = docs_meta.render_paths(True)
def get_tags(target):
if target.startswith('html') or target.startswith('dirhtml'):
return 'website'
else:
return 'print'
def timestamp(form='filename'):
if form == 'filename':
return datetime.datetime.now().strftime("%Y-%m-%d, %H:%M %p")
else:
return datetime.datetime.now().strftime("%Y%m%d%H:%M%p")
def get_sphinx_args(nitpick=None):
o = ''
if pkg_resources.get_distribution("sphinx").version.startswith('1.2'):
o += '-j ' + str(cpu_count() + 1)
if nitpick is not None:
o += '-n -w $(branch-output)/build.{0}.log'.format(timestamp('filename'))
return o
#!/usr/bin/python
import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
'../')))
import utils
from makecloth import MakefileCloth
from docs_meta import render_paths
m = MakefileCloth()
paths = render_paths('dict')
def pdf_makefile(name, tag):
name_tagged = '-'.join([name, tag])
name_tagged_pdf = name_tagged + '.pdf'
name_tagged_branch_pdf = '-'.join([name, tag, utils.get_branch()]) + '.pdf'
generated_latex = '{0}/latex/{1}.tex'.format(paths['branch-output'], name)
built_tex = '{0}/latex/{1}.tex'.format(paths['branch-output'], name_tagged)
built_pdf = '{0}/latex/{1}'.format(paths['branch-output'], name_tagged_pdf)
staged_pdf_branch = '{0}/{1}'.format(paths['branch-staging'],
name_tagged_branch_pdf)
staged_pdf = '{0}/{1}'.format(paths['branch-staging'], name_tagged_pdf)
m.section_break(name)
m.target(target=generated_latex, dependency='latex')
from fabric.api import cd, local, task, env, hide, settings
from fabric.utils import puts
from multiprocessing import cpu_count
import pkg_resources
import docs_meta
import datetime
paths = docs_meta.render_paths(True)
def get_tags(target):
if target.startswith('html') or target.startswith('dirhtml'):
return 'website'
else:
return 'print'
def timestamp(form='filename'):
if form == 'filename':
return datetime.datetime.now().strftime("%Y-%m-%d.%H-%M")
else:
return datetime.datetime.now().strftime("%Y-%m-%d, %H:%M %p")
def get_sphinx_args(nitpick=None):
o = ''
if pkg_resources.get_distribution("sphinx").version == '1.2b1-xgen-dev-20130529':
o += '-j ' + str(cpu_count() + 1) + ' '
if nitpick is not None:
o += '-n -w {0}/build.{1}.log'.format(paths['branch-output'], timestamp('filename'))
return o
#!/usr/bin/python
import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
'../')))
sys.path.append(
os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
from makecloth import MakefileCloth
from conf import intersphinx_mapping
import docs_meta
paths = docs_meta.render_paths('dict')
m = MakefileCloth()
def intersphinx_builders():
invs = []
for i in intersphinx_mapping:
output = '{0}/{1}.inv'.format(paths['output'], i)
m.target(output, block=i)
m.job('@fab {0}.url:{1}objects.inv {0}.file:{2} {0}.download'.format(
'intersphinx', intersphinx_mapping[i][0], output),
block=i)
m.newline(block=i)
invs.append(output)
m.newline(block='control')
m.target('intersphinx', invs, block='control')
invs = ' '.join(invs)
#!/usr/bin/python
import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
from makecloth import MakefileCloth
from conf import intersphinx_mapping
import docs_meta
paths = docs_meta.render_paths('dict')
m = MakefileCloth()
def intersphinx_builders():
invs = []
for i in intersphinx_mapping:
output = '{0}/{1}.inv'.format(paths['output'], i)
m.target(output, block=i)
m.job('@fab {0}.url:{1}objects.inv {0}.file:{2} {0}.download'.format('intersphinx', intersphinx_mapping[i][0], output), block=i)
m.newline(block=i)
invs.append(output)
m.newline(block='control')
m.target('intersphinx', invs, block='control')
invs = ' '.join(invs)
m.target('clean-intersphinx', block='control')
m.job('rm -f ' + invs, block='control')
m.msg('[intersphinx]: all existing intersphinx inv files removed.', block='control')
m.target('.PHONY', 'intersxpinx clean-intersphinx ' + invs, block='control')
import sys
import os.path
import argparse
import json
from multiprocessing import cpu_count
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../bin/')))
from utils import ingest_yaml_list
from docs_meta import render_paths
from makecloth import MakefileCloth
m = MakefileCloth()
paths = render_paths('dict')
def generate_targets(images):
image_files = []
image_rst_files = []
for image in images:
b = image['name']
source_base = '/'.join([image['dir'], image['name']])
source_file = source_base + '.svg'
m.section_break(image['name'], block=b)
m.newline(block=b)
for output in image['output']:
if 'tag' in output:
tag = '-' + output['tag']
else:
#!/usr/bin/python
import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "bin")))
import utils
from makecloth import MakefileCloth
from docs_meta import render_paths, get_manual_path, get_conf
m = MakefileCloth()
paths = render_paths("dict")
conf = get_conf()
correction = "'s/(index|bfcode)\{(.*!*)*--(.*)\}/\\1\{\\2-\{-\}\\3\}/g'"
pdf_latex_command = 'TEXINPUTS=".:{0}/latex/:" pdflatex --interaction batchmode --output-directory {0}/latex/ $(LATEXOPTS)'.format(
paths["branch-output"]
)
def pdf_makefile(name, tag=None, edition=None):
if tag is None:
name_tagged = name
name_tagged_branch_pdf = "-".join([name, utils.get_branch()]) + ".pdf"
else:
name_tagged = "-".join([name, tag])
name_tagged_branch_pdf = "-".join([name, tag, utils.get_branch()]) + ".pdf"
if conf.git.remote.upstream.endswith("mms-docs"):
site_url = "http://mms.10gen.com"
