def option_tasks(conf):
option_sources = conf.system.content.options.sources
o = OptionDataCache(option_sources, conf)
if len(option_sources) > 0 and not os.path.isdir(
conf.system.content.options.output_dir):
safe_create_directory(conf.system.content.options.output_dir)
tasks = []
for dep_fn, option in o.content_iter():
if option.program.startswith('_'):
continue
out_fn = hyph_concat(option.directive, option.program,
option.name) + '.rst'
output_fn = os.path.join(conf.system.content.options.fn_prefix, out_fn)
t = Task(job=write_options,
description='generating option file "{0}" from "{1}"'.format(
output_fn, dep_fn),
target=output_fn,
dependency=[dep_fn])
t.args = (option, output_fn, conf)
tasks.append(t)
logger.info("added tasks for {0} option generation tasks".format(
len(tasks)))
return tasks
def step_tasks(conf):
step_sources = conf.system.content.steps.sources
s = StepDataCache(step_sources, conf)
if len(step_sources) > 0 and not os.path.isdir(
conf.system.content.steps.output_dir):
safe_create_directory(conf.system.content.steps.output_dir)
tasks = []
for fn, stepf in s.file_iter():
basename = conf.system.content.steps.get_basename(fn)
out_fn = os.path.join(conf.system.content.steps.output_dir,
basename) + '.rst'
t = Task(job=write_steps,
description='generate a stepfile for ' + fn,
target=out_fn,
dependency=fn)
t.args = (stepf, out_fn, conf)
tasks.append(t)
logger.info("added tasks for {0} step generation tasks".format(len(tasks)))
return tasks
def example_tasks(conf):
# In the beginning of this operation, which executes in the main thread, we
# read all files in "source/includes/" and sub-directories that start with
# "example-*"
example_sources = conf.system.content.examples.sources
# process the corpus of example data.
d = ExampleDataCache(example_sources, conf)
if len(example_sources) > 0 and not os.path.isdir(
conf.system.content.examples.output_dir):
safe_create_directory(conf.system.content.examples.output_dir)
tasks = []
for fn, exmpf in d.file_iter():
if exmpf.collection is None or exmpf.collection.options.base_file is True:
continue
basename = conf.system.content.examples.get_basename(fn)
out_fn = os.path.join(conf.system.content.examples.output_dir,
basename) + '.rst'
t = Task(
job=write_full_example,
description='generate an example for ' + fn,
target=out_fn,
dependency=fn,
)
t.args = (exmpf.collection, exmpf.examples, out_fn)
tasks.append(t)
logger.info("added tasks for {0} example generation tasks".format(
len(tasks)))
return tasks
def finalize_single_html_tasks(builder, conf):
single_html_dir = get_single_html_dir(conf)
# create directory when registering tasks.
safe_create_directory(single_html_dir)
safe_create_directory(os.path.join(single_html_dir, '_static'))
if 'edition' in conf.project and conf.project.edition != conf.project.name:
artifact_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
'-'.join((builder, conf.project.edition)))
else:
artifact_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder)
tasks = [giza.libgiza.task.Task(job=finalize_single_html,
args=(single_html_dir, artifact_dir, conf),
target=True,
dependency=None,
description="migrating singlehtml")]
for fn in expand_tree(os.path.join(artifact_dir, '_static'), None):
target_fn = os.path.join(single_html_dir, '_static', os.path.basename(fn))
tasks.append(giza.libgiza.task.Task(job=copy_if_needed,
args=(fn, target_fn),
target=target_fn,
dependency=fn,
description="moving static files to the singlehtml build"))
return tasks
def download(path, url, conf):
try:
mtime = os.stat(path).st_mtime
except OSError:
mtime = -1
now = time.time()
if now < (mtime + MAX_AGE):
logger.debug('Intersphinx file still young: %s', url)
return
request = urllib.request.Request(
url, headers={'If-Modified-Since': email.utils.formatdate(mtime)})
safe_create_directory(os.path.dirname(path))
try:
response = urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS)
with open(path, 'wb') as f:
f.write(response.read())
except urllib.error.HTTPError as err:
if err.code == 304:
logger.debug('Not modified: %s', url)
return
logger.error('Error downloading %s: Got %d', url, err.code)
except urllib.error.URLError as err:
logger.error('Error downloading %s: %s', url, str(err))
def transfer_images(conf, sconf):
image_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_images)
if not os.path.isdir(image_dir):
return False
elif sconf.builder == 'latex':
if 'edition' in sconf and sconf.edition is not None:
builder_dir = '-'.join((sconf.builder, sconf.edition))
else:
builder_dir = sconf.builder
builder_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_dir)
safe_create_directory(builder_dir)
cmd = ('rsync -am '
'--include="*.png" --include="*.jpg" --include="*.eps" '
'--exclude="*" {0}/ {1}')
cmd = cmd.format(image_dir, builder_dir)
with open(os.devnull, 'w') as null:
for img_cmd in (shlex.split(cmd.replace('images', 'figures')),
shlex.split(cmd)):
subprocess.call(img_cmd, stdout=null, stderr=null)
logger.info('migrated images for latex build')
def json_output(conf):
list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
public_list_file = os.path.join(conf.paths.projectroot,
conf.paths.public_site_output,
'json', '.file_list')
cmd = ('rsync --recursive --times --delete '
'--exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" '
'{src} {dst}')
json_dst = os.path.join(conf.paths.projectroot, conf.paths.public_site_output, 'json')
safe_create_directory(json_dst)
builder = 'json'
if 'edition' in conf.project and conf.project.edition != conf.project.name:
builder += '-' + conf.project.edition
cmd_str = cmd.format(src=os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder) + '/',
dst=json_dst)
try:
subprocess.check_call(cmd_str.split())
copy_if_needed(list_file, public_list_file)
logger.info('deployed json files to local staging.')
except subprocess.CalledProcessError:
logger.error('error migrating json artifacts to local staging')
def download(path, url, conf):
try:
mtime = os.stat(path).st_mtime
except OSError:
mtime = -1
now = time.time()
if now < (mtime + MAX_AGE):
logger.debug('Intersphinx file still young: %s', url)
return
request = urllib.request.Request(url, headers={
'If-Modified-Since': email.utils.formatdate(mtime)
})
safe_create_directory(os.path.dirname(path))
try:
response = urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS)
with open(path, 'wb') as f:
f.write(response.read())
except urllib.error.HTTPError as err:
if err.code == 304:
logger.debug('Not modified: %s', url)
return
logger.error('Error downloading %s: Got %d', url, err.code)
except urllib.error.URLError as err:
logger.error('Error downloading %s: %s', url, str(err))
def transfer_images(conf, sconf):
image_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_images)
if not os.path.isdir(image_dir):
return False
elif sconf.builder == 'latex':
if 'edition' in sconf and sconf.edition is not None:
builder_dir = '-'.join((sconf.builder, sconf.edition))
else:
builder_dir = sconf.builder
builder_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder_dir)
safe_create_directory(builder_dir)
cmd = ('rsync -am '
'--include="*.png" --include="*.jpg" --include="*.eps" '
'--exclude="*" {0}/ {1}')
cmd = cmd.format(image_dir, builder_dir)
with open(os.devnull, 'w') as null:
for img_cmd in (shlex.split(cmd.replace('images', 'figures')),
shlex.split(cmd)):
subprocess.call(img_cmd, stdout=null, stderr=null)
logger.info('migrated images for latex build')
def toc_tasks(conf):
toc_sources = conf.system.content.toc.sources
tocs = TocDataCache(toc_sources, conf)
if len(toc_sources) > 0 and not os.path.isdir(
conf.system.content.toc.output_dir):
safe_create_directory(conf.system.content.toc.output_dir)
tasks = []
for dep_fn, toc_data in tocs.file_iter():
deps = [dep_fn]
if 'ref-toc-' in dep_fn:
base_offset = 8
else:
base_offset = 4
fn_basename = os.path.basename(dep_fn)[base_offset:].replace(
'yaml', 'rst')
toc_items = toc_data.ordered_items()
if toc_data.is_spec() is False:
out_fn = os.path.join(conf.system.content.toc.output_dir,
fn_basename)
t = Task(job=write_toc_tree_output,
target=out_fn,
dependency=dep_fn,
description="writing toctree to '{0}'".format(out_fn))
t.args = (out_fn, toc_items)
tasks.append(t)
else:
deps.extend(toc_data.spec_deps())
if 'ref-toc' in dep_fn:
out_fn = os.path.join(conf.system.content.toc.output_dir,
hyph_concat('table', fn_basename))
reft = Task(
job=write_toc_table,
target=out_fn,
dependency=deps,
description="write table of contents generator".format(out_fn))
reft.args = (out_fn, toc_items)
tasks.append(reft)
else:
out_fn = os.path.join(conf.system.content.toc.output_dir,
hyph_concat('dfn-list', fn_basename))
dt = Task(job=write_dfn_list_output,
target=out_fn,
dependency=deps,
description="write definition list toc to '{0}'".format(
out_fn))
dt.args = (out_fn, toc_items)
tasks.append(dt)
logger.info('added tasks for {0} toc generation tasks'.format(len(tasks)))
return tasks
def create_archive(files_to_archive, tarball_name):
# ready to write the tarball
safe_create_directory(os.path.dirname(tarball_name))
with tarfile.open(tarball_name, 'w:gz') as t:
for fn, arc_fn in files_to_archive:
t.add(name=fn, arcname=arc_fn)
def create_archive(files_to_archive, tarball_name):
# ready to write the tarball
safe_create_directory(os.path.dirname(tarball_name))
with tarfile.open(tarball_name, 'w:gz') as t:
for fn, arc_fn in files_to_archive:
t.add(name=fn, arcname=arc_fn)
def download_file(file, url):
cmd = ['curl', '-s', '--remote-time', url, '-o', file]
safe_create_directory(os.path.dirname(file))
try:
subprocess.check_call(cmd)
logger.info('downloaded {0}'.format(file))
return True
except subprocess.CalledProcessError:
logger.error('trouble downloading interspinx inventory: ' + file)
return False
def package_build_env(builders, editions, languages, conf):
arc_fn = hyph_concat('cache', conf.project.name, conf.git.branches.current,
datetime.datetime.utcnow().strftime('%s'),
conf.git.commit[:8]) + ".tar.gz"
archive_path = os.path.join(conf.paths.buildarchive, arc_fn)
safe_create_directory(conf.paths.buildarchive)
existing_archives = os.listdir(conf.paths.buildarchive)
for arc in existing_archives:
if conf.git.commit[:8] in arc:
logger.warning(
'archive "{0}" exists for current git hash, not recreating'.
format(archive_path))
return
logger.debug("no archive for commit '{0}' continuing".format(
conf.git.commit))
with cd(conf.paths.projectroot):
files_to_archive = set()
for edition, language, builder in itertools.product(
editions, languages, builders):
rconf, sconf = get_sphinx_build_configuration(
edition, language, builder, copy.deepcopy(conf.runstate))
builder_dirname = resolve_builder_path(builder, edition, language,
rconf)
files_to_archive.add(rconf.paths.branch_source)
files_to_archive.add(
os.path.join(rconf.paths.branch_output, builder_dirname))
files_to_archive.add(
os.path.join(rconf.paths.branch_output,
hyph_concat('doctrees', builder_dirname)))
files_to_archive.add(rconf.system.dependency_cache_fn)
files_to_archive = list(files_to_archive)
logger.info('prepped build cache archive. writing file now.')
for fn in files_to_archive:
if not os.path.exists(fn):
raise FileNotFoundError(fn)
try:
with tarfile.open(archive_path, 'w:gz') as t:
for fn in files_to_archive:
t.add(fn, exclude=is_git_dir)
logger.info("created build-cache archive: " + archive_path)
except Exception as e:
os.remove(archive_path)
logger.critical("failed to create archive: " + archive_path)
logger.error(e)
def download_file(file, url):
cmd = ['curl', '--silent', '--location', '--remote-time', url, '-o', file]
safe_create_directory(os.path.dirname(file))
try:
subprocess.check_call(cmd)
logger.info('downloaded {0}'.format(file))
return True
except subprocess.CalledProcessError:
logger.error('trouble downloading interspinx inventory: ' + file)
return False
def generate_list_file(outputs, path, conf):
dirname = os.path.dirname(path)
safe_create_directory(dirname)
url = get_site_url(conf)
url.append('json')
url = '/'.join(url)
with open(path, 'w') as f:
for fn in outputs:
if os.path.isfile(fn) is True:
line = '/'.join([url, fn.split('/', 3)[3:][0]])
f.write(line)
f.write('\n')
logger.info('rebuilt inventory of json output.')
def finalize_single_html_tasks(builder, conf, app):
single_html_dir = get_single_html_dir(conf)
safe_create_directory(single_html_dir)
found_src = False
for base_path in (builder, hyph_concat(builder, conf.project.edition)):
if found_src is True:
break
for fn in [
os.path.join(base_path, f)
for f in ('contents.html', 'index.html')
]:
src_fn = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, fn)
if os.path.exists(src_fn):
manual_single_html(input_file=src_fn,
output_file=os.path.join(
single_html_dir, 'index.html'))
copy_if_needed(
source_file=os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
base_path, 'objects.inv'),
target_file=os.path.join(single_html_dir, 'objects.inv'))
found_src = True
break
if found_src is not True:
raise FileNotFoundError('singlehtml source file')
single_path = os.path.join(single_html_dir, '_static')
for fn in expand_tree(os.path.join(os.path.dirname(src_fn), '_static'),
None):
target_fn = os.path.join(single_path, os.path.basename(fn))
task = app.add('task')
task.job = copy_if_needed
task.target = target_fn
task.dependency = fn
task.args = [fn, target_fn]
task.description = "migrating static files to the HTML build"
def task(task, conf):
if task == 'source':
legacy_tables = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.source,
'reference'), 'yaml')
dirname = os.path.join(conf.paths.projectroot, conf.paths.includes,
'apiargs')
safe_create_directory(dirname)
offset = len(os.path.join(conf.paths.projectroot, conf.paths.source))
elif task == 'branch':
legacy_tables = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.branch_source,
'reference'), 'yaml')
safe_create_directory(conf.system.content.apiargs.output_dir)
offset = len(
os.path.join(conf.paths.projectroot, conf.paths.branch_source))
else:
logger.critical('cannot perform apiarg migration for: ' + str(task))
return
new_apiarg = []
new_fns = []
for fn in legacy_tables:
new_data, new_fn = migrate_legacy_apiarg(task, fn, conf)
if new_fn in new_fns:
logger.error("duplicate: {0}, from: {1}".format(
os.path.basename(new_fn), os.path.basename(fn)))
else:
new_fns.append(new_fn[offset:])
new_apiarg.append((new_fn, new_data))
for fn, data in new_apiarg:
write_yaml(data, fn)
# for fn in legacy_tables:
# os.remove(fn)
new_sources = conf.system.content.apiargs.sources
if len(new_sources) != len(legacy_tables) and len(legacy_tables) != len(
new_fns):
logger.critical('problem in apiargs table migration.')
else:
logger.info('legacy apiargs tables migrated successfully.')
legacy_tables = [fn[offset:] for fn in legacy_tables]
return zip(legacy_tables, new_fns)
def package_build_env(builders, editions, languages, conf):
arc_fn = '-'.join(['cache',
conf.project.name,
conf.git.branches.current,
datetime.datetime.utcnow().strftime('%s'),
conf.git.commit[:8]]) + ".tar.gz"
archive_path = os.path.join(conf.paths.buildarchive, arc_fn)
safe_create_directory(conf.paths.buildarchive)
existing_archives = os.listdir(conf.paths.buildarchive)
for arc in existing_archives:
if conf.git.commit[:8] in arc:
m = 'archive "{0}" exists for current git hash, not recreating'
logger.warning(m.format(archive_path))
return
logger.debug("no archive for commit '{0}' continuing".format(conf.git.commit))
with cd(conf.paths.projectroot):
files_to_archive = set()
for ((edition, language, builder), (rconf, sconf)) in get_builder_jobs(conf):
files_to_archive.add(rconf.paths.branch_source)
files_to_archive.add(os.path.join(rconf.paths.branch_output,
sconf.build_output))
files_to_archive.add(os.path.join(rconf.paths.branch_output,
'-'.join(('doctrees', sconf.build_output))))
files_to_archive.add(rconf.system.dependency_cache_fn)
files_to_archive = list(files_to_archive)
logger.info('prepped build cache archive. writing file now.')
for fn in files_to_archive:
if not os.path.exists(fn):
raise FileNotFoundError(fn)
try:
with tarfile.open(archive_path, 'w:gz') as t:
for fn in files_to_archive:
t.add(fn, exclude=is_git_dir)
logger.info("created build-cache archive: " + archive_path)
except Exception as e:
os.remove(archive_path)
logger.critical("failed to create archive: " + archive_path)
logger.error(e)
def apiarg_tasks(conf):
with Timer('apiargs migrations'):
name_changes = migration_task(task='branch', conf=conf)
apiarg_sources = conf.system.content.apiargs.sources
a = ApiArgDataCache(apiarg_sources, conf)
if len(apiarg_sources) > 0 and not os.path.isdir(
conf.system.content.apiargs.output_dir):
safe_create_directory(conf.system.content.apiargs.output_dir)
tasks = []
# for dep_fn, table in a.file_iter():
# print dep_fn
logger.info(
'new apiargs not yet implemented, but there are {0} of them'.format(
str(len(conf.system.content.apiargs.sources))))
return []
def extract_tasks(conf):
extract_sources = conf.system.content.extracts.sources
extracts = ExtractDataCache(extract_sources, conf)
if len(extract_sources) > 0 and not os.path.isdir(
conf.system.content.extracts.output_dir):
safe_create_directory(conf.system.content.extracts.output_dir)
tasks = []
for dep_fn, extract in extracts.content_iter():
t = Task(job=write_extract_file,
description="generating extract file: " + extract.target,
target=extract.target,
dependency=dep_fn)
t.args = (extract, extract.target)
tasks.append(t)
include_statement = get_include_statement(extract.target_project_path)
for verb, adjc, noun in [(prepend_to_file, 'prepend', extract.prepend),
(append_to_file, 'append', extract.append)]:
if noun:
if not isinstance(noun, list):
files = [noun]
else:
files = noun
for fn in files:
t = Task(
job=verb,
target=fn,
dependency=dep_fn,
description="{0} extract include for '{0}' to '{1}'".
format(adjc, extract.target, fn))
t.args = (fn, include_statement)
tasks.append(t)
logger.info("added tasks for {0} extract generation tasks".format(
len(tasks)))
return tasks
def test_build_site(args):
args.languages_to_build = args.editions_to_build = []
args.builder = 'html'
conf = fetch_config(args)
safe_create_directory('build')
with BuildApp.new(pool_type=conf.runstate.runner,
pool_size=conf.runstate.pool_size,
force=conf.runstate.force).context() as app:
try:
sphinx_publication(conf, args, app)
except:
sphinx_publication(conf, args, app)
if os.path.exists('doc-tools'):
shutil.rmtree('docs-tools')
logger.info('bootstrapped makefile system')
logger.info('updated project skeleton in current directory.')
def dump_file_hashes(conf):
output = conf.system.dependency_cache
o = {'time': datetime.datetime.utcnow().strftime("%s"), 'files': {}}
files = expand_tree(
os.path.join(conf.paths.projectroot, conf.paths.branch_source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
safe_create_directory(os.path.dirname(output))
with open(output, 'w') as f:
json.dump(o, f)
logger.info('wrote dependency cache to: {0}'.format(output))
def transfer_source(conf, sconf):
target = os.path.join(conf.paths.projectroot, conf.paths.branch_source)
dir_exists = safe_create_directory(target)
# this operation is just for messaging the above operation, and error'ing
# appropriately.
if dir_exists is True:
logger.info('created directory for sphinx build: {0}'.format(target))
elif not os.path.isdir(target):
msg = '"{0}" exists and is not a directory'.format(target)
logger.error(msg)
raise InvalidFile(msg)
source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
exclusions = [
os.path.join('includes', 'table'),
os.path.join('includes', 'generated')
]
if conf.paths.images is not None:
image_dir = os.path.join(conf.paths.images[len(conf.paths.source) +
1:])
exclusions.append(image_dir + os.path.sep + '*.png')
exclusions.append(image_dir + os.path.sep + '*.rst')
prefix_len = len(
os.path.join(conf.paths.projectroot, conf.paths.branch_source)) + 1
exclusions.extend([
o for o in conf.system.content.output_directories(prefix_len)
if o != "includes/changelogs"
])
# we don't want rsync to delete directories that hold generated content in
# the target so we can have more incremental builds.
exclusions = "--exclude=" + ' --exclude='.join(exclusions)
cmd = 'rsync --links --checksum --recursive {2} --delete {0}/ {1}'
cmd = cmd.format(source_dir, target, exclusions)
try:
subprocess.check_call(shlex.split(cmd))
except subprocess.CalledProcessError as e:
logger.error('source transfer rsync had error: ' + str(e.returncode))
logger.info(cmd)
# remove files from the source tree specified in the sphinx config for this
# build.
source_exclusion(conf, sconf)
os.utime(target, None)
logger.info(
'prepared and migrated source for sphinx build in {0}'.format(target))
def dump_file_hashes(conf):
output = conf.system.dependency_cache
o = {'time': datetime.datetime.utcnow().strftime("%s"),
'files': {}}
files = expand_tree(os.path.join(conf.paths.projectroot, conf.paths.branch_source), None)
fmap = o['files']
for fn in files:
if os.path.exists(fn):
fmap[fn] = md5_file(fn)
safe_create_directory(os.path.dirname(output))
with open(output, 'w') as f:
json.dump(o, f)
logger.debug('wrote dependency cache to: {0}'.format(output))
def transfer_images(conf, sconf):
image_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_images)
if not os.path.isdir(image_dir):
return False
elif sconf.builder == 'latex':
if 'edition' in sconf and sconf.edition is not None:
builder_dir = hyph_concat(sconf.builder, sconf.edition)
else:
builder_dir = sconf.builder
builder_dir = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder_dir)
safe_create_directory(builder_dir)
cmd = 'rsync -am --include="*.png" --include="*.jpg" --include="*.eps" --exclude="*" {0}/ {1} '.format(image_dir, builder_dir)
command(cmd)
command(cmd.replace('images', 'figures'), ignore=True)
logger.info('migrated images for latex build')
def test_build_site(args):
args.languages_to_build = args.editions_to_build = []
args.builder = 'html'
conf = fetch_config(args)
safe_create_directory('build')
with BuildApp.new(pool_type=conf.runstate.runner,
pool_size=conf.runstate.pool_size,
force=conf.runstate.force).context() as app:
try:
sphinx_publication(conf, args, app)
except:
sphinx_publication(conf, args, app)
if os.path.exists('doc-tools'):
shutil.rmtree('docs-tools')
logger.info('bootstrapped makefile system')
logger.info('updated project skeleton in current directory.')
def json_output(conf):
list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
public_list_file = os.path.join(conf.paths.projectroot,
conf.paths.public_site_output,
'json', '.file_list')
cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'
json_dst = os.path.join(conf.paths.projectroot, conf.paths.public_site_output, 'json')
safe_create_directory(json_dst)
builder = 'json'
if 'edition' in conf.project and conf.project.edition != conf.project.name:
builder += '-' + conf.project.edition
command(cmd.format(src=os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder) + '/',
dst=json_dst))
copy_if_needed(list_file, public_list_file)
logger.info('deployed json files to local staging.')
def release_tasks(conf):
release_sources = conf.system.content.releases.sources
rel = ReleaseDataCache(release_sources, conf)
if len(release_sources) > 0 and not os.path.isdir(
conf.system.content.releases.output_dir):
safe_create_directory(conf.system.content.releases.output_dir)
tasks = []
for dep_fn, release in rel.content_iter():
t = Task(job=write_release_file,
description='generating release spec file: ' + release.target,
target=release.target,
dependency=dep_fn)
t.args = (release, release.target, conf)
tasks.append(t)
logger.info("added tasks for {0} release generation tasks".format(
len(tasks)))
return tasks
def run_sphinx(builder, sconf, conf):
if safe_create_directory(sconf.fq_build_output):
m = 'created directory "{1}" for sphinx builder {0}'
logger.info(m.format(builder, sconf.fq_build_output))
if 'language' in sconf and sconf.language is not None:
cmd_str = 'sphinx-intl build --language=' + sconf.language
try:
subprocess.check_call(shlex.split(cmd_str))
logger.info('compiled all PO files for translated build.')
except subprocess.CalledProcessError as e:
logger.error('sphinx-intl encountered error: ' + str(e.returncode))
logger.info(cmd_str)
logger.info('starting sphinx build {0}'.format(builder))
cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree
sphinx_cmd = cmd.format(
get_sphinx_args(sconf, conf),
os.path.join(conf.paths.projectroot,
conf.paths.branch_output), sconf.build_output,
os.path.join(conf.paths.projectroot, conf.paths.branch_source),
sconf.fq_build_output)
logger.debug(sphinx_cmd)
m = "running sphinx build for: {0}, {1}, {2}"
with Timer(m.format(builder, sconf.language, sconf.edition)):
try:
output = subprocess.check_output(shlex.split(sphinx_cmd),
stderr=subprocess.STDOUT)
return_code = 0
except subprocess.CalledProcessError as e:
output = e.output
return_code = e.returncode
logger.info(sphinx_cmd)
try:
os.utime(sconf.fq_build_output, None)
except:
pass
m = 'completed {0} sphinx build for {1}.{2}.{3} ({4})'
logger.info(
m.format(builder, conf.project.name, conf.project.edition,
conf.git.branches.current, return_code))
return return_code, output
def finalize_single_html_tasks(builder, conf):
single_html_dir = get_single_html_dir(conf)
# create directory when registering tasks.
safe_create_directory(single_html_dir)
safe_create_directory(os.path.join(single_html_dir, '_static'))
if 'edition' in conf.project and conf.project.edition != conf.project.name:
artifact_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, '-'.join(
(builder, conf.project.edition)))
else:
artifact_dir = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder)
tasks = [
libgiza.task.Task(job=finalize_single_html,
args=(single_html_dir, artifact_dir, conf),
target=True,
dependency=None,
description="migrating singlehtml")
]
for fn in expand_tree(os.path.join(artifact_dir, '_static'), None):
target_fn = os.path.join(single_html_dir, '_static',
os.path.basename(fn))
tasks.append(
libgiza.task.Task(
job=copy_if_needed,
args=(fn, target_fn),
target=target_fn,
dependency=fn,
description="moving static files to the singlehtml build"))
return tasks
def run_sphinx(builder, sconf, conf):
if safe_create_directory(sconf.fq_build_output):
m = 'created directory "{1}" for sphinx builder {0}'
logger.info(m.format(builder, sconf.fq_build_output))
if 'language' in sconf and sconf.language is not None:
cmd_str = 'sphinx-intl build --language=' + sconf.language
try:
subprocess.check_call(shlex.split(cmd_str))
logger.info('compiled all PO files for translated build.')
except subprocess.CalledProcessError as e:
logger.error('sphinx-intl encountered error: ' + str(e.returncode))
logger.info(cmd_str)
logger.info('starting sphinx build {0}'.format(builder))
cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree
sphinx_cmd = cmd.format(get_sphinx_args(sconf, conf),
os.path.join(conf.paths.projectroot, conf.paths.branch_output),
sconf.build_output,
os.path.join(conf.paths.projectroot, conf.paths.branch_source),
sconf.fq_build_output)
logger.debug(sphinx_cmd)
m = "running sphinx build for: {0}, {1}, {2}"
with Timer(m.format(builder, sconf.language, sconf.edition)):
try:
output = subprocess.check_output(shlex.split(sphinx_cmd), stderr=subprocess.STDOUT)
return_code = 0
except subprocess.CalledProcessError as e:
output = e.output
return_code = e.returncode
logger.info(sphinx_cmd)
try:
os.utime(sconf.fq_build_output, None)
except:
pass
m = 'completed {0} sphinx build for {1}.{2}.{3} ({4})'
logger.info(m.format(builder, conf.project.name, conf.project.edition,
conf.git.branches.current, return_code))
return return_code, output
def run_sphinx(builder, sconf, conf):
if safe_create_directory(sconf.fq_build_output):
logger.info('created directory "{1}" for sphinx builder {0}'.format(
builder, sconf.fq_build_output))
if 'language' in sconf and sconf.language is not None:
command('sphinx-intl build --language=' + sconf.language)
logger.info('compiled all PO files for translated build.')
logger.info('starting sphinx build {0}'.format(builder))
cmd = 'sphinx-build {0} -d {1}/doctrees-{2} {3} {4}' # per-builder-doctree
sphinx_cmd = cmd.format(
get_sphinx_args(sconf, conf),
os.path.join(conf.paths.projectroot,
conf.paths.branch_output), sconf.build_output,
os.path.join(conf.paths.projectroot, conf.paths.branch_source),
sconf.fq_build_output)
logger.debug(sphinx_cmd)
with Timer("running sphinx build for: {0}, {1}, {2}".format(
builder, sconf.language, sconf.edition)):
out = command(sphinx_cmd, capture=True, ignore=True)
logger.info('completed sphinx build {0}'.format(builder))
if True: # out.return_code == 0:
logger.info('successfully completed {0} sphinx build ({1})'.format(
builder, out.return_code))
finalizer_app = BuildApp(conf)
finalizer_app.pool = "thread"
finalizer_app.root_app = False
finalize_sphinx_build(sconf, conf, finalizer_app)
with Timer("finalize sphinx {0} build".format(builder)):
finalizer_app.run()
else:
logger.warning(
'the sphinx build {0} was not successful. not running finalize operation'
.format(builder))
output = '\n'.join([out.err, out.out])
return out.return_code, output
def transfer_source(conf, sconf):
target = os.path.join(conf.paths.projectroot, conf.paths.branch_source)
dir_exists = safe_create_directory(target)
# this operation is just for messaging the above operation, and error'ing
# appropriately.
if dir_exists is True:
logger.info('created directory for sphinx build: {0}'.format(target))
elif not os.path.isdir(target):
msg = '"{0}" exists and is not a directory'.format(target)
logger.error(msg)
raise InvalidFile(msg)
source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
image_dir = os.path.join(conf.paths.images[len(conf.paths.source) + 1:])
exclusions = [os.path.join('includes', 'table'),
os.path.join('includes', 'generated'),
image_dir + os.path.sep + "*.png",
image_dir + os.path.sep + "*.rst",
image_dir + os.path.sep + "*.eps"]
prefix_len = len(os.path.join(conf.paths.projectroot, conf.paths.branch_source)) + 1
exclusions.extend([o for o in conf.system.content.output_directories(prefix_len) if o != "includes/changelogs"])
# we don't want rsync to delete directories that hold generated content in
# the target so we can have more incremental builds.
exclusions = "--exclude=" + ' --exclude='.join(exclusions)
cmd = 'rsync --links --checksum --recursive {2} --delete {0}/ {1}'
cmd = cmd.format(source_dir, target, exclusions)
try:
subprocess.check_call(shlex.split(cmd))
except subprocess.CalledProcessError as e:
logger.error('source transfer rsync had error: ' + str(e.returncode))
logger.info(cmd)
# remove files from the source tree specified in the sphinx config for this
# build.
source_exclusion(conf, sconf)
os.utime(target, None)
logger.info('prepared and migrated source for sphinx build in {0}'.format(target))
def transfer_source(conf, sconf):
target = os.path.join(conf.paths.projectroot, conf.paths.branch_source)
dir_exists = safe_create_directory(target)
# this operation is just for messaging the above operation, and error'ing
# appropriately.
if dir_exists is True:
logger.info('created directory for sphinx build: {0}'.format(target))
elif not os.path.isdir(target):
msg = '"{0}" exists and is not a directory'.format(target)
logger.error(msg)
raise InvalidFile(msg)
source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
image_dir = os.path.join(conf.paths.images[len(conf.paths.source)+1:])
ref_dir = 'reference'
exclusions = [ os.path.join('includes', 'table'),
os.path.join('includes', 'generated'),
os.path.join(ref_dir, 'method') + os.path.sep + "*.rst",
os.path.join(ref_dir, 'command') + os.path.sep + "*.rst",
os.path.join(ref_dir, 'operator', 'query') + os.path.sep + "*.rst",
os.path.join(ref_dir, 'operator', 'aggregation') + os.path.sep + "*.rst",
ref_dir + os.path.sep + "*.rst",
image_dir + os.path.sep + "*.png",
image_dir + os.path.sep + "*.rst",
image_dir + os.path.sep + "*.eps" ]
prefix_len = len(os.path.join(conf.paths.projectroot, conf.paths.branch_source)) + 1
exclusions.extend([ o for o in conf.system.content.output_directories(prefix_len) ])
# we don't want rsync to delete directories that hold generated content in
# the target so we can have more incremental builds.
exclusions = "--exclude=" + ' --exclude='.join(exclusions)
cmd = 'rsync --checksum --recursive {2} --delete {0}/ {1}'.format(source_dir, target, exclusions)
command(cmd)
# remove files from the source tree specified in the sphinx config for this
# build.
source_exclusion(conf, sconf)
os.utime(target, None)
logger.info('prepared and migrated source for sphinx build in {0}'.format(target))
def make_parent_dirs(*paths):
for path in paths:
safe_create_directory(os.path.dirname(path))
def make_parent_dirs(*paths):
for path in paths:
safe_create_directory(os.path.dirname(path))