def htaccess(fn='.htaccess'):
conf = lazy_conf()
in_files = ( i
for i in expand_tree(conf.paths.builddata, 'yaml')
if os.path.basename(i).startswith('htaccess') )
sources = []
for i in in_files:
sources.extend(ingest_yaml_list(i))
dirname = os.path.dirname(fn)
if not dirname == '' and not os.path.exists(dirname):
os.makedirs(dirname)
lines = set( [ ] )
for redir in sources:
lines.add(generate_redirects(process_redirect(redir, conf), conf=conf, match=False))
with open(fn, 'w') as f:
f.writelines(lines)
f.write('\n')
f.writelines( ['<FilesMatch "\.(ttf|otf|eot|woff)$">','\n',
' Header set Access-Control-Allow-Origin "*"', '\n'
'</FilesMatch>', '\n'] )
logging.info('redirect: regenerated {0} with {1} redirects ({2} lines)'.format(fn, len(sources), len(lines)))
def htaccess(fn='.htaccess'):
conf = lazy_conf()
in_files = (i for i in expand_tree(conf.paths.builddata, 'yaml')
if os.path.basename(i).startswith('htaccess'))
sources = []
for i in in_files:
sources.extend(ingest_yaml_list(i))
dirname = os.path.dirname(fn)
if not dirname == '' and not os.path.exists(dirname):
os.makedirs(dirname)
lines = set([])
for redir in sources:
lines.add(
generate_redirects(process_redirect(redir, conf),
conf=conf,
match=False))
with open(fn, 'w') as f:
f.writelines(lines)
f.write('\n')
f.writelines([
'<FilesMatch "\.(ttf|otf|eot|woff)$">', '\n',
' Header set Access-Control-Allow-Origin "*"', '\n'
'</FilesMatch>', '\n'
])
logging.info(
'redirect: regenerated {0} with {1} redirects ({2} lines)'.format(
fn, len(sources), len(lines)))
def migrate(conf=None):
"Migrates all manual files to primer according to the spec. As needed."
conf = lazy_conf(conf)
migration_paths = get_migration_specifications(conf)
if conf.project.name != 'primer':
return False
elif len(migration_paths) == 0:
return False
else:
migrations = ingest_yaml_list(*migration_paths)
truncate_jobs = []
munge_jobs = []
migration_jobs = []
append_jobs = []
for page in migrations:
if 'sources' in page:
migrations.extend(convert_multi_source(page))
continue
page = fix_migration_paths(page)
fq_target = os.path.join(conf.paths.projectroot, conf.paths.source,
page['target'])
fq_source = os.path.join(conf.paths.manual_source, page['source'])
migration_jobs.append(build_migration_job(fq_target, fq_source))
if 'truncate' in page:
truncate_jobs.append(
build_truncate_job(page['truncate'], fq_target))
if 'transform' in page:
migration_jobs[-1]['job'] = copy_always
munge_jobs.append(
build_transform_job(page['transform'], fq_target))
if 'append' in page:
migration_jobs[-1]['job'] = copy_always
append_jobs.append(
build_append_job(page, fq_target, migration_paths))
with ProcessPool() as p:
migration_res = p.runner(migration_jobs)
munge_res = p.runner(post_process_jobs(tasks=munge_jobs))
truncate_res = p.runner(truncate_jobs)
append_res = p.runner(append_jobs)
msg = '[primer]: migrated {0}, munged {1}, truncated {2}, and appended to {3} pages.'
print(
msg.format(len(migration_res), len(munge_res), len(truncate_res),
len(append_res)))
return True
def package(target=None, conf=None):
"Builds a package from the current build output."
if conf is None:
conf = get_conf()
archive_path = os.path.join(conf.paths.projectroot, conf.paths.buildarchive)
fn = package_filename(archive_path, target, conf)
pconf = conf_from_list('target', ingest_yaml_list(os.path.join(conf.paths.projectroot,
conf.paths.builddata,
'push.yaml')))
if target is None:
pconf = pconf[pconf.keys()[0]]
else:
pconf = pconf[target]
if not os.path.exists(archive_path):
os.makedirs(archive_path)
puts('[deploy] [tarball]: creating {0} directory'.format(archive_path))
else:
if not os.path.isdir(archive_path):
abort('[ERROR]: {0} exists and is not a directory.'.format(archive_path))
arc_conf = os.path.join(conf.paths.projectroot,
conf.paths.branch_output,
'conf.json')
with open(arc_conf, 'w') as f:
json.dump(get_build_metadata(conf), f, indent=2)
with tarfile.open(fn, 'w:gz') as t:
if 'branched' in pconf.options:
input_path = os.path.join(conf.paths.projectroot,
conf.paths.output,
pconf.paths.local,
conf.git.branches.current)
output_path_name = conf.git.branches.current
else:
input_path = os.path.join(conf.paths.projectroot,
conf.paths.output,
pconf.paths.local)
output_path_name = os.path.split(pconf.paths.local)[-1]
t.add(name=input_path,
arcname=output_path_name)
t.add(arc_conf, arcname='conf.json')
if 'static' in pconf.paths:
for path in pconf.paths.static:
rendered_path = os.path.join(conf.paths.projectroot,
conf.paths.public, path)
if os.path.exists(rendered_path):
t.add(name=rendered_path,
arcname=path)
puts('[deploy] [tarball]: created {0} as archive of current build artifacts.'.format(fn))
def migrate(conf=None):
"Migrates all manual files to primer according to the spec. As needed."
conf = lazy_conf(conf)
migration_paths = get_migration_specifications(conf)
if conf.project.name != 'primer':
return False
elif len(migration_paths) == 0:
return False
else:
migrations = ingest_yaml_list(*migration_paths)
truncate_jobs = []
munge_jobs = []
migration_jobs = []
append_jobs = []
for page in migrations:
if 'sources' in page:
migrations.extend(convert_multi_source(page))
continue
page = fix_migration_paths(page)
fq_target = os.path.join(conf.paths.projectroot, conf.paths.source, page['target'])
fq_source = os.path.join(conf.paths.manual_source, page['source'])
migration_jobs.append(build_migration_job(fq_target, fq_source))
if 'truncate' in page:
truncate_jobs.append(build_truncate_job(page['truncate'], fq_target))
if 'transform' in page:
migration_jobs[-1]['job'] = copy_always
munge_jobs.append(build_transform_job(page['transform'], fq_target))
if 'append' in page:
migration_jobs[-1]['job'] = copy_always
append_jobs.append(build_append_job(page, fq_target, migration_paths))
with ProcessPool() as p:
migration_res = p.runner(migration_jobs)
munge_res = p.runner(post_process_jobs(tasks=munge_jobs))
truncate_res = p.runner(truncate_jobs)
append_res = p.runner(append_jobs)
msg = '[primer]: migrated {0}, munged {1}, truncated {2}, and appended to {3} pages.'
print(msg.format(len(migration_res), len(munge_res), len(truncate_res), len(append_res)))
return True
def primer_migrate_pages(conf=None):
"Migrates all manual files to primer according to the spec. As needed."
conf = lazy_conf(conf)
migration_paths = get_migration_specifications(conf)
if conf.project.name != "primer":
return False
elif len(migration_paths) == 0:
return False
else:
migrations = ingest_yaml_list(*migration_paths)
truncate_jobs = []
munge_jobs = []
migration_jobs = []
append_jobs = []
for page in migrations:
if "sources" in page:
migrations.extend(convert_multi_source(page))
continue
page = fix_migration_paths(page)
fq_target = os.path.join(conf.paths.projectroot, conf.paths.source, page["target"])
fq_source = os.path.abspath(os.path.join(conf.paths.projectroot, "..", "source", page["source"]))
migration_jobs.append(build_migration_job(fq_target, fq_source))
if "truncate" in page:
truncate_jobs.append(build_truncate_job(page["truncate"], fq_target))
if "transform" in page:
migration_jobs[-1]["job"] = copy_always
munge_jobs.append(build_transform_job(page["transform"], fq_target))
if "append" in page:
migration_jobs[-1]["job"] = copy_always
append_jobs.append(build_append_job(page, fq_target, migration_paths))
with ProcessPool() as p:
migration_res = p.runner(migration_jobs)
munge_res = p.runner(post_process_jobs(tasks=munge_jobs))
truncate_res = p.runner(truncate_jobs)
append_res = p.runner(append_jobs)
msg = "migrated {0}, munged {1}, truncated {2}, and appended to {3} pages."
logger.info(msg.format(len(migration_res), len(munge_res), len(truncate_res), len(append_res)))
return True
def package(target=None, conf=None):
"Builds a package from the current build output."
if conf is None:
conf = get_conf()
archive_path = os.path.join(conf.paths.projectroot, conf.paths.buildarchive)
fn = package_filename(archive_path, target, conf)
pconf = conf_from_list(
"target", ingest_yaml_list(os.path.join(conf.paths.projectroot, conf.paths.builddata, "push.yaml"))
)
if target is None:
pconf = pconf[pconf.keys()[0]]
else:
pconf = pconf[target]
if not os.path.exists(archive_path):
os.makedirs(archive_path)
puts("[deploy] [tarball]: creating {0} directory".format(archive_path))
else:
if not os.path.isdir(archive_path):
abort("[ERROR]: {0} exists and is not a directory.".format(archive_path))
arc_conf = os.path.join(conf.paths.projectroot, conf.paths.branch_output, "conf.json")
with open(arc_conf, "w") as f:
json.dump(get_build_metadata(conf), f, indent=2)
with tarfile.open(fn, "w:gz") as t:
if "branched" in pconf.options:
input_path = os.path.join(
conf.paths.projectroot, conf.paths.output, pconf.paths.local, conf.git.branches.current
)
output_path_name = conf.git.branches.current
else:
input_path = os.path.join(conf.paths.projectroot, conf.paths.output, pconf.paths.local)
output_path_name = os.path.split(pconf.paths.local)[-1]
t.add(name=input_path, arcname=output_path_name)
t.add(arc_conf, arcname="conf.json")
if "static" in pconf.paths:
for path in pconf.paths.static:
rendered_path = os.path.join(conf.paths.projectroot, conf.paths.public, path)
if os.path.exists(rendered_path):
t.add(name=rendered_path, arcname=path)
puts("[deploy] [tarball]: created {0} as archive of current build artifacts.".format(fn))
def get_build_metadata(conf=None):
if conf is None:
conf = get_conf()
o = dict(push=dict(), conf=conf, meta=dict())
for target in ingest_yaml_list(os.path.join(conf.paths.projectroot, conf.paths.builddata, "push.yaml")):
o["push"][target["target"]] = target
o["meta"]["user"] = pwd.getpwuid(os.getuid())[0]
o["meta"]["platform"] = sys.platform
o["meta"]["time"] = datetime.datetime.utcnow().isoformat()
o["meta"]["host"] = os.uname()[1]
return o
def get_build_metadata(conf=None):
if conf is None:
conf=get_conf()
o = dict(push=dict(), conf=conf, meta=dict())
for target in ingest_yaml_list(os.path.join(conf.paths.projectroot,
conf.paths.builddata,
'push.yaml')):
o['push'][target['target']] = target
o['meta']['user'] = pwd.getpwuid(os.getuid())[0]
o['meta']['platform'] = sys.platform
o['meta']['time'] = datetime.datetime.utcnow().isoformat()
o['meta']['host'] = os.uname()[1]
return o
def deploy(target, conf=None, pconf=None):
"""Deploys a site. Specifies the deployment target defined in 'push.yaml'"""
conf = lazy_conf(conf)
push_conf = ingest_yaml_list(os.path.join(conf.paths.projectroot,
conf.paths.builddata,
'push.yaml'))
pconf = conf_from_list('target', push_conf)[target]
if pconf['target'] != target:
abort('[deploy] [ERROR]: this build environment does not support the {0} target'.format(target))
res = runner(deploy_jobs(target, conf, pconf), pool=2)
puts('[deploy]: pushed {0} targets'.format(len(res)))
def clean(conf=None):
"Removes all migrated primer files according to the current spec."
conf = lazy_conf(conf)
migration_paths = get_migration_specifications(conf)
migrations = ingest_yaml_list(*migration_paths)
targets = []
for page in migrations:
if 'sources' in page:
migrations.extend(convert_multi_source(page))
continue
page = fix_migration_paths(page)
targets.append(os.path.join(conf.paths.projectroot, conf.paths.source, page['target']))
map(verbose_remove, targets)
print('[clean] [primer]: removed {0} files'.format(len(targets)))
def clean(conf=None):
"Removes all migrated primer files according to the current spec."
conf = lazy_conf(conf)
migration_paths = get_migration_specifications(conf)
migrations = ingest_yaml_list(*migration_paths)
targets = []
for page in migrations:
if 'sources' in page:
migrations.extend(convert_multi_source(page))
continue
page = fix_migration_paths(page)
targets.append(os.path.join(conf.paths.projectroot, conf.paths.source, page['target']))
map(verbose_remove, targets)
logger.info('clean: removed {0} files'.format(len(targets)))
def intersphinx_jobs(conf=None):
conf = lazy_conf(conf)
data_file = os.path.join(conf.paths.projectroot,
conf.paths.builddata,
'intersphinx.yaml')
if not os.path.exists(data_file):
return
intersphinx_mapping = ingest_yaml_list(data_file)
for i in intersphinx_mapping:
f = os.path.join(conf.paths.projectroot,
conf.paths.output, i['path'])
s = i['url'] + 'objects.inv'
yield {
'target': f,
'dependency': None,
'job': download,
'args': { 'f': f, 's': s }
}
def pdf_jobs(target, conf):
pdfs = ingest_yaml_list(os.path.join(conf.paths.builddata, 'pdfs.yaml'))
tex_regexes = [ ( re.compile(r'(index|bfcode)\{(.*)--(.*)\}'),
r'\1\{\2-\{-\}\3\}'),
( re.compile(r'\\PYGZsq{}'), "'"),
( re.compile(r'\\code\{/(?!.*{}/|etc|usr|data|var|srv)'),
r'\code{' + conf.project.url + r'/' + conf.project.tag) ]
# this is temporary
queue = ( [], [], [], [], [] )
for i in pdfs:
tagged_name = i['output'][:-4] + '-' + i['tag']
deploy_fn = tagged_name + '-' + conf.git.branches.current + '.pdf'
link_name = deploy_fn.replace('-' + conf.git.branches.current, '')
latex_dir = os.path.join(conf.paths.branch_output, target)
if 'edition' in i:
deploy_path = conf.paths.mms[i['edition']]
target_split = target.split('-')
if len(target_split) > 1:
if target_split[1] != i['edition']:
logger.debug('not making pdf {0}, because {1} is not {2}'.format(link_name, target_split[1], i['edition']))
continue
if i['edition'] != 'hosted':
deploy_fn = tagged_name + '.pdf'
link_name = deploy_fn
else:
deploy_path = conf.paths.public_site_output
i['source'] = os.path.join(latex_dir, i['output'])
i['processed'] = os.path.join(latex_dir, tagged_name + '.tex')
i['pdf'] = os.path.join(latex_dir, tagged_name + '.pdf')
i['deployed'] = os.path.join(deploy_path, deploy_fn)
i['link'] = os.path.join(deploy_path, link_name)
i['path'] = latex_dir
# these appends will become yields, once runner() can be dependency
# aware.
queue[0].append(dict(dependency=None,
target=i['source'],
job=_clean_sphinx_latex,
args=(i['source'], tex_regexes)))
queue[1].append(dict(dependency=i['source'],
target=i['processed'],
job=copy_if_needed,
args=(i['source'], i['processed'], 'pdf')))
queue[2].append(dict(dependency=i['processed'],
target=i['pdf'],
job=_render_tex_into_pdf,
args=(i['processed'], i['path'])))
queue[3].append(dict(dependency=i['pdf'],
target=i['deployed'],
job=copy_if_needed,
args=(i['pdf'], i['deployed'], 'pdf')))
if i['link'] != i['deployed']:
queue[4].append(dict(dependency=i['deployed'],
target=i['link'],
job=create_link,
args=(deploy_fn, i['link'])))
return queue
def pdf_jobs(target, conf):
pdfs = ingest_yaml_list(os.path.join(conf.paths.builddata, 'pdfs.yaml'))
tex_regexes = [(re.compile(r'(index|bfcode)\{(.*)--(.*)\}'),
r'\1\{\2-\{-\}\3\}'), (re.compile(r'\\PYGZsq{}'), "'"),
(re.compile(r'\\code\{/(?!.*{}/|etc|usr|data|var|srv)'),
r'\code{' + conf.project.url + r'/' + conf.project.tag)]
# this is temporary
queue = ([], [], [], [], [])
for i in pdfs:
tagged_name = i['output'][:-4] + '-' + i['tag']
deploy_fn = tagged_name + '-' + conf.git.branches.current + '.pdf'
link_name = deploy_fn.replace('-' + conf.git.branches.current, '')
latex_dir = os.path.join(conf.paths.branch_output, target)
if 'edition' in i:
deploy_path = os.path.join(conf.paths.public, i['edition'])
target_split = target.split('-')
if len(target_split) > 1:
if target_split[1] != i['edition']:
continue
if i['edition'] == 'hosted':
deploy_path = os.path.join(deploy_path,
conf.git.branches.current)
else:
deploy_fn = tagged_name + '.pdf'
link_name = deploy_fn
else:
deploy_path = conf.paths.branch_staging
i['source'] = os.path.join(latex_dir, i['output'])
i['processed'] = os.path.join(latex_dir, tagged_name + '.tex')
i['pdf'] = os.path.join(latex_dir, tagged_name + '.pdf')
i['deployed'] = os.path.join(deploy_path, deploy_fn)
i['link'] = os.path.join(deploy_path, link_name)
i['path'] = latex_dir
# these appends will become yields, once runner() can be dependency
# aware.
queue[0].append(
dict(dependency=None,
target=i['source'],
job=_clean_sphinx_latex,
args=(i['source'], tex_regexes)))
queue[1].append(
dict(dependency=i['source'],
target=i['processed'],
job=copy_if_needed,
args=(i['source'], i['processed'], 'pdf')))
queue[2].append(
dict(dependency=i['processed'],
target=i['pdf'],
job=_render_tex_into_pdf,
args=(i['processed'], i['path'])))
if conf.project.name == 'mms' and mms_should_migrate(target,
conf) is False:
pass
else:
queue[3].append(
dict(dependency=i['pdf'],
target=i['deployed'],
job=copy_if_needed,
args=(i['pdf'], i['deployed'], 'pdf')))
if i['link'] != i['deployed']:
queue[4].append(
dict(dependency=i['deployed'],
target=i['link'],
job=create_link,
args=(deploy_fn, i['link'])))
return queue
