def extract_assets(html, basedir):
"""
Extracts all embedded assets in A links
encoded using data: and save them to file.
These are all links of the type:
<a href="data:****"/>
<a href="data:****" download='filename'/>
"""
if not os.path.exists(basedir):
os.makedirs(basedir)
soup = BeautifulSoup(html, 'lxml', from_encoding='utf-8')
for tag in soup.select('a[href]'):
href = tag['href']
if href.startswith('data:'):
_mime, data = get_mime_data_from_base64_string(href)
if tag.has_attr('download'):
basename = tag['download']
else:
logger.warn('cannot find attr "download" in tag')
# print tag
continue
print('extracting asset %s' % basename)
filename = os.path.join(basedir, basename)
write_data_to_file(data, filename)
def write_slides(res, outdir):
slides = res.get_result()
for id_slides, html in slides.items():
filename = os.path.join(outdir, id_slides + '.html')
data = str(html)
write_data_to_file(data, filename)
def go(d, out):
d0 = os.path.dirname(out)
artefacts = get_artefacts(d0, d)
print "\n".join(map(str, artefacts))
links = get_links_from_artefacts(artefacts)
body = Tag(name='body')
body.append(links)
html = Tag(name='html')
head = Tag(name='head')
meta = Tag(name='meta')
meta.attrs['content'] = "text/html; charset=utf-8"
meta.attrs['http-equiv'] = "Content-Type"
style = Tag(name='style')
style.append("""
body {
column-count: 3;
}
""")
head.append(meta)
html.append(head)
html.append(body)
write_data_to_file(str(html), out)
def render_book(src_dirs,
generate_pdf,
data,
realpath,
main_file,
use_mathjax,
out_part_basename,
raise_errors,
filter_soup=None,
extra_css=None,
symbols=None):
from mcdp_docs.pipeline import render_complete
librarian = get_test_librarian()
# XXX: these might need to be changed
if not MCDPConstants.softy_mode:
librarian.find_libraries('.')
load_library_hooks = [librarian.load_library]
library = MCDPLibrary(load_library_hooks=load_library_hooks)
for src_dir in src_dirs:
library.add_search_dir(src_dir)
d = tempfile.mkdtemp()
library.use_cache_dir(d)
def filter_soup0(soup, library):
if filter_soup is not None:
filter_soup(soup=soup, library=library)
add_edit_links(soup, realpath)
try:
html_contents = render_complete(library=library,
s=data,
raise_errors=raise_errors,
realpath=realpath,
use_mathjax=use_mathjax,
symbols=symbols,
generate_pdf=generate_pdf,
filter_soup=filter_soup0)
except DPSyntaxError as e:
msg = 'Could not compile %s' % realpath
raise_wrapped(DPSyntaxError, e, msg, compact=True)
doc = get_minimal_document(html_contents,
add_markdown_css=True,
extra_css=extra_css)
dirname = main_file + '.parts'
if dirname and not os.path.exists(dirname):
try:
os.makedirs(dirname)
except:
pass
fn = os.path.join(dirname, '%s.html' % out_part_basename)
write_data_to_file(doc, fn)
return html_contents
def do_it(f, rel, current_slug):
f2 = f + '.old'
if not os.path.exists(f2):
shutil.copy(f, f2)
orig = open(f2).read()
soup = bs_entire_document(orig)
soup2 = make_changes(soup, f, rel, current_slug)
data = to_html_entire_document(soup2)
data = data.replace('<body>', '<body>\n<?php header1() ?>\n')
write_data_to_file(data, f, quiet=True)
def generate_metadata(src_dir):
template = MCDPManualConstants.pdf_metadata_template
if not os.path.exists(template):
msg = 'Metadata template does not exist: %s' % template
raise ValueError(msg)
out = MCDPManualConstants.pdf_metadata
s = open(template).read()
from .pipeline import replace_macros
s = replace_macros(s)
write_data_to_file(s, out)
def prerender_main():
f0 = sys.argv[1]
f1 = sys.argv[2]
html = open(f0).read()
parsed = bs_entire_document(html)
body = parsed.html.body
body_string = str(body)
res = AugmentedResult()
body2_string = prerender_mathjax_(body_string, res)
body2 = bs(body2_string)
parsed.html.body.replace_with(body2)
html2 = str(parsed)
write_data_to_file(html2, f1)
def compose_go(compose_config):
input_ = compose_config.input
output = compose_config.output
recipe = compose_config.recipe
remove_status = compose_config.remove_status
show_removed = compose_config.show_removed
data = open(input_).read()
soup = bs_entire_document(data)
permalink_prefix = compose_config.purl_prefix
aug = compose_go2(soup, recipe, permalink_prefix, remove_status,
show_removed)
soup = aug.get_result()
results = str(soup)
write_data_to_file(results, output)
def remove_spurious(output_dir, filenames):
ignore = [
'link.html', 'toc.html', 'errors.html', 'warnings.html', 'tasks.html',
'crossref.html'
]
found = os.listdir(output_dir)
for f in found:
if not f.endswith('.html'):
continue
if f in ignore:
continue
if f not in filenames:
fn = os.path.join(output_dir, f)
if 'SPURIOUS' in open(fn).read():
# already marked as spurious
continue
msg = 'I found a spurious file from earlier compilations: %s' % fn
# msg += '(%s not in %s) ' % (f, filenames)
logger.warning(msg)
soup = read_html_doc_from_file(fn)
e = soup.find('section')
if e is not None and 'id' in e.attrs:
if False:
id_ = e.attrs['id'].replace(':section', '')
if 'autoid' not in id_:
id_ = remove_prefix(id_)
url = 'http://purl.org/dt/master/' + id_
OTHER = ((
'<p>Maybe try this link to find the version on master '
'(no guarantees): <a href="%s">%s</a></p>') %
(url, url))
OTHER += '\n<p>If that does not work, the section was renamed.</p>'
else:
OTHER = ''
else:
OTHER = ''
else:
OTHER = ''
data = spurious.replace('OTHER', OTHER)
write_data_to_file(data, fn, quiet=True)
def write_crossref_info(data, id2filename, output_crossref, permalink_prefix):
soup = bs_entire_document(data)
cross = Tag(name='body')
container = Tag(name='div')
container.attrs['id'] = 'container'
cross.append(container)
for e in soup.select('[label-name]'):
# logger.debug('considering %s' % e)
if not 'id' in e.attrs:
continue
id_ = e.attrs['id']
if id_.startswith('bib:'):
# logger.warn('Excluding %r from cross refs' % id_)
continue
e2 = get_crossref_copy(e)
e2.attrs[MCDPManualConstants.ATTR_BASE_URL] = permalink_prefix
if id_ in id2filename:
basename = id2filename[id_]
e2.attrs['url'] = '%s/%s#%s' % (permalink_prefix, basename, id_)
# print e2.attrs['url']
a = Tag(name='a')
a.attrs['href'] = e2.attrs['url']
if not 'autoid' in id_:
code = Tag(name='code')
code.append(id_)
a.append(code)
a.append(' ')
a.append(br())
a.append(e2.attrs['label-name'])
# e2.insert(0, Tag(name='br'))
# e2.insert(0, ' ')
e2.insert(0, a)
else:
logger.error('Cannot find url for %s' % id_)
cross.append('\n\n\n')
cross.append(e2)
for img in list(cross.find_all('img')):
img.extract()
# print('writing cross ref info')
html = Tag(name='html')
html.append(cross)
head = Tag(name='head')
style = Tag(name='style')
style.append(CROSSREF_CSS)
head.append(style)
html.append(head)
script = Tag(name='script')
script.append(CROSSREF_SCRIPT)
cross.append(script)
# XXX: we are doing this multiple times
write_data_to_file(str(html), output_crossref, quiet=True)
def go(context,
worker_i,
num_workers,
data,
mathjax,
preamble,
output_dir,
assets_dir,
add_toc_if_not_existing,
extra_panel_content,
permalink_prefix=None,
output_crossref=None,
only_refs=False):
res = AugmentedResult()
soup = bs_entire_document(data)
# extract the main toc if it is there
with timeit("Extracting main toc"):
main_toc = soup.find(id=MCDPManualConstants.MAIN_TOC_ID)
if main_toc is None:
if add_toc_if_not_existing:
# logger.info('Generating TOC because it is not there')
tocg = generate_toc(soup)
main_toc = bs(tocg).ul
main_toc.attrs['class'] = 'toc' # XXX: see XXX13
assert main_toc is not None
substituting_empty_links(main_toc,
raise_errors=False,
res=res,
extra_refs=soup)
else:
msg = 'Could not find main toc (id #%s)' % MCDPManualConstants.MAIN_TOC_ID
res.note_error(msg)
main_toc = Tag(name='div')
main_toc.append('TOC NOT FOUND')
else:
main_toc = main_toc.__copy__()
if 'id' in main_toc.attrs:
del main_toc.attrs['id']
# XXX: this is not the place to do it
mark_toc_links_as_errored(main_toc, soup)
body = soup.html.body
with timeit("split_in_files"):
filename2contents = split_in_files(body)
id2filename = get_id2filename(filename2contents)
res.set_result(id2filename)
if output_crossref is not None:
from mcdp_docs.mcdp_render_manual import write_crossref_info
context.comp(write_crossref_info,
data=data,
id2filename=id2filename,
output_crossref=output_crossref,
permalink_prefix=permalink_prefix)
if only_refs:
logger.debug('Skipping rest because only_refs')
return res
with timeit("add_prev_next_links"):
filename2contents = add_prev_next_links(filename2contents)
with timeit("preparing assets dir"):
if not os.path.exists(output_dir):
try:
os.makedirs(output_dir)
except:
pass
with timeit("creating link.html and link.js"):
linkbase = 'link.html' # do not change (it's used by http://purl.org/dth)
linkbasejs = 'link.js'
lb = create_link_base(id2filename)
write_data_to_file(str(lb),
os.path.join(output_dir, linkbase),
quiet=True)
linkjs = create_link_base_js(id2filename)
write_data_to_file(str(linkjs),
os.path.join(output_dir, linkbasejs),
quiet=True)
if preamble is not None:
if preamble.endswith('.tex'): # XXX
preamble = open(preamble).read()
ids_to_use = []
for k in list(id2filename):
if not 'autoid' in k:
ids_to_use.append(k)
ids_to_use = sorted(ids_to_use)
pointed_to = []
for k in ids_to_use:
f = id2filename[k]
if not f in pointed_to:
pointed_to.append(f)
# data = ",".join(pointed_to)
head0 = soup.html.head
if True:
context.comp(remove_spurious, output_dir, list(filename2contents))
with timeit('main_toc copy'):
main_toc0 = main_toc.__copy__()
main_toc0_s = str(main_toc0)
asset_jobs = []
for i, (filename, contents) in enumerate(filename2contents.items()):
if i % num_workers != worker_i:
continue
with timeit('main_toc copy hack'):
main_toc = bs(main_toc0_s).ul
assert main_toc is not None
# Trick: we add the main_toc, and then ... (look below)
with timeit('make_page'):
add_home_link = 'index.html' not in filename2contents
html = make_page(contents,
head0,
main_toc,
extra_panel_content,
add_home_link=add_home_link)
with timeit("direct job"):
result = only_second_part(mathjax, preamble, html, id2filename,
filename)
# ... we remove it. In this way we don't have to copy it every time...
main_toc.extract()
fn = os.path.join(output_dir, filename)
h = get_md5(result)[:8]
r = context.comp(extract_assets_from_file,
result,
fn,
assets_dir,
job_id='%s-%s-assets' % (filename, h))
asset_jobs.append(r)
update_refs_('toc.html', main_toc, id2filename)
out_toc = os.path.join(output_dir, 'toc.html')
write_data_to_file(str(main_toc), out_toc, quiet=True)
return context.comp(wait_assets, res, asset_jobs)
def go(compose_config):
input_ = compose_config.input
output = compose_config.output
recipe = compose_config.recipe
permalink_prefix = compose_config.purl_prefix
# Read input file
logger.info('Reading %s' % input_)
data = open(input_).read()
soup = bs_entire_document(data)
# Create context
doc = soup.__copy__()
body = Tag(name='body')
doc.body.replace_with(body)
elements = recipe.make(RecipeContext(soup=soup))
check_isinstance(elements, list)
append_all(body, elements)
# Now remove stuff
for status in compose_config.remove_status:
removed = []
for section in list(body.select('section[status=%s]' % status)):
level = section.attrs['level']
if not level in ['sec', 'part']:
continue
section_id = section.attrs['id']
pure_id = section_id.replace(':section', '')
removed.append(section.attrs['id'])
if compose_config.show_removed:
# remove everything that is not a header
keep = ['h1', 'h2', 'h3', 'h4', 'h5']
for e in list(section.children):
if e.name not in keep:
e.extract()
else:
e.append(' [%s]' % status)
p = Tag(name='p')
p.append(
"This section has been removed because it is in status %r. "
% (status))
a = Tag(name='a')
a.attrs['href'] = 'http://purl.org/dt/master/%s' % pure_id
a.append(
"If you are feeling adventurous, you can read it on master."
)
p.append(a)
section.append(p)
p = Tag(name='p')
p.append(
"To disable this behavior, and completely hide the sections, "
)
p.append(
"set the parameter show_removed to false in fall2017.version.yaml."
)
section.append(p)
else:
section.extract()
# section.replace_with(div)
if not removed:
logger.info('Found no section with status = %r to remove.' %
status)
else:
logger.info('I removed %d sections with status %r.' %
(len(removed), status))
logger.debug('Removed: %s' % ", ".join(removed))
add_github_links_if_edit_url(doc, permalink_prefix=permalink_prefix)
generate_and_add_toc(doc)
doc = doc.__copy__()
# generate_and_add_toc(soup)
# substituting_empty_links(soup)
raise_errors = False
find_links_from_master(master_soup=soup,
version_soup=doc,
raise_errors=raise_errors)
document_final_pass_after_toc(doc)
results = str(doc)
write_data_to_file(results, output)
def composing1():
data1 = """
docs:
file0.md: |
<div id='toc'></div>
# All units {#part:all}
file1.md: |
# Audacious {#sa status=ready}
This is section Audacious.
Linking to:
- (number name) <a href="#sa" class="number_name"></a>; (empty) [](#sa)
- (number name) <a href="#sb" class="number_name"></a>; (empty) [](#sb)
- (number name) <a href="#sc" class="number_name"></a>; (empty) [](#sc)
- And linking to [](#elephant).
file2.md: |
# Bumblebee {#sb status=ready}
This is section Bumblebee.
Linking to:
- (number name) <a href="#sa" class="number_name"></a>; (empty) [](#sa)
- (number name) <a href="#sb" class="number_name"></a>; (empty) [](#sb)
- (number name) <a href="#sc" class="number_name"></a>; (empty) [](#sc)
- And linking to [](#elephant).
## This one will be removed {#to-remove}
I don't like this section
# Elephant {#elephant status=draft}
Section Elephant is not ready.
file3.md: |
# The cat section {#sc status=ready}
This is section Cat.
Linking to:
- (number name) <a href="#sa" class="number_name"></a>; (empty) [](#sa)
- (number name) <a href="#sb" class="number_name"></a>; (empty) [](#sb)
- (number name) <a href="#sc" class="number_name"></a>; (empty) [](#sc)
00_main_template.html: |
<html>
<head></head>
<body</body>
</html>
book.version.yaml: |
input: dist/master/book.html
recipe:
- toc
- make-part: part1
title: First part
contents:
- add: sb
except: to-remove
- make-part: part2
title: Second part
contents:
- add: sa
- add: elephant
output: dist/version/book.html
purl_prefix: http://purl.org/dt/fall2017/
remove_status: [draft]
.compmake.rc:
config echo 1
"""
use = None
# to use a specific dir for debugging:
# use = '/tmp/composing1'
with with_dir_content(data1, use_dir=use):
repo = Repo.init('.')
fn = 'readme'
write_data_to_file('', fn)
repo.index.add([fn])
repo.index.commit("initial commit")
url = '[email protected]:AndreaCensi/example.git'
repo.create_remote('origin', url)
res = 'dist/master/book.html'
run_app(RenderManual, [
'--src', 'docs', '--stylesheet', 'v_manual_split', '--mathjax',
'0', '-o', 'out/out1', '--no_resolve_references', '--output_file',
res
])
assert os.path.exists(res)
data = bs_entire_document(open(res).read())
assert data.find(id='sa:section') is not None
assert data.find(id='sb:section') is not None
assert data.find(id='to-remove:section') is not None
run_app(Split, [
'--filename', 'dist/master/book.html', '--output_dir',
'dist/master/book'
])
run_app(Compose, ['--config', 'book.version.yaml'])
version_whole = bs_entire_document(
open('dist/version/book.html').read())
assert version_whole.find(id='sa:section') is not None
assert version_whole.find(id='sb:section') is not None
assert version_whole.find(id='to-remove:section') is None
# Now it's preserved
# assert version_whole.find(id='elephant:section') is None
run_app(Split, [
'--filename', 'dist/version/book.html', '--output_dir',
'dist/version/book'
])
def write_errors_and_warnings_files(aug, d):
if aug.has_result():
id2filename = aug.get_result()
else:
id2filename = {}
# print('id2filename: %s' % sorted(id2filename))
assert isinstance(aug, AugmentedResult)
aug.update_refs(id2filename)
header = get_notes_panel(aug)
manifest = []
nwarnings = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_WARNING))
fn = os.path.join(d, 'warnings.html')
html = html_list_of_notes(aug,
MCDPManualConstants.NOTE_TAG_WARNING,
'warnings',
'warning',
header=header)
# update_refs_('warnings', html, id2filename)
write_data_to_file(str(html), fn, quiet=True)
if nwarnings:
manifest.append(
dict(display='%d warnings' % nwarnings, filename='warnings.html'))
msg = 'There were %d warnings: %s' % (nwarnings, fn)
logger.warn(msg)
ntasks = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_TASK))
fn = os.path.join(d, 'tasks.html')
html = html_list_of_notes(aug,
MCDPManualConstants.NOTE_TAG_TASK,
'tasks',
'task',
header=header)
# update_refs_('tasks', html, id2filename)
write_data_to_file(str(html), fn, quiet=True)
if nwarnings:
manifest.append(
dict(display='%d tasks' % ntasks, filename='tasks.html'))
msg = 'There are %d open tasks: %s' % (ntasks, fn)
logger.info(msg)
nerrors = len(aug.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_ERROR))
fn = os.path.join(d, 'errors.html')
html = html_list_of_notes(aug,
MCDPManualConstants.NOTE_TAG_ERROR,
'errors',
'error',
header=header)
# update_refs_('tasks', html, id2filename)
write_data_to_file(str(html), fn, quiet=True)
if nerrors:
manifest.append(
dict(display='%d errors' % nerrors, filename='errors.html'))
msg = 'I am sorry to say that there were %d errors.\n\nPlease see: %s' % (
nerrors, fn)
logger.error('\n\n\n' + indent(msg, ' ' * 15) + '\n\n')
fn = os.path.join(d, 'errors_and_warnings.manifest.yaml')
write_data_to_file(yaml.dump(manifest), fn, quiet=False)
fn = os.path.join(d, 'errors_and_warnings.pickle')
res = AugmentedResult()
res.merge(aug)
write_data_to_file(pickle.dumps(res), fn, quiet=False)
def write(s, out):
write_data_to_file(s, out)
def write_manifest_html(d):
manifest = [dict(display='html', filename='index.html')]
fn = os.path.join(d, 'output-html.manifest.yaml')
write_data_to_file(yaml.dump(manifest), fn, quiet=False)
def render_book(
src_dirs,
generate_pdf,
data,
realpath,
use_mathjax,
raise_errors,
filter_soup=None,
symbols=None,
ignore_ref_errors=False,
):
""" Returns an AugmentedResult(str) """
res = AugmentedResult()
from mcdp_docs.pipeline import render_complete
librarian = get_test_librarian()
# XXX: these might need to be changed
if not MCDPConstants.softy_mode:
for src_dir in src_dirs:
librarian.find_libraries(src_dir)
load_library_hooks = [librarian.load_library]
library_ = MCDPLibrary(load_library_hooks=load_library_hooks)
for src_dir in src_dirs:
library_.add_search_dir(src_dir)
d = tempfile.mkdtemp()
library_.use_cache_dir(d)
location = LocalFile(realpath)
# print('location:\n%s' % location)
def filter_soup0(soup, library):
if filter_soup is not None:
filter_soup(soup=soup, library=library)
add_edit_links2(soup, location)
add_last_modified_info(soup, location)
try:
html_contents = render_complete(library=library_,
s=data,
raise_errors=raise_errors,
realpath=realpath,
use_mathjax=use_mathjax,
symbols=symbols,
generate_pdf=generate_pdf,
filter_soup=filter_soup0,
location=location,
res=res,
ignore_ref_errors=ignore_ref_errors)
except DPSyntaxError as e:
msg = 'Could not compile %s' % realpath
location0 = LocationInString(e.where, location)
res.note_error(msg, locations=location0)
fail = "<p>This file could not be compiled</p>"
res.set_result(fail)
return res
# raise_wrapped(DPSyntaxError, e, msg, compact=True)
if False: # write minimal doc
doc = get_minimal_document(html_contents,
add_markdown_css=True,
extra_css=extra_css)
dirname = main_file + '.parts'
if dirname and not os.path.exists(dirname):
try:
os.makedirs(dirname)
except:
pass
fn = os.path.join(dirname, '%s.html' % out_part_basename)
write_data_to_file(doc, fn)
res.set_result(html_contents)
return res
def write(s_aug, out):
s = s_aug.get_result()
write_data_to_file(s, out)
def write_manifest_pdf(out_pdf):
d = os.path.dirname(out_pdf)
basename = os.path.basename(out_pdf)
manifest = [dict(display='PDF', filename=basename)]
fn = os.path.join(d, 'output-pdf.manifest.yaml')
write_data_to_file(yaml.dump(manifest), fn, quiet=False)
def savefile(filename_hint, data):
""" must return the url (might be equal to filename) """
where = os.path.join(assets_dir, filename_hint)
write_data_to_file(data, where)
relative = os.path.relpath(where, os.path.dirname(fo))
return relative
def go(context, worker_i, num_workers, ifilename, mathjax, preamble,
output_dir):
with timeit("reading %s" % ifilename):
soup = read_html_doc_from_file(ifilename)
# extract the main toc if it is there
with timeit("Extracting main_toc"):
main_toc = soup.find(id='main_toc')
if main_toc is None:
msg = 'Could not find the element #main_toc.'
raise ValueError(msg)
main_toc = main_toc.__copy__()
del main_toc.attrs['id']
body = soup.html.body
with timeit("split_in_files"):
filename2contents = split_in_files(body)
with timeit("add_prev_next_links"):
filename2contents = add_prev_next_links(filename2contents)
with timeit("preparing assets dir"):
if not os.path.exists(output_dir):
try:
os.makedirs(output_dir)
except:
pass
assets_dir = os.path.join(output_dir, 'assets')
with timeit("creating link.html and link.js"):
id2filename = get_id2filename(filename2contents)
#<<<<<<< HEAD
linkbase = 'link.html' # do not change (it's used by http://purl.org/dth)
linkbasejs = 'link.js'
#=======
# linkbase = 'link.html' # do not change (it's used by http://purl.org/dth)
#>>>>>>> nl
lb = create_link_base(id2filename)
write_data_to_file(str(lb), os.path.join(output_dir, linkbase))
linkjs = create_link_base_js(id2filename)
write_data_to_file(str(linkjs), os.path.join(output_dir, linkbasejs))
if preamble:
preamble = open(preamble).read()
ids_to_use = []
for k in list(id2filename):
if not 'autoid' in k:
ids_to_use.append(k)
ids_to_use = sorted(ids_to_use)
pointed_to = []
for k in ids_to_use:
f = id2filename[k]
if not f in pointed_to:
pointed_to.append(f)
data = ",".join(pointed_to)
links_hash = get_md5(data)[:8]
# if self.options.faster_but_imprecise:
# links_hash = "nohash"
#
# logger.debug('hash data: %r' % data)
logger.debug('hash value: %r' % links_hash)
head0 = soup.html.head
if True:
context.comp(remove_spurious, output_dir, list(filename2contents))
tmpd = create_tmpdir()
n = len(filename2contents)
for i, (filename, contents) in enumerate(filename2contents.items()):
if (i % num_workers != worker_i):
continue
# contents_hash = get_md5(str(contents) + str(preamble))[:8]
# job_id = '%s-%s-%s' % (filename, links_hash, contents_hash)
#<<<<<<< HEAD
# Trick: we add the main_toc, and then ... (look below)
with timeit('make_page'):
html = make_page(contents, head0, main_toc)
with timeit('main_toc copy'):
main_toc = main_toc.__copy__()
logger.debug('%d/%d: %s' % (i, n, filename))
with timeit("direct job"):
result = only_second_part(mathjax, preamble, html, id2filename,
filename)
# ... we remove it. In this way we don't have to copy it
main_toc.extract()
fn = os.path.join(output_dir, filename)
fn0 = os.path.join(tmpd, filename)
write_data_to_file(result, fn0, quiet=True)
h = get_md5(result)[:8]
context.comp(extract_assets_from_file,
fn0,
fn,
assets_dir,
job_id='assets-%s' % h)
def write_html_doc_to_file(soup, filename, quiet=False):
from mcdp_utils_misc import write_data_to_file
html = to_html_entire_document(soup)
write_data_to_file(html, filename, quiet=quiet)
def go():
groups = OrderedDict(yaml.load(BOOKS))
import os
dist = 'duckuments-dist'
html = Tag(name='html')
head = Tag(name='head')
meta = Tag(name='meta')
meta.attrs['content'] = "text/html; charset=utf-8"
meta.attrs['http-equiv'] = "Content-Type"
stylesheet = 'v_manual_split'
link = Tag(name='link')
link['rel'] = 'stylesheet'
link['type'] = 'text/css'
link['href'] = get_css_filename('compiled/%s' % stylesheet)
head.append(link)
body = Tag(name='body')
style = Tag(name='style')
style.append(CSS)
head.append(style)
head.append(meta)
html.append(head)
html.append(body)
divgroups = Tag(name='div')
all_crossrefs = Tag(name='div')
res = AugmentedResult()
for id_group, group in groups.items():
divgroup = Tag(name='div')
divgroup.attrs['class'] = 'group'
divgroup.attrs['id'] = id_group
h0 = Tag(name='h1')
h0.append(group['title'])
divgroup.append(h0)
if 'abstract' in group:
p = Tag(name='p')
p.append(group['abstract'])
divgroup.append(p)
books = group['books']
# divbook = Tag(name='div')
books = OrderedDict(books)
for id_book, book in books.items():
d = os.path.join(dist, id_book)
change_frame(d, '../../', current_slug=id_book)
d0 = dist
errors_and_warnings = os.path.join(d, 'out', 'errors_and_warnings.pickle')
if os.path.exists(errors_and_warnings):
resi = pickle.loads(open(errors_and_warnings).read())
# print(errors_and_warnings)
resi.update_file_path(prefix=os.path.join(id_book, 'out'))
res.merge(resi)
else:
msg = 'Path does not exist: %s' % errors_and_warnings
logger.error(msg)
artefacts = get_artefacts(d0, d)
div = Tag(name='div')
div.attrs['class'] = 'book-div'
div.attrs['id'] = id_book
div_inside = Tag(name='div')
div_inside.attrs['class'] = 'div_inside'
links = get_links2(artefacts)
for a in links.select('a'):
s = gettext(a)
if 'error' in s or 'warning' in s or 'task' in s:
a['class'] = 'EWT'
if False:
h = Tag(name='h3')
h.append(book['title'])
# div_inside.append(h)
if 'abstract' in book:
p = Tag(name='p')
p.append(book['abstract'])
div_inside.append(p)
div_inside.append(links)
div.append(div_inside)
toc = os.path.join(d, 'out/toc.html')
if os.path.exists(toc):
data = open(toc).read()
x = bs(data)
for a in x.select('a[href]'):
href = a.attrs['href']
a.attrs['href'] = id_book + '/out/' + href
x.name = 'div' # not fragment
div.append(x)
crossrefs = os.path.join(d, 'crossref.html')
if os.path.exists(crossrefs):
x = bs(open(crossrefs).read())
for e in x.select('[url]'):
all_crossrefs.append('\n\n')
all_crossrefs.append(e.__copy__())
else:
logger.error('File does not exist %s' % crossrefs)
divgroup.append(div)
divgroups.append(divgroup)
out_pickle = sys.argv[3]
nwarnings = len(res.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_WARNING))
ntasks = len(res.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_TASK))
nerrors = len(res.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_ERROR))
logger.info('%d tasks' % ntasks)
logger.warning('%d warnings' % nwarnings)
logger.error('%d nerrors' % nerrors)
from mcdp_docs.mcdp_render_manual import write_errors_and_warnings_files
write_errors_and_warnings_files(res, os.path.dirname(out_pickle))
out_junit = os.path.join(os.path.dirname(out_pickle), 'junit', 'notes', 'junit.xml')
s = get_junit_xml(res)
write_data_to_file(s.encode('utf8'), out_junit)
# write_data_to_file(pickle.dumps(res), out_pickle, quiet=False)
extra = get_extra_content(res)
extra.attrs['id'] = 'extra'
body.append(extra)
body.append(divgroups)
embed_css_files(html)
for e in body.select('.notes-panel'):
e.extract()
out = sys.argv[1]
data = str(html)
data = data.replace('<body>', '<body>\n<?php header1() ?>\n')
write_data_to_file(data, out)
manifest = [dict(display='index', filename=os.path.basename(out))]
mf = os.path.join(os.path.dirname(out), 'summary.manifest.yaml')
write_data_to_file(yaml.dump(manifest), mf)
out_crossrefs = sys.argv[2]
html = Tag(name='html')
head = Tag(name='head')
body = Tag(name='body')
style = Tag(name='style')
style.append(CROSSREF_CSS)
head.append(style)
html.append(head)
script = Tag(name='script')
script.append(CROSSREF_SCRIPT)
container = Tag(name='div')
container.attrs['id'] = 'container'
body.append(container)
details = Tag(name='details')
summary = Tag(name='summary')
summary.append('See all references')
details.append(summary)
details.append(all_crossrefs)
body.append(details)
body.append(script)
html.append(body)
write_data_to_file(str(html), out_crossrefs)
if nerrors > 0:
sys.exit(nerrors)