def get_document_properties(soup):
""" Reads a document's <meta> tags into a dict """
metas = list(soup.select('meta'))
FK, FV = 'name', 'content'
properties = {}
for e in metas:
if not FK in e.attrs or not FV in e.attrs:
msg = 'Expected "%s" and "%s" attribute for meta tag.' % (FK, FV)
raise_desc(ValueError, msg, tag=describe_tag(e))
properties[e[FK]] = e[FV]
return properties
def col_macro_(e, ncols):
"""
Bug: For some reasone bd4 removes the whitespace I use for indentation.
"""
assert e.name == 'div'
assert e.has_attr('make-col%d' % ncols)
# print describe_tag(e)
children = list(e.children)
# remove strings from this
is_string = lambda x: isinstance(x, NavigableString)
strings = [_ for _ in children if is_string(_)]
children = [_ for _ in children if not is_string(_)]
if len(children) < ncols:
msg = ('Cannot create table with %r cols with only %d children' %
(ncols, len(children)))
raise_desc(ValueError, msg, tag=describe_tag(e))
for c in children:
c.extract()
for s in strings:
ss = str(s)
empty = not ss.strip()
if not empty:
msg = 'Found nonempty string %r between children.' % ss
raise_desc(ValueError, msg, tag=describe_tag(e))
# remove it
s.extract()
nchildren = len(children)
nrows = int(math.ceil(nchildren / float(ncols)))
parent = e.parent
original_position = parent.index(e)
e.extract()
table = e
e.name = 'table'
add_class(table, 'col%d' % ncols)
add_class(table, 'colN')
wrapper = Tag(name='div')
add_class(wrapper, 'col%d-wrap' % ncols)
add_class(wrapper, 'colN-wrap')
NL = '\n'
# S = '-' * 4
# XXX: change to above to see the problem with indentation
S = ' ' * 4
tbody = Tag(name='tbody')
for row in range(nrows):
tbody.append(NavigableString(NL))
tbody.append(NavigableString(S+S))
tr = Tag(name='tr')
tr.append(NavigableString(NL))
for col in range(ncols):
td = Tag(name='td')
i = col + row * ncols
if i < len(children):
child = children[i]
td.append(child)
else:
td.append(Comment('empty row %d col %d' % (row, col)))
tr.append(NavigableString(S+S+S))
tr.append(td)
tr.append(NavigableString(NL))
tr.append(S+S)
if row == 0 and ('labels-row1' in e.attrs.get('class', '')):
thead = Tag(name='thead')
thead.append(tr)
table.append(thead) # add in table, not tbody
else:
tbody.append(tr) # add in tbody
tbody.append(NavigableString(NL+S))
table.append(tbody)
wrapper.append(NavigableString(NL + S))
wrapper.append(table)
wrapper.append(NavigableString(NL))
parent.insert(original_position, wrapper)
def highlight_mcdp_code(library,
soup,
realpath,
res,
location,
generate_pdf=False,
raise_errors=False):
# print(indent(frag, 'highlight_mcdp_code '))
""" Looks for codes like:
<pre class="mcdp">mcdp {
# empty model
}
</pre>
and does syntax hihglighting.
"""
assert soup.name == 'fragment'
def go(selector, parse_expr, extension, use_pre=True, refine=None):
for tag in soup.select(selector):
source_code = '<unset>' # XXX
try:
if tag.string is None: # or not tag.string.strip():
if not tag.has_attr('id'):
msg = "If <pre> is empty then it needs to have an id."
raise_desc(ValueError, msg, tag=describe_tag(tag))
# load it
tag_id = tag['id'].encode('utf-8')
if '.' in tag_id:
i = tag_id.index('.')
libname, name = tag_id[:i], tag_id[i + 1:]
use_library = library.load_library(libname)
else:
name = tag_id
use_library = library
basename = '%s.%s' % (name, extension)
data = use_library._get_file_data(basename)
source_code = data['data']
else:
source_code = get_source_code(tag)
# prettify.
# remove spurious indentation
source_code = source_code.strip()
do_apply_suggestions = (not tag.has_attr('noprettify')
and not tag.has_attr('np'))
# then apply suggestions
try:
if do_apply_suggestions:
x = parse_wrap(parse_expr, source_code)[0]
xr = parse_ndp_refine(x, Context())
suggestions = get_suggestions(xr)
source_code = apply_suggestions(
source_code, suggestions)
except DPSyntaxError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
continue
# we don't want the browser to choose different tab size
# source_code = source_code.replace('\t', ' ' * 4)
# we are not using it
_realpath = realpath
context = Context()
def postprocess(x):
if refine is not None:
return refine(x, context=context)
else:
return x
# print('rendering source code %r' % source_code)
html = ast_to_html(source_code,
parse_expr=parse_expr,
add_line_gutter=False,
postprocess=postprocess)
for w in context.warnings:
if w.where is not None:
from mcdp_web.editor_fancy.app_editor_fancy_generic import html_mark
html = html_mark(html, w.where, "language_warning")
frag2 = BeautifulSoup(html, 'lxml', from_encoding='utf-8')
if use_pre:
rendered = Tag(name='div', attrs={'class': 'rendered'})
pre = frag2.pre
pre.extract()
rendered.append(pre)
if not rendered.has_attr('class'):
rendered['class'] = ""
if tag.has_attr('label'):
text = tag['label']
tag_label = Tag(name='span')
add_class(tag_label, 'label')
add_class(tag_label, 'label_inside')
tag_label.append(NavigableString(text))
pre.insert(0, tag_label)
tag_label_outside = Tag(name='span')
add_class(tag_label_outside, 'label')
add_class(tag_label_outside, 'label_outside')
tag_label_outside.append(NavigableString(text))
rendered.insert(0, tag_label_outside)
max_len = max_len_of_pre_html(html)
if tag.has_attr('label'):
add_class(rendered, 'has_label')
max_len = max(max_len, len(tag['label']) + 6)
style = ''
else:
# using <code>
rendered = frag2.pre.code
rendered.extract()
if not rendered.has_attr('class'):
rendered['class'] = ""
style = ''
if tag.has_attr('style'):
style = style + tag['style']
if style:
rendered['style'] = style
if tag.has_attr('class'):
add_class(rendered, tag['class'])
if tag.has_attr('id'):
rendered['id'] = tag['id']
if use_pre:
if generate_pdf:
pdf = get_ast_as_pdf(source_code, parse_expr)
if tag.has_attr('id'):
basename = tag['id']
else:
hashcode = hashlib.sha224(
source_code).hexdigest()[-8:]
basename = 'code-%s' % hashcode
docname = os.path.splitext(
os.path.basename(realpath))[0]
download = docname + '.' + basename + '.source_code.pdf'
a = create_a_to_data(download=download,
data_format='pdf',
data=pdf)
a['class'] = 'pdf_data'
a.append(NavigableString(download))
div = Tag(name='div')
div.append(rendered)
div.append(a)
tag.replaceWith(div)
else:
tag.replaceWith(rendered)
else:
tag.replaceWith(rendered)
except DPSyntaxError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
# note_error(tag, e)
if tag.string is None:
tag.string = "`%s" % tag['id']
continue
except DPSemanticError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
# note_error(tag, e)
if tag.string is None:
tag.string = "`%s" % tag['id']
continue
except DPInternalError as ex:
msg = 'Error while interpreting the code:\n\n'
msg += indent(source_code, ' | ')
raise_wrapped(DPInternalError, ex, msg, exc=sys.exc_info())
abbrevs = {
# tag name: (new name, classes to add)
'fname': ('code', ['FName']),
'rname': ('code', ['RName']),
'poset': ('code', ['mcdp_poset']),
'value': ('code', ['mcdp_value']),
'fvalue': ('code', ['mcdp_value', 'fvalue']),
'rvalue': ('code', ['mcdp_value', 'rvalue']),
'impname': ('code', ['impname']),
'k': ('code', ['keyword']),
'program': ('code', ['program']),
'f': ('span', ['f']),
'r': ('span', ['r']),
'imp': ('span', ['imp']),
'kf': ('code', ['f', 'keyword']),
'kr': ('code', ['r', 'keyword']),
'cf': ('code', ['f']),
'cr': ('code', ['r']),
}
for original_tag_name, (new_tag_name, classes_to_add) in abbrevs.items():
for e in soup.select(original_tag_name):
e.name = new_tag_name
for c in classes_to_add:
add_class(e, c)
# warn not to get confused ith '_' and '-'
special_classes = [
'mcdp_poset', 'mcdp_fvalue', 'mcdp_rvalue', 'mcdp_value'
]
for x in special_classes:
# we do not expect to see an element that has class with '-' instead of '_'
erroring = x.replace('_', '-')
mistakes = list(soup.select('.%s' % erroring))
if mistakes:
msg = 'You cannot use %r as a class; use lowercase.' % erroring
tags = "\n\n".join(
indent(describe_tag(_), ' | ') for _ in mistakes)
raise_desc(ValueError, msg, tags=tags)
for x in special_classes:
# mcdp_poset -> mcdp-poset
corresponding = x.replace('_', '-')
for e in soup.select(corresponding):
# e2 = Tag(name='code')
# copy_string_and_attrs(e, e2)
e.name = 'code'
# THEN add class
add_class(e, x)
prettify = list(soup.select('fname')) + list(soup.select('rname'))
for e in prettify:
if e.has_attr('np') or e.has_attr('noprettify'):
x0 = e.text.encode('utf-88')
x1 = get_suggested_identifier(x0)
e.text = unicode(x1, 'utf-8')
mcdp_dev_warning('lets try if this goes away') # XXX
# this is a bug with bs4. The replace_with above only adds an escaped
# text rather than the actual tag (!).
# soup = bs(to_html_stripping_fragment(soup))
# assert soup.name == 'fragment'
go('pre.mcdp',
Syntax.ndpt_dp_rvalue,
"mcdp",
use_pre=True,
refine=parse_ndp_refine)
go('pre.mcdp_poset',
Syntax.space,
"mcdp_poset",
use_pre=True,
refine=parse_poset_refine)
go('pre.mcdp_template',
Syntax.template,
"mcdp_template",
use_pre=True,
refine=parse_template_refine)
go('pre.mcdp_statements',
Syntax.dp_model_statements,
"mcdp_statements",
use_pre=True)
go('pre.mcdp_fvalue', Syntax.fvalue, "mcdp_fvalue", use_pre=True)
go('pre.mcdp_rvalue', Syntax.rvalue, "mcdp_rvalue", use_pre=True)
# todo: add deprecation
go('pre.mcdp_value', Syntax.rvalue, "mcdp_value", use_pre=True)
go('code.mcdp', Syntax.ndpt_dp_rvalue, "mcdp", use_pre=False)
go('code.mcdp_poset', Syntax.space, "mcdp_poset", use_pre=False)
go('code.mcdp_value', Syntax.rvalue, "mcdp_value", use_pre=False)
go('code.mcdp_template', Syntax.template, "mcdp_template", use_pre=False)
# this is a bug with bs4...
for pre in soup.select('pre + pre'):
# print('adding br between PREs')
br_ = br()
br_['class'] = 'pre_after_pre'
pre.parent.insert(pre.parent.index(pre), br_)
def go(selector, parse_expr, extension, use_pre=True, refine=None):
for tag in soup.select(selector):
source_code = '<unset>' # XXX
try:
if tag.string is None: # or not tag.string.strip():
if not tag.has_attr('id'):
msg = "If <pre> is empty then it needs to have an id."
raise_desc(ValueError, msg, tag=describe_tag(tag))
# load it
tag_id = tag['id'].encode('utf-8')
if '.' in tag_id:
i = tag_id.index('.')
libname, name = tag_id[:i], tag_id[i + 1:]
use_library = library.load_library(libname)
else:
name = tag_id
use_library = library
basename = '%s.%s' % (name, extension)
data = use_library._get_file_data(basename)
source_code = data['data']
else:
source_code = get_source_code(tag)
# prettify.
# remove spurious indentation
source_code = source_code.strip()
do_apply_suggestions = (not tag.has_attr('noprettify')
and not tag.has_attr('np'))
# then apply suggestions
try:
if do_apply_suggestions:
x = parse_wrap(parse_expr, source_code)[0]
xr = parse_ndp_refine(x, Context())
suggestions = get_suggestions(xr)
source_code = apply_suggestions(
source_code, suggestions)
except DPSyntaxError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
continue
# we don't want the browser to choose different tab size
# source_code = source_code.replace('\t', ' ' * 4)
# we are not using it
_realpath = realpath
context = Context()
def postprocess(x):
if refine is not None:
return refine(x, context=context)
else:
return x
# print('rendering source code %r' % source_code)
html = ast_to_html(source_code,
parse_expr=parse_expr,
add_line_gutter=False,
postprocess=postprocess)
for w in context.warnings:
if w.where is not None:
from mcdp_web.editor_fancy.app_editor_fancy_generic import html_mark
html = html_mark(html, w.where, "language_warning")
frag2 = BeautifulSoup(html, 'lxml', from_encoding='utf-8')
if use_pre:
rendered = Tag(name='div', attrs={'class': 'rendered'})
pre = frag2.pre
pre.extract()
rendered.append(pre)
if not rendered.has_attr('class'):
rendered['class'] = ""
if tag.has_attr('label'):
text = tag['label']
tag_label = Tag(name='span')
add_class(tag_label, 'label')
add_class(tag_label, 'label_inside')
tag_label.append(NavigableString(text))
pre.insert(0, tag_label)
tag_label_outside = Tag(name='span')
add_class(tag_label_outside, 'label')
add_class(tag_label_outside, 'label_outside')
tag_label_outside.append(NavigableString(text))
rendered.insert(0, tag_label_outside)
max_len = max_len_of_pre_html(html)
if tag.has_attr('label'):
add_class(rendered, 'has_label')
max_len = max(max_len, len(tag['label']) + 6)
style = ''
else:
# using <code>
rendered = frag2.pre.code
rendered.extract()
if not rendered.has_attr('class'):
rendered['class'] = ""
style = ''
if tag.has_attr('style'):
style = style + tag['style']
if style:
rendered['style'] = style
if tag.has_attr('class'):
add_class(rendered, tag['class'])
if tag.has_attr('id'):
rendered['id'] = tag['id']
if use_pre:
if generate_pdf:
pdf = get_ast_as_pdf(source_code, parse_expr)
if tag.has_attr('id'):
basename = tag['id']
else:
hashcode = hashlib.sha224(
source_code).hexdigest()[-8:]
basename = 'code-%s' % hashcode
docname = os.path.splitext(
os.path.basename(realpath))[0]
download = docname + '.' + basename + '.source_code.pdf'
a = create_a_to_data(download=download,
data_format='pdf',
data=pdf)
a['class'] = 'pdf_data'
a.append(NavigableString(download))
div = Tag(name='div')
div.append(rendered)
div.append(a)
tag.replaceWith(div)
else:
tag.replaceWith(rendered)
else:
tag.replaceWith(rendered)
except DPSyntaxError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
# note_error(tag, e)
if tag.string is None:
tag.string = "`%s" % tag['id']
continue
except DPSemanticError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
# note_error(tag, e)
if tag.string is None:
tag.string = "`%s" % tag['id']
continue
except DPInternalError as ex:
msg = 'Error while interpreting the code:\n\n'
msg += indent(source_code, ' | ')
raise_wrapped(DPInternalError, ex, msg, exc=sys.exc_info())
def make_figure_from_figureid_attr(soup):
"""
Makes a figure:
<e figure-id='fig:ure' figure-caption='ciao'/>
<figure id="fig:ure">
<e figure-id='fig:ure' figure-caption='ciao'/>
<figcaption>ciao</figcaption>
</figure>
Makes a table:
<e figure-id='tab:ure' figure-caption='ciao'/>
becomes
figure-id
figure-class
"""
from mcdp_docs.highlight import add_class
for towrap in soup.select('[figure-id]'):
ID = towrap['figure-id']
parent = towrap.parent
fig = Tag(name='figure')
fig['id'] = ID
caption_below = True
if ID.startswith('fig:'):
add_class(fig, 'figure')
elif ID.startswith('subfig:'):
add_class(fig, 'subfloat')
elif ID.startswith('tab:'):
add_class(fig, 'table')
caption_below = False
elif ID.startswith('code:'):
add_class(fig, 'code')
pass
else:
msg = 'The ID %r should start with fig: or tab: or code:' % ID
raise_desc(ValueError, msg, tag=describe_tag(towrap))
if 'caption-left' in towrap.attrs.get('figure-class', ''):
caption_below = False
external_caption_id = '%s:caption' % ID
external_caption = soup.find(id=external_caption_id)
if external_caption is None:
external_caption = towrap.find(name='figcaption')
if external_caption is not None:
# print('using external caption %s' % str(external_caption))
external_caption.extract()
if external_caption.name != 'figcaption':
logger.error('Element %s#%r should have name figcaption.' %
(external_caption.name, external_caption_id))
external_caption.name = 'figcaption'
figcaption = external_caption
if towrap.has_attr('figure-caption'):
msg = 'Already using external caption for %s' % ID
raise_desc(ValueError, msg, describe_tag(towrap))
else:
# print('could not find external caption %s' % external_caption_id)
if towrap.has_attr('figure-caption'):
caption = towrap['figure-caption']
else:
caption = ''
figcaption = Tag(name='figcaption')
figcaption.append(NavigableString(caption))
outside = Tag(name='div')
outside['id'] = ID + '-wrap'
if towrap.has_attr('figure-style'):
outside['style'] = towrap['figure-style']
if towrap.has_attr('figure-class'):
for k in towrap['figure-class'].split(' '):
add_class(towrap, k)
add_class(outside, k )
i = parent.index(towrap)
towrap.extract()
figcontent = Tag(name='div', attrs={'class':'figcontent'})
figcontent.append(towrap)
fig.append(figcontent)
if caption_below:
fig.append(figcaption)
else:
fig.insert(0, figcaption)
add_class(outside, 'generated-figure-wrap')
add_class(fig, 'generated-figure')
outside.append(fig)
parent.insert(i, outside)