def get_xml(self):
xml = u'<?xml version="1.0" encoding="UTF-8" ?>\n'
root = Tag(name=self.task)
text = Tag(name='TEXT')
text.append(CData(self.text()))
tags = self.TAGS
tokens = (BS(
self.tokenizer.get_tokenized_as_xml().encode('utf-8'),
'xml'
)).TOKENS
elements = [u'\n', text, u'\n', tags, u'\n', tokens, u'\n']
for element in elements:
if element: # if missing tags, system will crash
root.append(element)
xml += unicode(root)
return xml
def setProperty(self,name,value):
if type(name)!=str or type(value)!=str:
print_error('key and value must be str')
return False
if not verify(name,value):
return False
ele=self.soup.find(name=NAME,text=name)
if ele==None:
#print "create it"
p=Tag(name='property')
n=Tag(name=NAME)
n.string=name
v=Tag(name=VALUE)
v.string=value
p.append(n)
p.append(v)
configuration_tag=self.soup.find('configuration')
configuration_tag.append(p)
else:
ele.parent.find(VALUE).string=value
#print "set succed it"
return True
def display_file(element, defaults, raise_errors):
assert element.name == 'display-file'
assert 'src' in element.attrs
src = element.attrs['src']
assert src.startswith('github:')
ref = parse_github_file_ref(src)
try:
ref = resolve_reference(ref, defaults=defaults)
except CouldNotResolveRef as e:
msg = 'Could not resolve reference %r' % src
if raise_errors:
raise_wrapped(DPSemanticError, e, msg, compact=True)
else:
msg += '\n\n' + indent(str(e), '> ')
note_error2(element, 'reference error', msg)
return
lines = ref.contents.split('\n')
a = ref.from_line if ref.from_line is not None else 0
b = ref.to_line if ref.to_line is not None else len(lines) - 1
portion = lines[a:b + 1]
contents = "\n".join(portion)
div = Tag(name='div')
base = os.path.basename(ref.path)
short = base + '-%d-%d' % (a, b)
div.attrs['figure-id'] = 'code:%s' % short
figcaption = Tag(name='figcaption')
t = Tag(name='code')
t.append(base)
a = Tag(name='a')
a.append(t)
a.attrs['href'] = ref.url
figcaption.append(a)
div.append(figcaption)
pre = Tag(name='pre')
code = Tag(name='code')
pre.append(code)
code.append(contents)
div.append(pre)
element.replace_with(div)
def td(x):
t = Tag(name="td")
if x is not None:
t.append(x)
return t
def make_section(_i, id_log, log, url_to_resource):
# id_log = id_log.decode('utf-8', 'ignore')
d = Tag(name="div")
classes = ["log-details"]
h = Tag(name="h2")
h.append(id_log)
d.append(h)
d.attrs["id"] = id_log
rel = get_small_video2(log, url_to_resource)
if rel:
video = video_for_source(rel)
d.append(video)
else:
pass
c = Tag(name="pre")
s = [
f"Vehicle: {log.vehicle}", f"Date: {log.date}",
f"Length: {log.length:.1f} s"
]
c.append("\n".join(s))
d.append(c)
rel = get_large_video2(log, url_to_resource)
if rel:
a = Tag(name="a")
a.attrs["href"] = rel
a.append("Watch video")
p = Tag(name="p")
p.append(a)
d.append(a)
else:
msg = "No video found for this log."
p = Tag(name="p")
p.append(msg)
d.append(p)
p = Tag(name="p")
n = append_urls(id_log, log, p, url_to_resource)
if n == 0:
msg = "No URL found for this log."
p = Tag(name="p")
p.append(msg)
d.append(p)
rel = get_thumbnails(log, url_to_resource)
if rel:
img = Tag(name="img")
img.attrs["class"] = "thumbnail"
img.attrs["src"] = rel
d.append(img)
else:
msg = "No thumbnail found for this log."
p = Tag(name="p")
p.append(msg)
d.append(p)
if not log.valid:
classes.append("invalid")
else:
classes.append("valid")
d.attrs["class"] = " ".join(classes)
return d
def go(soup, towrap, ID, figure_class, res, location):
from mcdp_docs.highlight import add_class
parent = towrap.parent
fig = Tag(name='figure')
fig['id'] = ID
caption_below = True
if ID.startswith('fig:'):
add_class(fig, 'figure')
elif ID.startswith('subfig:'):
add_class(fig, 'subfloat')
elif ID.startswith('tab:'):
add_class(fig, 'table')
caption_below = False
elif ID.startswith('code:'):
add_class(fig, 'code')
pass
else:
msg = 'The ID %r should start with fig: or tab: or code:' % ID
res.note_error(msg,
locations=HTMLIDLocation.for_element(towrap, location))
return
if 'caption-left' in figure_class:
caption_below = False
external_caption_id = '%s:caption' % ID
external_caption = soup.find(id=external_caption_id)
if external_caption is None:
external_caption = towrap.find(name='figcaption')
if external_caption is not None:
# print('using external caption %s' % str(external_caption))
external_caption.extract()
if external_caption.name != 'figcaption':
logger.error('Element %s#%r should have name figcaption.' %
(external_caption.name, external_caption_id))
external_caption.name = 'figcaption'
figcaption = external_caption
if towrap.has_attr('figure-caption'):
msg = 'Already using external caption for %s' % ID
res.note_error(msg,
location=HTMLIDLocation.for_element(
towrap, location))
return
else:
# print('could not find external caption %s' % external_caption_id)
if towrap.has_attr('figure-caption'):
caption = towrap['figure-caption']
else:
caption = ''
figcaption = Tag(name='figcaption')
figcaption.append(NavigableString(caption))
outside = Tag(name='div')
outside['id'] = ID + '-wrap'
if towrap.has_attr('figure-style'):
outside['style'] = towrap['figure-style']
for k in figure_class:
# logger.debug('figure-class: %s' % k)
add_class(towrap, k)
## XXX but not to figure itself?
add_class(fig, k)
add_class(outside, k)
i = parent.index(towrap)
towrap.extract()
figcontent = Tag(name='div', attrs={'class': 'figcontent'})
if towrap.name == 'figure':
towrap.name = 'div'
add_class(towrap, 'figure-conv-to-div')
figcontent.append(towrap)
# <div style='clear: both;'></div> <!-- for floating stuff-->
# Not 100% where it should go
breaking_div = Tag(name='div')
breaking_div.attrs['style'] = 'clear: both'
figcontent.append(breaking_div)
fig.append(figcontent)
if caption_below:
fig.append(figcaption)
else:
fig.insert(0, figcaption)
add_class(outside, 'generated-figure-wrap')
add_class(fig, 'generated-figure')
outside.append(fig)
parent.insert(i, outside)
def make_tag(tag0, klass, data, ndp=None, template=None, poset=None):
svg = data['svg']
tag_svg = BeautifulSoup(svg, 'lxml', from_encoding='utf-8').svg
assert tag_svg.name == 'svg'
if tag_svg.has_attr('width'):
ws = tag_svg['width']
hs = tag_svg['height']
assert 'pt' in ws
w = float(ws.replace('pt', ''))
h = float(hs.replace('pt', ''))
scale = MCDPConstants.scale_svg
w2 = w * scale
h2 = h * scale
tag_svg['width'] = w2
tag_svg['height'] = h2
tag_svg['rescaled'] = 'Rescaled from %s %s, scale = %s' % (ws, hs,
scale)
else:
print('no width in SVG tag: %s' % tag_svg)
tag_svg['class'] = klass
if tag0.has_attr('style'):
tag_svg['style'] = tag0['style']
if tag0.has_attr('id'):
tag_svg['id'] = tag0['id']
if generate_pdf:
pdf0 = data['pdf']
pdf = crop_pdf(pdf0, margins=0)
div = Tag(name='div')
att = MCDPConstants.ATTR_LOAD_NAME
if tag0.has_attr('id'):
basename = tag0['id']
elif ndp is not None and hasattr(ndp, att):
basename = getattr(ndp, att)
elif template is not None and hasattr(template, att):
basename = getattr(template, att)
elif poset is not None and hasattr(poset, att):
basename = getattr(poset, att)
else:
hashcode = hashlib.sha224(tag0.string).hexdigest()[-8:]
basename = 'code-%s' % hashcode
docname = os.path.splitext(os.path.basename(realpath))[0]
download = docname + "." + basename + "." + klass + '.pdf'
a = create_a_to_data(download=download,
data_format='pdf',
data=pdf)
a['class'] = 'pdf_data'
a.append(NavigableString(download))
div.append(tag_svg)
div.append(a)
return div
else:
return tag_svg
def make(self, context): # @UnusedVariable
t = Tag(name='div')
t.attrs['id'] = 'toc'
t.append('TOC placeholder')
return [t]
def visualize(summary):
hostnames = summary.get_sorted_hostnames()
sorted_test_names = summary.get_sorted_test_names()
body = Tag(name='body')
table = Tag(name='table')
tr = Tag(name='tr')
td = Tag(name='td')
td.append('hostname')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
td.attrs['class'] = 'hostname'
td.append(hostname)
tr.append(td)
table.append(tr)
tr = Tag(name='tr')
td = Tag(name='td')
td.append('type')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
stype = summary.get_host_type(hostname)
td.attrs['class'] = stype
td.append(stype)
tr.append(td)
table.append(tr)
tr = Tag(name='tr')
td = Tag(name='td')
td.append('owner')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
td.attrs['class'] = 'owner'
owner = summary.get_owner(hostname) or '(no owner)'
td.append(owner)
tr.append(td)
table.append(tr)
for test_name in sorted_test_names:
tr = Tag(name='tr')
td = Tag(name='td')
td.attrs['class'] = 'test_name'
# vis = test_name.replace(' : ', )
td.append(test_name)
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
s = Tag(name='span')
d = summary.get_data(test_name=test_name, hostname=hostname)
if d is None:
td.attrs['class'] = 'n-a'
td.append('-')
else:
td.attrs['class'] = d['status']
out = " ||| ".join([str(d['out_short']), str(d['out_long'])])
td.attrs['title'] = out
upload_date = d['upload_event_date']
elapsedTime = datetime.datetime.now() - upload_date
when = duration_compact(elapsedTime.total_seconds())
# td.append(d['status'])
# td.append(Tag(name='br'))
# td.append('(%s)' % when)
#
s.append(when)
td.append(s)
tr.append(td)
table.append(tr)
css = """
td.passed {
background-color: green;
color: white;
}
td.skipped {
background-color: yellow;
color: black;
}
td.failed {
background-color: red;
color: white;
}
td.hostname {
font-family: monospace;
}
td.test_name {
width: 30em;
max-width: 30em;
}
body {
font-size: 8pt;
}
td {
overflow-wrap: break-word;
min-width: 4em;
max-width: 4em;
padding-bottom: 3px;
}
td {
border-bottom: solid 1px black;
}
"""
style = Tag(name='style')
style.append(css)
body.append(style)
body.append(table)
return body
def make_figure_from_figureid_attr(soup):
"""
Makes a figure:
<e figure-id='fig:ure' figure-caption='ciao'/>
<figure id="fig:ure">
<e figure-id='fig:ure' figure-caption='ciao'/>
<figcaption>ciao</figcaption>
</figure>
Makes a table:
<e figure-id='tab:ure' figure-caption='ciao'/>
becomes
figure-id
figure-class
"""
from mcdp_docs.highlight import add_class
for towrap in soup.select('[figure-id]'):
ID = towrap['figure-id']
parent = towrap.parent
fig = Tag(name='figure')
fig['id'] = ID
caption_below = True
if ID.startswith('fig:'):
add_class(fig, 'figure')
elif ID.startswith('subfig:'):
add_class(fig, 'subfloat')
elif ID.startswith('tab:'):
add_class(fig, 'table')
caption_below = False
elif ID.startswith('code:'):
add_class(fig, 'code')
pass
else:
msg = 'The ID %r should start with fig: or tab: or code:' % ID
raise_desc(ValueError, msg, tag=describe_tag(towrap))
if 'caption-left' in towrap.attrs.get('figure-class', ''):
caption_below = False
external_caption_id = '%s:caption' % ID
external_caption = soup.find(id=external_caption_id)
if external_caption is None:
external_caption = towrap.find(name='figcaption')
if external_caption is not None:
# print('using external caption %s' % str(external_caption))
external_caption.extract()
if external_caption.name != 'figcaption':
logger.error('Element %s#%r should have name figcaption.' %
(external_caption.name, external_caption_id))
external_caption.name = 'figcaption'
figcaption = external_caption
if towrap.has_attr('figure-caption'):
msg = 'Already using external caption for %s' % ID
raise_desc(ValueError, msg, describe_tag(towrap))
else:
# print('could not find external caption %s' % external_caption_id)
if towrap.has_attr('figure-caption'):
caption = towrap['figure-caption']
else:
caption = ''
figcaption = Tag(name='figcaption')
figcaption.append(NavigableString(caption))
outside = Tag(name='div')
outside['id'] = ID + '-wrap'
if towrap.has_attr('figure-style'):
outside['style'] = towrap['figure-style']
if towrap.has_attr('figure-class'):
for k in towrap['figure-class'].split(' '):
add_class(towrap, k)
add_class(outside, k )
i = parent.index(towrap)
towrap.extract()
figcontent = Tag(name='div', attrs={'class':'figcontent'})
figcontent.append(towrap)
fig.append(figcontent)
if caption_below:
fig.append(figcaption)
else:
fig.insert(0, figcaption)
add_class(outside, 'generated-figure-wrap')
add_class(fig, 'generated-figure')
outside.append(fig)
parent.insert(i, outside)
def create_link_base(id2filename):
''' Returns a Tag <html> containing the page that is responsible to translate links '''
html = Tag(name='html')
head = Tag(name='head')
html.append(head)
body = Tag(name='body')
html.append(body)
s = create_link_base_js(id2filename)
script = Tag(name='script')
script.append(s)
body.append(script)
script = Tag(name='script')
script.append(jump_script)
body.append(script)
# pre = Tag(name='pre')
# pre.append(str(id2filename))
# body.append(pre)
return html
def add_prev_next_links(filename2contents, only_for=None):
new_one = OrderedDict()
for filename, contents in list(filename2contents.items()):
if only_for and not filename in only_for: continue
id_prev = contents.attrs[ATTR_PREV]
a_prev = Tag(name='a')
a_prev.attrs['href'] = '#' + str(id_prev)
a_prev.attrs['class'] = 'link_prev'
a_prev.append('prev')
id_next = contents.attrs[ATTR_NEXT]
a_next = Tag(name='a')
a_next.attrs['href'] = '#' + str(id_next)
a_next.attrs['class'] = 'link_next'
a_next.append('next')
S = Tag(name='div')
S.attrs['class'] = ['super']
nav1 = Tag(name='div')
add_class(nav1, 'navigation')
if id_prev:
nav1.append(a_prev.__copy__())
if id_next:
nav1.append(a_next.__copy__())
spacer = Tag(name='div')
spacer.attrs['style'] = 'clear:both'
nav1.append(spacer)
add_class(contents, 'main-section-for-page')
contents2 = contents
S.append(contents2)
from .source_info_imp import get_main_header
actual_id = get_main_header(contents2)
if False: # just checking
e = contents2.find(id=actual_id)
if e is not None:
pass
else:
logger.error('not found %r' % actual_id)
S.attrs['id'] = actual_id
contents2.insert(0, nav1.__copy__())
contents2.append(nav1.__copy__())
new_one[filename] = S
return new_one
def mark_console_pres_highlight(soup):
for code in soup.select('pre code'):
pre = code.parent
if code.string is None:
continue
s0 = code.string
from HTMLParser import HTMLParser
h = HTMLParser()
s = h.unescape(s0)
if s != s0:
# print('decoded %r -> %r' % (s0, s))
pass
beg = s.strip()
# is it a console line?
ct = is_console_line(beg)
if ct is None:
continue
add_class(pre, 'console')
# add class "on-hostname"
if ct.hostname is not None:
cn = 'on-%s' % str(ct.hostname)
add_class(pre, cn)
code.string = ''
lines = s.split('\n')
def is_program(x, l):
if x == 'git' and 'apt' in l:
return False
return x in programs
for j, line in enumerate(lines):
tokens = line.split(' ')
for i, token in enumerate(tokens):
previous_is_sudo_or_dollar = i >= 1 and tokens[i - 1] in ['$', 'sudo']
if token in ['$', 'DOLLAR']:
# add <span class=console_sign>$</span>
e = Tag(name='span')
e['class'] = 'console_sign'
e.string = '$'
code.append(e)
elif i == 0 and token == ct.hostname:
# it's the hostname
e = Tag(name='span')
e['class'] = 'hostname'
e.string = token
code.append(e)
elif is_program(token, line) and previous_is_sudo_or_dollar:
e = Tag(name='span')
e['class'] = '%s program' % token
e.string = token
code.append(e)
elif token in program_commands:
e = Tag(name='span')
e['class'] = '%s program_command' % token
e.string = token
code.append(e)
elif token and token[0] == '-':
e = Tag(name='span')
e['class'] = 'program_option'
e.string = token
code.append(e)
else:
code.append(NavigableString(token))
is_last = i == len(tokens) - 1
if not is_last:
before = '![' in ' '.join(tokens[:i + 1])
if not before:
# XXX: this is a bug
space = Tag(name='span')
space.append(' ')
space['class'] = 'space'
code.append(space)
else:
code.append(' ')
is_last_line = j == len(lines) - 1
if not is_last_line:
code.append(NavigableString('\n'))
def f(s):
td_ = Tag(name='td')
td_.append(s)
tr.append(td_)
def h(s):
td_ = Tag(name='td')
td_.append(s)
th.append(td_)
def get_build_table(builds):
table = Tag(name='table')
th = Tag(name='tr')
def h(s):
td_ = Tag(name='td')
td_.append(s)
th.append(td_)
h('build')
h('commit')
h('branch')
h('status')
h('')
h('duration')
h('results')
h('author')
h('commit text')
table.append(th)
table.attrs['id'] = 'builds'
for build in builds.values():
tr = Tag(name='tr')
def f(s):
td_ = Tag(name='td')
td_.append(s)
tr.append(td_)
table.append(tr)
outcome = build.get_outcome()
tr.attrs['class'] = outcome
a = Tag(name='a')
a.attrs['class'] = 'build_url'
a.attrs['href'] = build.get_build_url()
a.append('#%s' % build.get_build_num())
f(a)
a = Tag(name='a')
a.attrs['class'] = 'commit_url'
a.attrs['href'] = build.get_commit_url()
a.append(build.get_commit()[-7:])
f(a)
f(build.get_branch())
f(build.get_outcome_string())
stop_time = build.get_stop_time()
start_time = build.get_start_time()
duration_human = build.get_duration_human()
if stop_time is not None:
f(date_tag(stop_time))
else:
td = Tag(name='span')
if start_time is not None:
td.append('started ')
td.append(date_tag(start_time))
f(td)
if duration_human is not None:
f(duration_human)
else:
f('')
links = Tag(name='span')
links.attrs['class'] = 'links'
if build.artefacts:
links.append(get_links(build))
f(links)
author = Tag(name='span')
avatar_url = build.get_avatar_url()
author_name = build.get_author_name()
if avatar_url is not None:
img = Tag(name='img')
img.attrs['class'] = 'avatar'
img.attrs['src'] = avatar_url
img(a)
author.append(img)
author.append(author_name)
f(author)
f(build.get_subject())
return table
def make_sections(body,
is_marker,
preserve=lambda _: False,
element_name='section',
copy=True,
attrs={}):
sections = []
def make_new():
x = Tag(name=element_name)
for k, v in attrs.items():
x.attrs[k] = v
return x
current_section = make_new()
current_section['id'] = 'before-any-match-of-%s' % is_marker.__name__
current_section['class'] = 'without-header-inside'
# sections.append(current_section)
for x in body.contents:
if is_marker(x):
#print('starting %s' % str(x))
if contains_something_else_than_space(current_section):
sections.append(current_section)
current_section = make_new()
current_section['id'] = x.attrs.get(
'id', 'unnamed-h1') + ':' + element_name
logger.debug('marker %s' % current_section['id'])
current_section['class'] = x.attrs.get('class', '')
#print('%s/section %s %s' % (is_marker.__name__, x.attrs.get('id','unnamed'), current_section['id']))
current_section.append(x.__copy__())
current_section['class'] = 'with-header-inside'
elif preserve(x):
if contains_something_else_than_space(current_section):
sections.append(current_section)
#current_section['id'] = x.attrs.get('id', 'unnamed-h1') + ':' + element_name
#print('%s/preserve %s' % (preserve.__name__, current_section['id']))
sections.append(x.__copy__())
current_section = make_new()
current_section.attrs['comment'] = "Triggered by %r" % x
else:
#x2 = x.__copy__() if copy else x
x2 = x.__copy__() if copy else x.extract()
current_section.append(x2)
if contains_something_else_than_space(current_section):
sections.append(current_section) # XXX
new_body = Tag(name=body.name)
# if len(sections) < 3:
# msg = 'Only %d sections found (%s).' % (len(sections), is_marker.__name__)
# raise ValueError(msg)
logger.info('make_sections: %s found using marker %s' %
(len(sections), is_marker.__name__))
for i, s in enumerate(sections):
if add_debug_comments:
new_body.append('\n')
new_body.append(
Comment('Start of %s section %d/%d' %
(is_marker.__name__, i, len(sections))))
new_body.append('\n')
new_body.append(s)
new_body.append('\n')
if add_debug_comments:
new_body.append(
Comment('End of %s section %d/%d' %
(is_marker.__name__, i, len(sections))))
new_body.append('\n')
return new_body
def get_minimal_document(body_contents,
title=None,
add_markdown_css=False,
add_manual_css=False,
stylesheet=None,
extra_css=None):
""" Creates the minimal html document with MCDPL css.
add_markdown_css: language + markdown
add_manual_css: language + markdown + (manual*)
extra_css = additional CSS contents
"""
check_html_fragment(body_contents)
soup = bs("")
assert soup.name == 'fragment'
if title is None:
title = ''
html = Tag(name='html')
head = Tag(name='head')
body = Tag(name='body')
head.append(
Tag(name='meta',
attrs={
'http-equiv': "Content-Type",
'content': "application/xhtml+xml; charset=utf-8"
}))
if stylesheet is None:
stylesheet = 'v_mcdp_render_default'
if add_markdown_css or add_manual_css:
link = Tag(name='link')
link['rel'] = 'stylesheet'
link['type'] = 'text/css'
link['href'] = get_css_filename('compiled/%s' % stylesheet)
head.append(link)
tag_title = Tag(name='title')
tag_title.append(NavigableString(title))
head.append(tag_title)
parsed = bs(body_contents)
assert parsed.name == 'fragment'
parsed.name = 'div'
body.append(parsed)
html.append(head)
html.append(body)
soup.append(html)
if extra_css is not None:
add_extra_css(soup, extra_css)
s = to_html_stripping_fragment_document(soup)
assert not 'DOCTYPE' in s
# s = html.prettify() # no: it removes empty text nodes
# ns="""<?xml version="1.0" encoding="utf-8" ?>"""
ns = """<!DOCTYPE html PUBLIC
"-//W3C//DTD XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN"
"http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd">"""
res = ns + '\n' + s
# if add_manual_css and MCDPConstants.manual_link_css_instead_of_including:
# assert 'manual.css' in res, res
res = res.replace('<div><!DOCTYPE html>', '<div>')
return res
def make(self, context): # @UnusedVariable
""" Returns a BS4 element """
t = Tag(name='span')
t.append('Not implemented')
return [t]
def make_page(contents, head0, add_toc):
""" Returns html (Beautiful Soup document) """
html = Tag(name='html')
head = head0.__copy__()
html.append(head)
body = Tag(name='body')
with timeit('make_page() / copy toc'):
if add_toc:
tocdiv = Tag(name='div')
tocdiv.attrs['id'] = 'tocdiv'
# toc = main_toc
# toc.extract()
# del toc.attrs['id']
tocdiv.append(add_toc)
section_name = get_first_header_title(contents)
if section_name is not None:
title2 = bs(section_name)
title2.name = 'title'
title = head.find('title')
if title is None:
# msg = 'Cannot find the "title" element'
# msg += '\n' + indent(str(head)[:500], 'head')
# raise Exception(msg)
head.append(title2)
else:
title.replace_with(title2)
body.append(tocdiv)
not_toc = Tag(name='div')
not_toc.attrs['id'] = 'not-toc'
not_toc.append(contents)
body.append(not_toc)
html.append(body)
# delete the original one
main_toc = contents.find(id='main_toc')
if main_toc is not None:
main_toc.extract()
return html
def go(selector, parse_expr, extension, use_pre=True, refine=None):
for tag in soup.select(selector):
source_code = '<unset>' # XXX
try:
if tag.string is None: # or not tag.string.strip():
if not tag.has_attr('id'):
msg = "If <pre> is empty then it needs to have an id."
raise_desc(ValueError, msg, tag=describe_tag(tag))
# load it
tag_id = tag['id'].encode('utf-8')
if '.' in tag_id:
i = tag_id.index('.')
libname, name = tag_id[:i], tag_id[i + 1:]
use_library = library.load_library(libname)
else:
name = tag_id
use_library = library
basename = '%s.%s' % (name, extension)
data = use_library._get_file_data(basename)
source_code = data['data']
else:
source_code = get_source_code(tag)
# prettify.
# remove spurious indentation
source_code = source_code.strip()
do_apply_suggestions = (not tag.has_attr('noprettify')
and not tag.has_attr('np'))
# then apply suggestions
try:
if do_apply_suggestions:
x = parse_wrap(parse_expr, source_code)[0]
xr = parse_ndp_refine(x, Context())
suggestions = get_suggestions(xr)
source_code = apply_suggestions(
source_code, suggestions)
except DPSyntaxError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
continue
# we don't want the browser to choose different tab size
# source_code = source_code.replace('\t', ' ' * 4)
# we are not using it
_realpath = realpath
context = Context()
def postprocess(x):
if refine is not None:
return refine(x, context=context)
else:
return x
# print('rendering source code %r' % source_code)
html = ast_to_html(source_code,
parse_expr=parse_expr,
add_line_gutter=False,
postprocess=postprocess)
for w in context.warnings:
if w.where is not None:
from mcdp_web.editor_fancy.app_editor_fancy_generic import html_mark
html = html_mark(html, w.where, "language_warning")
frag2 = BeautifulSoup(html, 'lxml', from_encoding='utf-8')
if use_pre:
rendered = Tag(name='div', attrs={'class': 'rendered'})
pre = frag2.pre
pre.extract()
rendered.append(pre)
if not rendered.has_attr('class'):
rendered['class'] = ""
if tag.has_attr('label'):
text = tag['label']
tag_label = Tag(name='span')
add_class(tag_label, 'label')
add_class(tag_label, 'label_inside')
tag_label.append(NavigableString(text))
pre.insert(0, tag_label)
tag_label_outside = Tag(name='span')
add_class(tag_label_outside, 'label')
add_class(tag_label_outside, 'label_outside')
tag_label_outside.append(NavigableString(text))
rendered.insert(0, tag_label_outside)
max_len = max_len_of_pre_html(html)
if tag.has_attr('label'):
add_class(rendered, 'has_label')
max_len = max(max_len, len(tag['label']) + 6)
style = ''
else:
# using <code>
rendered = frag2.pre.code
rendered.extract()
if not rendered.has_attr('class'):
rendered['class'] = ""
style = ''
if tag.has_attr('style'):
style = style + tag['style']
if style:
rendered['style'] = style
if tag.has_attr('class'):
add_class(rendered, tag['class'])
if tag.has_attr('id'):
rendered['id'] = tag['id']
if use_pre:
if generate_pdf:
pdf = get_ast_as_pdf(source_code, parse_expr)
if tag.has_attr('id'):
basename = tag['id']
else:
hashcode = hashlib.sha224(
source_code).hexdigest()[-8:]
basename = 'code-%s' % hashcode
docname = os.path.splitext(
os.path.basename(realpath))[0]
download = docname + '.' + basename + '.source_code.pdf'
a = create_a_to_data(download=download,
data_format='pdf',
data=pdf)
a['class'] = 'pdf_data'
a.append(NavigableString(download))
div = Tag(name='div')
div.append(rendered)
div.append(a)
tag.replaceWith(div)
else:
tag.replaceWith(rendered)
else:
tag.replaceWith(rendered)
except DPSyntaxError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
# note_error(tag, e)
if tag.string is None:
tag.string = "`%s" % tag['id']
continue
except DPSemanticError as e:
if raise_errors:
raise
else:
res.note_error(str(e), HTMLIDLocation.for_element(tag))
# note_error(tag, e)
if tag.string is None:
tag.string = "`%s" % tag['id']
continue
except DPInternalError as ex:
msg = 'Error while interpreting the code:\n\n'
msg += indent(source_code, ' | ')
raise_wrapped(DPInternalError, ex, msg, exc=sys.exc_info())
def html_list_of_notes(aug, tag, how_to_call_them, klass, header=None):
from mcdp_docs.elements_abbrevs import format_name
notes = aug.get_notes_by_tag(tag)
# print('%d notes for tag %s' % (len(notes), tag))
html = Tag(name='html')
head = Tag(name='head')
meta = Tag(name='meta')
meta.attrs['content'] = "text/html; charset=utf-8"
meta.attrs['http-equiv'] = "Content-Type"
head.append(meta)
head.append('\n')
html.append(head)
html.append('\n')
body = Tag(name='body')
if header is not None:
body.append(header)
body.append('\n')
html.append(body)
html.append('\n')
if not notes:
p = Tag(name='p')
p.append('There were no %s.' % how_to_call_them)
body.append(p)
body.append('\n')
else:
p = Tag(name='p')
p.append('There were %d %s.' % (len(notes), how_to_call_them))
body.append(p)
body.append('\n')
if klass == 'task': # XXX
sorted_notes = sort_by_dest(notes)
order = sorted(sorted_notes, key=lambda x: -len(sorted_notes[x]) if x else 1000)
table = Tag(name='table')
for who in order:
section_notes = sorted_notes[who]
tr = Tag(name='tr')
td = Tag(name='td')
if who is None:
a = Tag(name='a')
a.attrs['href'] = '#for:%s' % who
who = 'unassigned'
a.append(who)
td.append(a)
else:
a = Tag(name='a')
a.attrs['href'] = '#for:%s' % who
a.append("Tasks assigned to")
td.append(a)
td.append(' ')
td.append(format_name(who))
tr.append(td)
td = Tag(name='td')
td.append('%d tasks' % len(section_notes))
tr.append(td)
table.append(tr)
body.append(table)
for who, section_notes in sorted_notes.items():
sec = Tag(name='div')
sec.attrs['id'] = 'for:%s' % who
if who is None:
t = '%s unassigned tasks' % len(section_notes)
else:
t = '%d tasks assigned to %s' % (len(section_notes), who)
h = Tag(name='h3')
h.append(t)
sec.append(h)
for i, note in enumerate(section_notes):
div = note.as_html()
div.attrs['class'] = klass
sec.append(div)
sec.append('\n\n')
body.append(sec)
body.append('\n\n')
else:
for i, note in enumerate(notes):
div = note.as_html()
div.attrs['class'] = klass
body.append(div)
body.append('\n\n')
body.append(get_html_style())
return html
def compact_desc_tag(element):
t = Tag(name=element.name)
t.attrs.update(element.attrs)
t.append(' ... ')
return str(t)
def mark_in_html_notes(notes, soup, note_type, index_url, klasses):
from mcdp_docs.check_missing_links import get_id2element
id2element, duplicates = get_id2element(soup, 'id')
ids_ordered = list(id2element)
N = len(notes)
indices = [None for _ in range(len(notes))]
for i in range(len(notes)):
eid, note = notes[i]
if eid in id2element:
indices[i] = ids_ordered.index(eid)
else:
indices[i] = None
indices = sorted(range(N), key=lambda _: indices[_])
def idfor(x):
return 'note-%s-%d' % (note_type, x)
def linkto(x):
eid_, _ = notes[indices[x]]
return '#%s' % eid_
from mcdp_utils_xml import stag
# TODO: there is the case where there are multiple notes that refer to the same element..
for b, i in enumerate(indices):
eid, note = notes[i]
element = id2element.get(eid, None)
if element is None:
msg = 'Cannot find the element corresponding to ID %r' % eid
msg += '\n\nI cannot mark this note:'
msg += '\n\n' + indent(note, ' | ')
# raise Exception(msg) # FIXME
logger.error(msg)
else:
if element.name == 'code' and element.parent and element.parent.name == 'pre':
element = element.parent
note_html = note.as_html(inline=True)
inset = insert_inset(element, short=note_type, # XXX
long_error=note_html,
klasses=klasses)
inset.attrs['id'] = idfor(b)
summary = inset.find('summary')
summary.append('\n')
if b > 0:
a = Tag(name='a')
a.attrs['class'] = 'note-navigation'
a.append('previous ')
a.attrs['href'] = linkto(b - 1)
summary.insert(0, a)
if b < N - 1:
a = Tag(name='a')
a.attrs['class'] = 'note-navigation'
a.append(' next')
a.attrs['href'] = linkto(b + 1)
summary.append(a)
summary.append(' (%d of %s) ' % (b + 1, len(notes)))
summary.append('\n')
summary.append(stag('a', 'index', href=index_url))
summary.append('\n')
def col_macro_(e, ncols):
"""
Bug: For some reasone bd4 removes the whitespace I use for indentation.
"""
assert e.name == 'div'
assert e.has_attr('make-col%d' % ncols)
# print describe_tag(e)
children = list(e.children)
# remove strings from this
is_string = lambda x: isinstance(x, NavigableString)
strings = [_ for _ in children if is_string(_)]
children = [_ for _ in children if not is_string(_)]
if len(children) < ncols:
msg = ('Cannot create table with %r cols with only %d children' %
(ncols, len(children)))
raise_desc(ValueError, msg, tag=describe_tag(e))
for c in children:
c.extract()
for s in strings:
ss = str(s)
empty = not ss.strip()
if not empty:
msg = 'Found nonempty string %r between children.' % ss
raise_desc(ValueError, msg, tag=describe_tag(e))
# remove it
s.extract()
nchildren = len(children)
nrows = int(math.ceil(nchildren / float(ncols)))
parent = e.parent
original_position = parent.index(e)
e.extract()
table = e
e.name = 'table'
add_class(table, 'col%d' % ncols)
add_class(table, 'colN')
wrapper = Tag(name='div')
add_class(wrapper, 'col%d-wrap' % ncols)
add_class(wrapper, 'colN-wrap')
NL = '\n'
# S = '-' * 4
# XXX: change to above to see the problem with indentation
S = ' ' * 4
tbody = Tag(name='tbody')
for row in range(nrows):
tbody.append(NavigableString(NL))
tbody.append(NavigableString(S+S))
tr = Tag(name='tr')
tr.append(NavigableString(NL))
for col in range(ncols):
td = Tag(name='td')
i = col + row * ncols
if i < len(children):
child = children[i]
td.append(child)
else:
td.append(Comment('empty row %d col %d' % (row, col)))
tr.append(NavigableString(S+S+S))
tr.append(td)
tr.append(NavigableString(NL))
tr.append(S+S)
if row == 0 and ('labels-row1' in e.attrs.get('class', '')):
thead = Tag(name='thead')
thead.append(tr)
table.append(thead) # add in table, not tbody
else:
tbody.append(tr) # add in tbody
tbody.append(NavigableString(NL+S))
table.append(tbody)
wrapper.append(NavigableString(NL + S))
wrapper.append(table)
wrapper.append(NavigableString(NL))
parent.insert(original_position, wrapper)
def as_html(self, inline=False):
div = Tag(name='div')
for tag in self.tags:
s = Tag(name='span')
s.attrs['class'] = 'note-tag'
css = """
font-size: 70%;
font-family: arial;
background-color: #dee;
padding: 5px;
border-radius: 5px;
margin: 0.4em;
"""
s.attrs['style'] = css.replace('\n', ' ').strip()
s.append(tag)
div.append(s)
div.attrs['class'] = 'note'
container = Tag(name='div')
if self.msg is not None:
if isinstance(self.msg, Tag):
container.append(self.msg.__copy__())
else:
pre = Tag(name='pre')
pre.append(self.msg)
container.append(pre)
else:
p = Tag(name='p')
p.append("No message given.")
container.append(p)
div.append(container)
if self.locations:
if len(self.locations) > 1:
p = Tag(name='p')
p.append('These are the locations indicated:')
div.append(p)
dl = Tag(name='dl')
for k, v in self.locations.items():
dt = Tag(name='dt')
dt.append(k)
dl.append(dt)
dd = Tag(name='dd')
# setting inline - False
dd.append(v.as_html(inline=False))
dl.append(dd)
div.append(dl)
else:
# p = Tag(name='p')
# p.append("Location:")
# div.append(p)
location = list(self.locations.values())[0]
div.append(location.as_html(inline=inline))
else:
p = Tag(name='p')
p.append("(No locations provided)")
div.append(p)
from mcdp_utils_xml import stag
s = Tag(name='span')
s.append('Created by function ')
s.append(stag('code', self.created_function))
# s.append(br())
s.append(' in module ')
s.append(stag('code', self.created_module))
s.append('.')
p = Tag(name='p')
p.append(s)
div.append(s)
return div
def get_row(i, log, url_to_resource):
trh = Tag(name="tr")
tr = Tag(name="tr")
def td(x):
t = Tag(name="td")
if x is not None:
t.append(x)
return t
trh.append(td("index"))
tr.append(td(str(i)))
trh.append(td(""))
rel = get_small_video2(log, url_to_resource)
if rel:
video = video_for_source(rel)
tr.append(td(video))
else:
tr.append(td("-"))
# tr.append(td(log.log_name))
# trh.append(td('video'))
f = Tag(name="td")
rel = get_large_video2(log, url_to_resource)
if rel:
a = Tag(name="a")
a.attrs["href"] = rel
a.append("video")
f.append(a)
rel = get_thumbnails2(log, url_to_resource)
if rel:
# f.append(Tag(name='br'))
f.append(" ")
a = Tag(name="a")
a.attrs["href"] = rel
a.append("thumbnails")
f.append(a)
# n = append_urls(log, f)
# urls = [x for x in log.resources['bag']['urls'] if show_url(x)]
# for url in urls:
# f.append(' ')
# a = Tag(name='a')
# a.attrs['href'] = url
# a.append('bag')
# f.append(a)
trh.append(td("misc"))
tr.append(f)
#
# tr.append(td(a))
# else:
# tr.append(td(''))
trh.append(td("date"))
tr.append(td(log.date))
if log.length is not None:
l = f"{log.length:5.1f} s"
else:
l = "(none)"
trh.append(td("length"))
tr.append(td(l))
trh.append(td("vehicle"))
tr.append(td(log.vehicle))
if False:
if log.valid:
sr = "Yes."
else:
sr = log.error_if_invalid
trh.append(td("valid"))
tr.append(td(sr))
trh.append(td("ID"))
a = Tag(name="a")
a.append(log.log_name)
a.attrs["href"] = f"#{log.log_name}"
tr.append(td(a))
if not log.valid:
tr.attrs["class"] = ["invalid"]
else:
tr.attrs["class"] = ["valid"]
return trh, tr
def make_last_modified(files_contents, nmax=100):
res = AugmentedResult()
files_contents = [DocToJoin(*x) for x in files_contents]
files_contents = [_ for _ in files_contents if _.source_info]
files_contents = list(
sorted(files_contents,
key=lambda x: x.source_info.last_modified,
reverse=True))
r = Tag(name='fragment')
r.append('\n')
h = Tag(name='h1')
h.append('Last modified')
h.attrs['id'] = 'sec:last-modified'
r.append(h)
r.append('\n')
ul = Tag(name='ul')
ul.append('\n')
for d in files_contents[:nmax]:
li = Tag(name='li')
when = d.source_info.last_modified
when_s = time.strftime("%a, %b %d", when)
# %H:%M
li.append(when_s)
li.append(': ')
hid = get_main_header(bs(d.contents))
if hid is None:
what = "File %s" % d.docname
else:
what = Tag(name='a')
what.attrs['href'] = '#' + hid
what.attrs['class'] = MCDPManualConstants.CLASS_NUMBER_NAME
li.append(what)
li.append(' (')
name = d.source_info.author.name
li.append(name)
li.append(')')
ul.append(li)
ul.append('\n')
r.append(ul)
s = to_html_stripping_fragment(r)
# print s
res.set_result(s)
return res
def get_report(logs, url_to_resource, initial_screens: bool = True) -> str:
length = 0
vehicles = set()
for _, log in list(logs.items()):
length += log.length
vehicles.add(log.vehicle)
html = Tag(name="html")
body = Tag(name="body")
head = Tag(name="head")
link = Tag(name="link")
link.attrs["type"] = "text/css"
link.attrs["rel"] = "stylesheet"
link.attrs["href"] = "style.css"
title = Tag(name="title")
title.append("Duckietown Logs Database")
head.append(link)
html.append(head)
html.append(body)
h = Tag(name="h1")
h.append("Duckietown Logs Database")
body.append(h)
c = f"Showing {len(logs)} logs from {len(vehicles)} different Duckiebots, for a total length of " \
f"{length / 3600.0:.1f} hours."
p = Tag(name="p")
p.append(c)
body.append(p)
if initial_screens:
t = summary_table(logs, url_to_resource)
body.append(t)
body.append(html_table_from_table(logs, url_to_resource))
make_sections(body, logs, url_to_resource)
s = str(html)
return s
def visualize(summary):
hostnames = summary.get_sorted_hostnames()
sorted_test_names = summary.get_sorted_test_names()
body = Tag(name='body')
table = Tag(name='table')
tr = Tag(name='tr')
td = Tag(name='td')
td.append('hostname')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
td.attrs['class'] = 'hostname'
td.append(hostname)
tr.append(td)
table.append(tr)
tr = Tag(name='tr')
td = Tag(name='td')
td.append('type')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
stype = summary.get_host_type(hostname)
td.attrs['class'] = stype
td.append(cute_stype(stype))
tr.append(td)
table.append(tr)
# tr = Tag(name='tr')
# td = Tag(name='td')
# td.append('last heard')
# tr.append(td)
# for hostname in hostnames:
# td = Tag(name='td')
# date = summary.get_last_upload_date(hostname)
# if date is not None:
# date_s = date.isoformat()
# else:
# date_s = '?'
# td.attrs['class'] = 'date'
# td.append(date_s)
# tr.append(td)
# table.append(tr)
tr = Tag(name='tr')
td = Tag(name='td')
td.append('owner/username')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
td.attrs['class'] = 'owner'
owner = summary.get_owner(hostname) or '(no owner)'
td.append(owner)
tr.append(td)
table.append(tr)
tr = Tag(name='tr')
td = Tag(name='td')
td.append('location')
tr.append(td)
for hostname in hostnames:
td = Tag(name='td')
td.attrs['class'] = 'location'
country = summary.get_country(hostname) or ''
td.append(cute_country(country))
tr.append(td)
table.append(tr)
for test_name in sorted_test_names:
tr = Tag(name='tr')
td = Tag(name='td')
td.attrs['class']= 'test_name'
# vis = test_name.replace(' : ', )
td.append(test_name)
tr.append(td)
n_failed_or_invalid = 0
for hostname in hostnames:
d = summary.get_data(test_name=test_name, hostname=hostname)
if d is not None and d['status'] in [ChecksConstants.FAIL, ChecksConstants.ERROR]:
n_failed_or_invalid += 1
if n_failed_or_invalid == 0:
continue
if "dt_live_instagram_" in test_name or 'dt_augmented_reality_' in test_name:
continue
for hostname in hostnames:
td = Tag(name='td')
s = Tag(name='span')
d = summary.get_data(test_name=test_name, hostname=hostname)
if d is None:
td.attrs['class'] = 'n-a'
td.append('-')
else:
td.attrs['class'] = d['status']
out_short = d['out_short']
out_long = d['out_long']
#
# out_short = out_short.encode('utf8', 'ignore')
# out_long = out_long.encode('utf8', 'ignore')
# print out_short.__repr__()
# print out_long.__repr__()
out = u" ||| ".join([out_short, out_long])
td.attrs['title'] = out
upload_date = d['upload_event_date']
elapsedTime = datetime.datetime.now() - upload_date
when = duration_compact(elapsedTime.total_seconds())
# td.append(d['status'])
# td.append(Tag(name='br'))
# td.append('(%s)' % when)
#
s.append(when)
td.append(s)
tr.append(td)
table.append(tr)
css = """
td.passed {
background-color: green;
color: white;
}
td.invalid {
background-color: purple;
color: white;
}
td.skipped {
background-color: yellow;
color: black;
}
td.failed {
background-color: red;
color: white;
}
td.hostname {
font-family: monospace;
}
td.test_name {
width: 30em;
max-width: 30em;
}
body {
font-size: 8pt;
}
td {
overflow-wrap: break-word;
min-width: 4em;
max-width: 4em;
padding-bottom: 3px;
}
td {
border-bottom: solid 1px black;
}
"""
style = Tag(name='style')
style.append(css)
body.append(style)
body.append(table)
html = Tag(name='html')
head = Tag(name='head')
meta = Tag(name='meta')
meta.attrs['charset'] = 'UTF-8'
head.append(meta)
html.append(head)
html.append(body)
return html
def do_bib(soup, bibhere):
""" find used bibliography entries put them there """
used = []
unused = set()
for a in soup.find_all('a'):
href = a.attrs.get('href', '')
if href.startswith('#bib:'):
used.append(href[1:]) # no "#"
logger.debug('I found %d references, to these: %s' % (len(used), used))
# collect all the <cite>
id2cite = {}
for c in soup.find_all('cite'):
ID = c.attrs.get('id', None)
id2cite[ID] = c
if ID in used:
add_class(c, 'used')
else:
unused.add(ID)
add_class(c, 'unused')
# divide in found and not found
found = []
notfound = []
for ID in used:
if not ID in id2cite:
if not ID in notfound:
notfound.append(ID)
else:
found.append(ID)
# now create additional <cite> for the ones that are not found
for ID in notfound:
cite = Tag(name='cite')
s = 'Reference %s not found.' % ID
cite.append(NavigableString(s))
cite.attrs['class'] = ['errored', 'error'] # XXX
soup.append(cite)
id2cite[ID] = cite
# now number the cites
n = 1
id2number = {}
for ID in used:
if not ID in id2number:
id2number[ID] = n
n += 1
# now add the attributes for cross-referencing
for ID in used:
number = id2number[ID]
cite = id2cite[ID]
cite.attrs[LABEL_NAME] = '[%s]' % number
cite.attrs[LABEL_SELF] = '[%s]' % number
cite.attrs[LABEL_NUMBER] = number
cite.attrs[LABEL_WHAT] = 'Reference'
cite.attrs[LABEL_WHAT_NUMBER_NAME] = '[%s]' % number
cite.attrs[LABEL_WHAT_NUMBER] = '[%s]' % number
# now put the cites at the end of the document
for ID in used:
c = id2cite[ID]
# remove it from parent
c.extract()
# add to bibliography
bibhere.append(c)
s = ("Bib cites: %d\nBib used: %s\nfound: %s\nnot found: %s\nunused: %d" %
(len(id2cite), len(used), len(found), len(notfound), len(unused)))
logger.info(s)
def get_links_from_artefacts(artefacts, branch=None, build_num=None):
links = Tag(name='span')
links.attrs['class'] = 'links'
groups = sorted(set([art.group for art in artefacts]))
for j, g in enumerate(groups):
if j > 0:
links.append(br())
tag_g = Tag(name='span')
tag_g.attrs['class'] = 'group'
tag_g.append(' ')
tag_g.append(g)
tag_g.append(br())
links.append(tag_g)
links.append('[')
arts = [_ for _ in artefacts if _.group == g]
div = get_links2(arts, branch, build_num)
links.append(div)
links.append(']')
return links
