def get_xml(self): xml = u'<?xml version="1.0" encoding="UTF-8" ?>\n' root = Tag(name=self.task) text = Tag(name='TEXT') text.append(CData(self.text())) tags = self.TAGS tokens = (BS( self.tokenizer.get_tokenized_as_xml().encode('utf-8'), 'xml' )).TOKENS elements = [u'\n', text, u'\n', tags, u'\n', tokens, u'\n'] for element in elements: if element: # if missing tags, system will crash root.append(element) xml += unicode(root) return xml
def setProperty(self,name,value): if type(name)!=str or type(value)!=str: print_error('key and value must be str') return False if not verify(name,value): return False ele=self.soup.find(name=NAME,text=name) if ele==None: #print "create it" p=Tag(name='property') n=Tag(name=NAME) n.string=name v=Tag(name=VALUE) v.string=value p.append(n) p.append(v) configuration_tag=self.soup.find('configuration') configuration_tag.append(p) else: ele.parent.find(VALUE).string=value #print "set succed it" return True
def display_file(element, defaults, raise_errors): assert element.name == 'display-file' assert 'src' in element.attrs src = element.attrs['src'] assert src.startswith('github:') ref = parse_github_file_ref(src) try: ref = resolve_reference(ref, defaults=defaults) except CouldNotResolveRef as e: msg = 'Could not resolve reference %r' % src if raise_errors: raise_wrapped(DPSemanticError, e, msg, compact=True) else: msg += '\n\n' + indent(str(e), '> ') note_error2(element, 'reference error', msg) return lines = ref.contents.split('\n') a = ref.from_line if ref.from_line is not None else 0 b = ref.to_line if ref.to_line is not None else len(lines) - 1 portion = lines[a:b + 1] contents = "\n".join(portion) div = Tag(name='div') base = os.path.basename(ref.path) short = base + '-%d-%d' % (a, b) div.attrs['figure-id'] = 'code:%s' % short figcaption = Tag(name='figcaption') t = Tag(name='code') t.append(base) a = Tag(name='a') a.append(t) a.attrs['href'] = ref.url figcaption.append(a) div.append(figcaption) pre = Tag(name='pre') code = Tag(name='code') pre.append(code) code.append(contents) div.append(pre) element.replace_with(div)
def td(x): t = Tag(name="td") if x is not None: t.append(x) return t
def make_section(_i, id_log, log, url_to_resource): # id_log = id_log.decode('utf-8', 'ignore') d = Tag(name="div") classes = ["log-details"] h = Tag(name="h2") h.append(id_log) d.append(h) d.attrs["id"] = id_log rel = get_small_video2(log, url_to_resource) if rel: video = video_for_source(rel) d.append(video) else: pass c = Tag(name="pre") s = [ f"Vehicle: {log.vehicle}", f"Date: {log.date}", f"Length: {log.length:.1f} s" ] c.append("\n".join(s)) d.append(c) rel = get_large_video2(log, url_to_resource) if rel: a = Tag(name="a") a.attrs["href"] = rel a.append("Watch video") p = Tag(name="p") p.append(a) d.append(a) else: msg = "No video found for this log." p = Tag(name="p") p.append(msg) d.append(p) p = Tag(name="p") n = append_urls(id_log, log, p, url_to_resource) if n == 0: msg = "No URL found for this log." p = Tag(name="p") p.append(msg) d.append(p) rel = get_thumbnails(log, url_to_resource) if rel: img = Tag(name="img") img.attrs["class"] = "thumbnail" img.attrs["src"] = rel d.append(img) else: msg = "No thumbnail found for this log." p = Tag(name="p") p.append(msg) d.append(p) if not log.valid: classes.append("invalid") else: classes.append("valid") d.attrs["class"] = " ".join(classes) return d
def go(soup, towrap, ID, figure_class, res, location): from mcdp_docs.highlight import add_class parent = towrap.parent fig = Tag(name='figure') fig['id'] = ID caption_below = True if ID.startswith('fig:'): add_class(fig, 'figure') elif ID.startswith('subfig:'): add_class(fig, 'subfloat') elif ID.startswith('tab:'): add_class(fig, 'table') caption_below = False elif ID.startswith('code:'): add_class(fig, 'code') pass else: msg = 'The ID %r should start with fig: or tab: or code:' % ID res.note_error(msg, locations=HTMLIDLocation.for_element(towrap, location)) return if 'caption-left' in figure_class: caption_below = False external_caption_id = '%s:caption' % ID external_caption = soup.find(id=external_caption_id) if external_caption is None: external_caption = towrap.find(name='figcaption') if external_caption is not None: # print('using external caption %s' % str(external_caption)) external_caption.extract() if external_caption.name != 'figcaption': logger.error('Element %s#%r should have name figcaption.' % (external_caption.name, external_caption_id)) external_caption.name = 'figcaption' figcaption = external_caption if towrap.has_attr('figure-caption'): msg = 'Already using external caption for %s' % ID res.note_error(msg, location=HTMLIDLocation.for_element( towrap, location)) return else: # print('could not find external caption %s' % external_caption_id) if towrap.has_attr('figure-caption'): caption = towrap['figure-caption'] else: caption = '' figcaption = Tag(name='figcaption') figcaption.append(NavigableString(caption)) outside = Tag(name='div') outside['id'] = ID + '-wrap' if towrap.has_attr('figure-style'): outside['style'] = towrap['figure-style'] for k in figure_class: # logger.debug('figure-class: %s' % k) add_class(towrap, k) ## XXX but not to figure itself? add_class(fig, k) add_class(outside, k) i = parent.index(towrap) towrap.extract() figcontent = Tag(name='div', attrs={'class': 'figcontent'}) if towrap.name == 'figure': towrap.name = 'div' add_class(towrap, 'figure-conv-to-div') figcontent.append(towrap) # <div style='clear: both;'></div> <!-- for floating stuff--> # Not 100% where it should go breaking_div = Tag(name='div') breaking_div.attrs['style'] = 'clear: both' figcontent.append(breaking_div) fig.append(figcontent) if caption_below: fig.append(figcaption) else: fig.insert(0, figcaption) add_class(outside, 'generated-figure-wrap') add_class(fig, 'generated-figure') outside.append(fig) parent.insert(i, outside)
def make_tag(tag0, klass, data, ndp=None, template=None, poset=None): svg = data['svg'] tag_svg = BeautifulSoup(svg, 'lxml', from_encoding='utf-8').svg assert tag_svg.name == 'svg' if tag_svg.has_attr('width'): ws = tag_svg['width'] hs = tag_svg['height'] assert 'pt' in ws w = float(ws.replace('pt', '')) h = float(hs.replace('pt', '')) scale = MCDPConstants.scale_svg w2 = w * scale h2 = h * scale tag_svg['width'] = w2 tag_svg['height'] = h2 tag_svg['rescaled'] = 'Rescaled from %s %s, scale = %s' % (ws, hs, scale) else: print('no width in SVG tag: %s' % tag_svg) tag_svg['class'] = klass if tag0.has_attr('style'): tag_svg['style'] = tag0['style'] if tag0.has_attr('id'): tag_svg['id'] = tag0['id'] if generate_pdf: pdf0 = data['pdf'] pdf = crop_pdf(pdf0, margins=0) div = Tag(name='div') att = MCDPConstants.ATTR_LOAD_NAME if tag0.has_attr('id'): basename = tag0['id'] elif ndp is not None and hasattr(ndp, att): basename = getattr(ndp, att) elif template is not None and hasattr(template, att): basename = getattr(template, att) elif poset is not None and hasattr(poset, att): basename = getattr(poset, att) else: hashcode = hashlib.sha224(tag0.string).hexdigest()[-8:] basename = 'code-%s' % hashcode docname = os.path.splitext(os.path.basename(realpath))[0] download = docname + "." + basename + "." + klass + '.pdf' a = create_a_to_data(download=download, data_format='pdf', data=pdf) a['class'] = 'pdf_data' a.append(NavigableString(download)) div.append(tag_svg) div.append(a) return div else: return tag_svg
def make(self, context): # @UnusedVariable t = Tag(name='div') t.attrs['id'] = 'toc' t.append('TOC placeholder') return [t]
def visualize(summary): hostnames = summary.get_sorted_hostnames() sorted_test_names = summary.get_sorted_test_names() body = Tag(name='body') table = Tag(name='table') tr = Tag(name='tr') td = Tag(name='td') td.append('hostname') tr.append(td) for hostname in hostnames: td = Tag(name='td') td.attrs['class'] = 'hostname' td.append(hostname) tr.append(td) table.append(tr) tr = Tag(name='tr') td = Tag(name='td') td.append('type') tr.append(td) for hostname in hostnames: td = Tag(name='td') stype = summary.get_host_type(hostname) td.attrs['class'] = stype td.append(stype) tr.append(td) table.append(tr) tr = Tag(name='tr') td = Tag(name='td') td.append('owner') tr.append(td) for hostname in hostnames: td = Tag(name='td') td.attrs['class'] = 'owner' owner = summary.get_owner(hostname) or '(no owner)' td.append(owner) tr.append(td) table.append(tr) for test_name in sorted_test_names: tr = Tag(name='tr') td = Tag(name='td') td.attrs['class'] = 'test_name' # vis = test_name.replace(' : ', ) td.append(test_name) tr.append(td) for hostname in hostnames: td = Tag(name='td') s = Tag(name='span') d = summary.get_data(test_name=test_name, hostname=hostname) if d is None: td.attrs['class'] = 'n-a' td.append('-') else: td.attrs['class'] = d['status'] out = " ||| ".join([str(d['out_short']), str(d['out_long'])]) td.attrs['title'] = out upload_date = d['upload_event_date'] elapsedTime = datetime.datetime.now() - upload_date when = duration_compact(elapsedTime.total_seconds()) # td.append(d['status']) # td.append(Tag(name='br')) # td.append('(%s)' % when) # s.append(when) td.append(s) tr.append(td) table.append(tr) css = """ td.passed { background-color: green; color: white; } td.skipped { background-color: yellow; color: black; } td.failed { background-color: red; color: white; } td.hostname { font-family: monospace; } td.test_name { width: 30em; max-width: 30em; } body { font-size: 8pt; } td { overflow-wrap: break-word; min-width: 4em; max-width: 4em; padding-bottom: 3px; } td { border-bottom: solid 1px black; } """ style = Tag(name='style') style.append(css) body.append(style) body.append(table) return body
def make_figure_from_figureid_attr(soup): """ Makes a figure: <e figure-id='fig:ure' figure-caption='ciao'/> <figure id="fig:ure"> <e figure-id='fig:ure' figure-caption='ciao'/> <figcaption>ciao</figcaption> </figure> Makes a table: <e figure-id='tab:ure' figure-caption='ciao'/> becomes figure-id figure-class """ from mcdp_docs.highlight import add_class for towrap in soup.select('[figure-id]'): ID = towrap['figure-id'] parent = towrap.parent fig = Tag(name='figure') fig['id'] = ID caption_below = True if ID.startswith('fig:'): add_class(fig, 'figure') elif ID.startswith('subfig:'): add_class(fig, 'subfloat') elif ID.startswith('tab:'): add_class(fig, 'table') caption_below = False elif ID.startswith('code:'): add_class(fig, 'code') pass else: msg = 'The ID %r should start with fig: or tab: or code:' % ID raise_desc(ValueError, msg, tag=describe_tag(towrap)) if 'caption-left' in towrap.attrs.get('figure-class', ''): caption_below = False external_caption_id = '%s:caption' % ID external_caption = soup.find(id=external_caption_id) if external_caption is None: external_caption = towrap.find(name='figcaption') if external_caption is not None: # print('using external caption %s' % str(external_caption)) external_caption.extract() if external_caption.name != 'figcaption': logger.error('Element %s#%r should have name figcaption.' % (external_caption.name, external_caption_id)) external_caption.name = 'figcaption' figcaption = external_caption if towrap.has_attr('figure-caption'): msg = 'Already using external caption for %s' % ID raise_desc(ValueError, msg, describe_tag(towrap)) else: # print('could not find external caption %s' % external_caption_id) if towrap.has_attr('figure-caption'): caption = towrap['figure-caption'] else: caption = '' figcaption = Tag(name='figcaption') figcaption.append(NavigableString(caption)) outside = Tag(name='div') outside['id'] = ID + '-wrap' if towrap.has_attr('figure-style'): outside['style'] = towrap['figure-style'] if towrap.has_attr('figure-class'): for k in towrap['figure-class'].split(' '): add_class(towrap, k) add_class(outside, k ) i = parent.index(towrap) towrap.extract() figcontent = Tag(name='div', attrs={'class':'figcontent'}) figcontent.append(towrap) fig.append(figcontent) if caption_below: fig.append(figcaption) else: fig.insert(0, figcaption) add_class(outside, 'generated-figure-wrap') add_class(fig, 'generated-figure') outside.append(fig) parent.insert(i, outside)
def create_link_base(id2filename): ''' Returns a Tag <html> containing the page that is responsible to translate links ''' html = Tag(name='html') head = Tag(name='head') html.append(head) body = Tag(name='body') html.append(body) s = create_link_base_js(id2filename) script = Tag(name='script') script.append(s) body.append(script) script = Tag(name='script') script.append(jump_script) body.append(script) # pre = Tag(name='pre') # pre.append(str(id2filename)) # body.append(pre) return html
def add_prev_next_links(filename2contents, only_for=None): new_one = OrderedDict() for filename, contents in list(filename2contents.items()): if only_for and not filename in only_for: continue id_prev = contents.attrs[ATTR_PREV] a_prev = Tag(name='a') a_prev.attrs['href'] = '#' + str(id_prev) a_prev.attrs['class'] = 'link_prev' a_prev.append('prev') id_next = contents.attrs[ATTR_NEXT] a_next = Tag(name='a') a_next.attrs['href'] = '#' + str(id_next) a_next.attrs['class'] = 'link_next' a_next.append('next') S = Tag(name='div') S.attrs['class'] = ['super'] nav1 = Tag(name='div') add_class(nav1, 'navigation') if id_prev: nav1.append(a_prev.__copy__()) if id_next: nav1.append(a_next.__copy__()) spacer = Tag(name='div') spacer.attrs['style'] = 'clear:both' nav1.append(spacer) add_class(contents, 'main-section-for-page') contents2 = contents S.append(contents2) from .source_info_imp import get_main_header actual_id = get_main_header(contents2) if False: # just checking e = contents2.find(id=actual_id) if e is not None: pass else: logger.error('not found %r' % actual_id) S.attrs['id'] = actual_id contents2.insert(0, nav1.__copy__()) contents2.append(nav1.__copy__()) new_one[filename] = S return new_one
def mark_console_pres_highlight(soup): for code in soup.select('pre code'): pre = code.parent if code.string is None: continue s0 = code.string from HTMLParser import HTMLParser h = HTMLParser() s = h.unescape(s0) if s != s0: # print('decoded %r -> %r' % (s0, s)) pass beg = s.strip() # is it a console line? ct = is_console_line(beg) if ct is None: continue add_class(pre, 'console') # add class "on-hostname" if ct.hostname is not None: cn = 'on-%s' % str(ct.hostname) add_class(pre, cn) code.string = '' lines = s.split('\n') def is_program(x, l): if x == 'git' and 'apt' in l: return False return x in programs for j, line in enumerate(lines): tokens = line.split(' ') for i, token in enumerate(tokens): previous_is_sudo_or_dollar = i >= 1 and tokens[i - 1] in ['$', 'sudo'] if token in ['$', 'DOLLAR']: # add <span class=console_sign>$</span> e = Tag(name='span') e['class'] = 'console_sign' e.string = '$' code.append(e) elif i == 0 and token == ct.hostname: # it's the hostname e = Tag(name='span') e['class'] = 'hostname' e.string = token code.append(e) elif is_program(token, line) and previous_is_sudo_or_dollar: e = Tag(name='span') e['class'] = '%s program' % token e.string = token code.append(e) elif token in program_commands: e = Tag(name='span') e['class'] = '%s program_command' % token e.string = token code.append(e) elif token and token[0] == '-': e = Tag(name='span') e['class'] = 'program_option' e.string = token code.append(e) else: code.append(NavigableString(token)) is_last = i == len(tokens) - 1 if not is_last: before = '![' in ' '.join(tokens[:i + 1]) if not before: # XXX: this is a bug space = Tag(name='span') space.append(' ') space['class'] = 'space' code.append(space) else: code.append(' ') is_last_line = j == len(lines) - 1 if not is_last_line: code.append(NavigableString('\n'))
def f(s): td_ = Tag(name='td') td_.append(s) tr.append(td_)
def h(s): td_ = Tag(name='td') td_.append(s) th.append(td_)
def get_build_table(builds): table = Tag(name='table') th = Tag(name='tr') def h(s): td_ = Tag(name='td') td_.append(s) th.append(td_) h('build') h('commit') h('branch') h('status') h('') h('duration') h('results') h('author') h('commit text') table.append(th) table.attrs['id'] = 'builds' for build in builds.values(): tr = Tag(name='tr') def f(s): td_ = Tag(name='td') td_.append(s) tr.append(td_) table.append(tr) outcome = build.get_outcome() tr.attrs['class'] = outcome a = Tag(name='a') a.attrs['class'] = 'build_url' a.attrs['href'] = build.get_build_url() a.append('#%s' % build.get_build_num()) f(a) a = Tag(name='a') a.attrs['class'] = 'commit_url' a.attrs['href'] = build.get_commit_url() a.append(build.get_commit()[-7:]) f(a) f(build.get_branch()) f(build.get_outcome_string()) stop_time = build.get_stop_time() start_time = build.get_start_time() duration_human = build.get_duration_human() if stop_time is not None: f(date_tag(stop_time)) else: td = Tag(name='span') if start_time is not None: td.append('started ') td.append(date_tag(start_time)) f(td) if duration_human is not None: f(duration_human) else: f('') links = Tag(name='span') links.attrs['class'] = 'links' if build.artefacts: links.append(get_links(build)) f(links) author = Tag(name='span') avatar_url = build.get_avatar_url() author_name = build.get_author_name() if avatar_url is not None: img = Tag(name='img') img.attrs['class'] = 'avatar' img.attrs['src'] = avatar_url img(a) author.append(img) author.append(author_name) f(author) f(build.get_subject()) return table
def make_sections(body, is_marker, preserve=lambda _: False, element_name='section', copy=True, attrs={}): sections = [] def make_new(): x = Tag(name=element_name) for k, v in attrs.items(): x.attrs[k] = v return x current_section = make_new() current_section['id'] = 'before-any-match-of-%s' % is_marker.__name__ current_section['class'] = 'without-header-inside' # sections.append(current_section) for x in body.contents: if is_marker(x): #print('starting %s' % str(x)) if contains_something_else_than_space(current_section): sections.append(current_section) current_section = make_new() current_section['id'] = x.attrs.get( 'id', 'unnamed-h1') + ':' + element_name logger.debug('marker %s' % current_section['id']) current_section['class'] = x.attrs.get('class', '') #print('%s/section %s %s' % (is_marker.__name__, x.attrs.get('id','unnamed'), current_section['id'])) current_section.append(x.__copy__()) current_section['class'] = 'with-header-inside' elif preserve(x): if contains_something_else_than_space(current_section): sections.append(current_section) #current_section['id'] = x.attrs.get('id', 'unnamed-h1') + ':' + element_name #print('%s/preserve %s' % (preserve.__name__, current_section['id'])) sections.append(x.__copy__()) current_section = make_new() current_section.attrs['comment'] = "Triggered by %r" % x else: #x2 = x.__copy__() if copy else x x2 = x.__copy__() if copy else x.extract() current_section.append(x2) if contains_something_else_than_space(current_section): sections.append(current_section) # XXX new_body = Tag(name=body.name) # if len(sections) < 3: # msg = 'Only %d sections found (%s).' % (len(sections), is_marker.__name__) # raise ValueError(msg) logger.info('make_sections: %s found using marker %s' % (len(sections), is_marker.__name__)) for i, s in enumerate(sections): if add_debug_comments: new_body.append('\n') new_body.append( Comment('Start of %s section %d/%d' % (is_marker.__name__, i, len(sections)))) new_body.append('\n') new_body.append(s) new_body.append('\n') if add_debug_comments: new_body.append( Comment('End of %s section %d/%d' % (is_marker.__name__, i, len(sections)))) new_body.append('\n') return new_body
def get_minimal_document(body_contents, title=None, add_markdown_css=False, add_manual_css=False, stylesheet=None, extra_css=None): """ Creates the minimal html document with MCDPL css. add_markdown_css: language + markdown add_manual_css: language + markdown + (manual*) extra_css = additional CSS contents """ check_html_fragment(body_contents) soup = bs("") assert soup.name == 'fragment' if title is None: title = '' html = Tag(name='html') head = Tag(name='head') body = Tag(name='body') head.append( Tag(name='meta', attrs={ 'http-equiv': "Content-Type", 'content': "application/xhtml+xml; charset=utf-8" })) if stylesheet is None: stylesheet = 'v_mcdp_render_default' if add_markdown_css or add_manual_css: link = Tag(name='link') link['rel'] = 'stylesheet' link['type'] = 'text/css' link['href'] = get_css_filename('compiled/%s' % stylesheet) head.append(link) tag_title = Tag(name='title') tag_title.append(NavigableString(title)) head.append(tag_title) parsed = bs(body_contents) assert parsed.name == 'fragment' parsed.name = 'div' body.append(parsed) html.append(head) html.append(body) soup.append(html) if extra_css is not None: add_extra_css(soup, extra_css) s = to_html_stripping_fragment_document(soup) assert not 'DOCTYPE' in s # s = html.prettify() # no: it removes empty text nodes # ns="""<?xml version="1.0" encoding="utf-8" ?>""" ns = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN" "http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd">""" res = ns + '\n' + s # if add_manual_css and MCDPConstants.manual_link_css_instead_of_including: # assert 'manual.css' in res, res res = res.replace('<div><!DOCTYPE html>', '<div>') return res
def make(self, context): # @UnusedVariable """ Returns a BS4 element """ t = Tag(name='span') t.append('Not implemented') return [t]
def make_page(contents, head0, add_toc): """ Returns html (Beautiful Soup document) """ html = Tag(name='html') head = head0.__copy__() html.append(head) body = Tag(name='body') with timeit('make_page() / copy toc'): if add_toc: tocdiv = Tag(name='div') tocdiv.attrs['id'] = 'tocdiv' # toc = main_toc # toc.extract() # del toc.attrs['id'] tocdiv.append(add_toc) section_name = get_first_header_title(contents) if section_name is not None: title2 = bs(section_name) title2.name = 'title' title = head.find('title') if title is None: # msg = 'Cannot find the "title" element' # msg += '\n' + indent(str(head)[:500], 'head') # raise Exception(msg) head.append(title2) else: title.replace_with(title2) body.append(tocdiv) not_toc = Tag(name='div') not_toc.attrs['id'] = 'not-toc' not_toc.append(contents) body.append(not_toc) html.append(body) # delete the original one main_toc = contents.find(id='main_toc') if main_toc is not None: main_toc.extract() return html
def go(selector, parse_expr, extension, use_pre=True, refine=None): for tag in soup.select(selector): source_code = '<unset>' # XXX try: if tag.string is None: # or not tag.string.strip(): if not tag.has_attr('id'): msg = "If <pre> is empty then it needs to have an id." raise_desc(ValueError, msg, tag=describe_tag(tag)) # load it tag_id = tag['id'].encode('utf-8') if '.' in tag_id: i = tag_id.index('.') libname, name = tag_id[:i], tag_id[i + 1:] use_library = library.load_library(libname) else: name = tag_id use_library = library basename = '%s.%s' % (name, extension) data = use_library._get_file_data(basename) source_code = data['data'] else: source_code = get_source_code(tag) # prettify. # remove spurious indentation source_code = source_code.strip() do_apply_suggestions = (not tag.has_attr('noprettify') and not tag.has_attr('np')) # then apply suggestions try: if do_apply_suggestions: x = parse_wrap(parse_expr, source_code)[0] xr = parse_ndp_refine(x, Context()) suggestions = get_suggestions(xr) source_code = apply_suggestions( source_code, suggestions) except DPSyntaxError as e: if raise_errors: raise else: res.note_error(str(e), HTMLIDLocation.for_element(tag)) continue # we don't want the browser to choose different tab size # source_code = source_code.replace('\t', ' ' * 4) # we are not using it _realpath = realpath context = Context() def postprocess(x): if refine is not None: return refine(x, context=context) else: return x # print('rendering source code %r' % source_code) html = ast_to_html(source_code, parse_expr=parse_expr, add_line_gutter=False, postprocess=postprocess) for w in context.warnings: if w.where is not None: from mcdp_web.editor_fancy.app_editor_fancy_generic import html_mark html = html_mark(html, w.where, "language_warning") frag2 = BeautifulSoup(html, 'lxml', from_encoding='utf-8') if use_pre: rendered = Tag(name='div', attrs={'class': 'rendered'}) pre = frag2.pre pre.extract() rendered.append(pre) if not rendered.has_attr('class'): rendered['class'] = "" if tag.has_attr('label'): text = tag['label'] tag_label = Tag(name='span') add_class(tag_label, 'label') add_class(tag_label, 'label_inside') tag_label.append(NavigableString(text)) pre.insert(0, tag_label) tag_label_outside = Tag(name='span') add_class(tag_label_outside, 'label') add_class(tag_label_outside, 'label_outside') tag_label_outside.append(NavigableString(text)) rendered.insert(0, tag_label_outside) max_len = max_len_of_pre_html(html) if tag.has_attr('label'): add_class(rendered, 'has_label') max_len = max(max_len, len(tag['label']) + 6) style = '' else: # using <code> rendered = frag2.pre.code rendered.extract() if not rendered.has_attr('class'): rendered['class'] = "" style = '' if tag.has_attr('style'): style = style + tag['style'] if style: rendered['style'] = style if tag.has_attr('class'): add_class(rendered, tag['class']) if tag.has_attr('id'): rendered['id'] = tag['id'] if use_pre: if generate_pdf: pdf = get_ast_as_pdf(source_code, parse_expr) if tag.has_attr('id'): basename = tag['id'] else: hashcode = hashlib.sha224( source_code).hexdigest()[-8:] basename = 'code-%s' % hashcode docname = os.path.splitext( os.path.basename(realpath))[0] download = docname + '.' + basename + '.source_code.pdf' a = create_a_to_data(download=download, data_format='pdf', data=pdf) a['class'] = 'pdf_data' a.append(NavigableString(download)) div = Tag(name='div') div.append(rendered) div.append(a) tag.replaceWith(div) else: tag.replaceWith(rendered) else: tag.replaceWith(rendered) except DPSyntaxError as e: if raise_errors: raise else: res.note_error(str(e), HTMLIDLocation.for_element(tag)) # note_error(tag, e) if tag.string is None: tag.string = "`%s" % tag['id'] continue except DPSemanticError as e: if raise_errors: raise else: res.note_error(str(e), HTMLIDLocation.for_element(tag)) # note_error(tag, e) if tag.string is None: tag.string = "`%s" % tag['id'] continue except DPInternalError as ex: msg = 'Error while interpreting the code:\n\n' msg += indent(source_code, ' | ') raise_wrapped(DPInternalError, ex, msg, exc=sys.exc_info())
def html_list_of_notes(aug, tag, how_to_call_them, klass, header=None): from mcdp_docs.elements_abbrevs import format_name notes = aug.get_notes_by_tag(tag) # print('%d notes for tag %s' % (len(notes), tag)) html = Tag(name='html') head = Tag(name='head') meta = Tag(name='meta') meta.attrs['content'] = "text/html; charset=utf-8" meta.attrs['http-equiv'] = "Content-Type" head.append(meta) head.append('\n') html.append(head) html.append('\n') body = Tag(name='body') if header is not None: body.append(header) body.append('\n') html.append(body) html.append('\n') if not notes: p = Tag(name='p') p.append('There were no %s.' % how_to_call_them) body.append(p) body.append('\n') else: p = Tag(name='p') p.append('There were %d %s.' % (len(notes), how_to_call_them)) body.append(p) body.append('\n') if klass == 'task': # XXX sorted_notes = sort_by_dest(notes) order = sorted(sorted_notes, key=lambda x: -len(sorted_notes[x]) if x else 1000) table = Tag(name='table') for who in order: section_notes = sorted_notes[who] tr = Tag(name='tr') td = Tag(name='td') if who is None: a = Tag(name='a') a.attrs['href'] = '#for:%s' % who who = 'unassigned' a.append(who) td.append(a) else: a = Tag(name='a') a.attrs['href'] = '#for:%s' % who a.append("Tasks assigned to") td.append(a) td.append(' ') td.append(format_name(who)) tr.append(td) td = Tag(name='td') td.append('%d tasks' % len(section_notes)) tr.append(td) table.append(tr) body.append(table) for who, section_notes in sorted_notes.items(): sec = Tag(name='div') sec.attrs['id'] = 'for:%s' % who if who is None: t = '%s unassigned tasks' % len(section_notes) else: t = '%d tasks assigned to %s' % (len(section_notes), who) h = Tag(name='h3') h.append(t) sec.append(h) for i, note in enumerate(section_notes): div = note.as_html() div.attrs['class'] = klass sec.append(div) sec.append('\n\n') body.append(sec) body.append('\n\n') else: for i, note in enumerate(notes): div = note.as_html() div.attrs['class'] = klass body.append(div) body.append('\n\n') body.append(get_html_style()) return html
def compact_desc_tag(element): t = Tag(name=element.name) t.attrs.update(element.attrs) t.append(' ... ') return str(t)
def mark_in_html_notes(notes, soup, note_type, index_url, klasses): from mcdp_docs.check_missing_links import get_id2element id2element, duplicates = get_id2element(soup, 'id') ids_ordered = list(id2element) N = len(notes) indices = [None for _ in range(len(notes))] for i in range(len(notes)): eid, note = notes[i] if eid in id2element: indices[i] = ids_ordered.index(eid) else: indices[i] = None indices = sorted(range(N), key=lambda _: indices[_]) def idfor(x): return 'note-%s-%d' % (note_type, x) def linkto(x): eid_, _ = notes[indices[x]] return '#%s' % eid_ from mcdp_utils_xml import stag # TODO: there is the case where there are multiple notes that refer to the same element.. for b, i in enumerate(indices): eid, note = notes[i] element = id2element.get(eid, None) if element is None: msg = 'Cannot find the element corresponding to ID %r' % eid msg += '\n\nI cannot mark this note:' msg += '\n\n' + indent(note, ' | ') # raise Exception(msg) # FIXME logger.error(msg) else: if element.name == 'code' and element.parent and element.parent.name == 'pre': element = element.parent note_html = note.as_html(inline=True) inset = insert_inset(element, short=note_type, # XXX long_error=note_html, klasses=klasses) inset.attrs['id'] = idfor(b) summary = inset.find('summary') summary.append('\n') if b > 0: a = Tag(name='a') a.attrs['class'] = 'note-navigation' a.append('previous ') a.attrs['href'] = linkto(b - 1) summary.insert(0, a) if b < N - 1: a = Tag(name='a') a.attrs['class'] = 'note-navigation' a.append(' next') a.attrs['href'] = linkto(b + 1) summary.append(a) summary.append(' (%d of %s) ' % (b + 1, len(notes))) summary.append('\n') summary.append(stag('a', 'index', href=index_url)) summary.append('\n')
def col_macro_(e, ncols): """ Bug: For some reasone bd4 removes the whitespace I use for indentation. """ assert e.name == 'div' assert e.has_attr('make-col%d' % ncols) # print describe_tag(e) children = list(e.children) # remove strings from this is_string = lambda x: isinstance(x, NavigableString) strings = [_ for _ in children if is_string(_)] children = [_ for _ in children if not is_string(_)] if len(children) < ncols: msg = ('Cannot create table with %r cols with only %d children' % (ncols, len(children))) raise_desc(ValueError, msg, tag=describe_tag(e)) for c in children: c.extract() for s in strings: ss = str(s) empty = not ss.strip() if not empty: msg = 'Found nonempty string %r between children.' % ss raise_desc(ValueError, msg, tag=describe_tag(e)) # remove it s.extract() nchildren = len(children) nrows = int(math.ceil(nchildren / float(ncols))) parent = e.parent original_position = parent.index(e) e.extract() table = e e.name = 'table' add_class(table, 'col%d' % ncols) add_class(table, 'colN') wrapper = Tag(name='div') add_class(wrapper, 'col%d-wrap' % ncols) add_class(wrapper, 'colN-wrap') NL = '\n' # S = '-' * 4 # XXX: change to above to see the problem with indentation S = ' ' * 4 tbody = Tag(name='tbody') for row in range(nrows): tbody.append(NavigableString(NL)) tbody.append(NavigableString(S+S)) tr = Tag(name='tr') tr.append(NavigableString(NL)) for col in range(ncols): td = Tag(name='td') i = col + row * ncols if i < len(children): child = children[i] td.append(child) else: td.append(Comment('empty row %d col %d' % (row, col))) tr.append(NavigableString(S+S+S)) tr.append(td) tr.append(NavigableString(NL)) tr.append(S+S) if row == 0 and ('labels-row1' in e.attrs.get('class', '')): thead = Tag(name='thead') thead.append(tr) table.append(thead) # add in table, not tbody else: tbody.append(tr) # add in tbody tbody.append(NavigableString(NL+S)) table.append(tbody) wrapper.append(NavigableString(NL + S)) wrapper.append(table) wrapper.append(NavigableString(NL)) parent.insert(original_position, wrapper)
def as_html(self, inline=False): div = Tag(name='div') for tag in self.tags: s = Tag(name='span') s.attrs['class'] = 'note-tag' css = """ font-size: 70%; font-family: arial; background-color: #dee; padding: 5px; border-radius: 5px; margin: 0.4em; """ s.attrs['style'] = css.replace('\n', ' ').strip() s.append(tag) div.append(s) div.attrs['class'] = 'note' container = Tag(name='div') if self.msg is not None: if isinstance(self.msg, Tag): container.append(self.msg.__copy__()) else: pre = Tag(name='pre') pre.append(self.msg) container.append(pre) else: p = Tag(name='p') p.append("No message given.") container.append(p) div.append(container) if self.locations: if len(self.locations) > 1: p = Tag(name='p') p.append('These are the locations indicated:') div.append(p) dl = Tag(name='dl') for k, v in self.locations.items(): dt = Tag(name='dt') dt.append(k) dl.append(dt) dd = Tag(name='dd') # setting inline - False dd.append(v.as_html(inline=False)) dl.append(dd) div.append(dl) else: # p = Tag(name='p') # p.append("Location:") # div.append(p) location = list(self.locations.values())[0] div.append(location.as_html(inline=inline)) else: p = Tag(name='p') p.append("(No locations provided)") div.append(p) from mcdp_utils_xml import stag s = Tag(name='span') s.append('Created by function ') s.append(stag('code', self.created_function)) # s.append(br()) s.append(' in module ') s.append(stag('code', self.created_module)) s.append('.') p = Tag(name='p') p.append(s) div.append(s) return div
def get_row(i, log, url_to_resource): trh = Tag(name="tr") tr = Tag(name="tr") def td(x): t = Tag(name="td") if x is not None: t.append(x) return t trh.append(td("index")) tr.append(td(str(i))) trh.append(td("")) rel = get_small_video2(log, url_to_resource) if rel: video = video_for_source(rel) tr.append(td(video)) else: tr.append(td("-")) # tr.append(td(log.log_name)) # trh.append(td('video')) f = Tag(name="td") rel = get_large_video2(log, url_to_resource) if rel: a = Tag(name="a") a.attrs["href"] = rel a.append("video") f.append(a) rel = get_thumbnails2(log, url_to_resource) if rel: # f.append(Tag(name='br')) f.append(" ") a = Tag(name="a") a.attrs["href"] = rel a.append("thumbnails") f.append(a) # n = append_urls(log, f) # urls = [x for x in log.resources['bag']['urls'] if show_url(x)] # for url in urls: # f.append(' ') # a = Tag(name='a') # a.attrs['href'] = url # a.append('bag') # f.append(a) trh.append(td("misc")) tr.append(f) # # tr.append(td(a)) # else: # tr.append(td('')) trh.append(td("date")) tr.append(td(log.date)) if log.length is not None: l = f"{log.length:5.1f} s" else: l = "(none)" trh.append(td("length")) tr.append(td(l)) trh.append(td("vehicle")) tr.append(td(log.vehicle)) if False: if log.valid: sr = "Yes." else: sr = log.error_if_invalid trh.append(td("valid")) tr.append(td(sr)) trh.append(td("ID")) a = Tag(name="a") a.append(log.log_name) a.attrs["href"] = f"#{log.log_name}" tr.append(td(a)) if not log.valid: tr.attrs["class"] = ["invalid"] else: tr.attrs["class"] = ["valid"] return trh, tr
def make_last_modified(files_contents, nmax=100): res = AugmentedResult() files_contents = [DocToJoin(*x) for x in files_contents] files_contents = [_ for _ in files_contents if _.source_info] files_contents = list( sorted(files_contents, key=lambda x: x.source_info.last_modified, reverse=True)) r = Tag(name='fragment') r.append('\n') h = Tag(name='h1') h.append('Last modified') h.attrs['id'] = 'sec:last-modified' r.append(h) r.append('\n') ul = Tag(name='ul') ul.append('\n') for d in files_contents[:nmax]: li = Tag(name='li') when = d.source_info.last_modified when_s = time.strftime("%a, %b %d", when) # %H:%M li.append(when_s) li.append(': ') hid = get_main_header(bs(d.contents)) if hid is None: what = "File %s" % d.docname else: what = Tag(name='a') what.attrs['href'] = '#' + hid what.attrs['class'] = MCDPManualConstants.CLASS_NUMBER_NAME li.append(what) li.append(' (') name = d.source_info.author.name li.append(name) li.append(')') ul.append(li) ul.append('\n') r.append(ul) s = to_html_stripping_fragment(r) # print s res.set_result(s) return res
def get_report(logs, url_to_resource, initial_screens: bool = True) -> str: length = 0 vehicles = set() for _, log in list(logs.items()): length += log.length vehicles.add(log.vehicle) html = Tag(name="html") body = Tag(name="body") head = Tag(name="head") link = Tag(name="link") link.attrs["type"] = "text/css" link.attrs["rel"] = "stylesheet" link.attrs["href"] = "style.css" title = Tag(name="title") title.append("Duckietown Logs Database") head.append(link) html.append(head) html.append(body) h = Tag(name="h1") h.append("Duckietown Logs Database") body.append(h) c = f"Showing {len(logs)} logs from {len(vehicles)} different Duckiebots, for a total length of " \ f"{length / 3600.0:.1f} hours." p = Tag(name="p") p.append(c) body.append(p) if initial_screens: t = summary_table(logs, url_to_resource) body.append(t) body.append(html_table_from_table(logs, url_to_resource)) make_sections(body, logs, url_to_resource) s = str(html) return s
def visualize(summary): hostnames = summary.get_sorted_hostnames() sorted_test_names = summary.get_sorted_test_names() body = Tag(name='body') table = Tag(name='table') tr = Tag(name='tr') td = Tag(name='td') td.append('hostname') tr.append(td) for hostname in hostnames: td = Tag(name='td') td.attrs['class'] = 'hostname' td.append(hostname) tr.append(td) table.append(tr) tr = Tag(name='tr') td = Tag(name='td') td.append('type') tr.append(td) for hostname in hostnames: td = Tag(name='td') stype = summary.get_host_type(hostname) td.attrs['class'] = stype td.append(cute_stype(stype)) tr.append(td) table.append(tr) # tr = Tag(name='tr') # td = Tag(name='td') # td.append('last heard') # tr.append(td) # for hostname in hostnames: # td = Tag(name='td') # date = summary.get_last_upload_date(hostname) # if date is not None: # date_s = date.isoformat() # else: # date_s = '?' # td.attrs['class'] = 'date' # td.append(date_s) # tr.append(td) # table.append(tr) tr = Tag(name='tr') td = Tag(name='td') td.append('owner/username') tr.append(td) for hostname in hostnames: td = Tag(name='td') td.attrs['class'] = 'owner' owner = summary.get_owner(hostname) or '(no owner)' td.append(owner) tr.append(td) table.append(tr) tr = Tag(name='tr') td = Tag(name='td') td.append('location') tr.append(td) for hostname in hostnames: td = Tag(name='td') td.attrs['class'] = 'location' country = summary.get_country(hostname) or '' td.append(cute_country(country)) tr.append(td) table.append(tr) for test_name in sorted_test_names: tr = Tag(name='tr') td = Tag(name='td') td.attrs['class']= 'test_name' # vis = test_name.replace(' : ', ) td.append(test_name) tr.append(td) n_failed_or_invalid = 0 for hostname in hostnames: d = summary.get_data(test_name=test_name, hostname=hostname) if d is not None and d['status'] in [ChecksConstants.FAIL, ChecksConstants.ERROR]: n_failed_or_invalid += 1 if n_failed_or_invalid == 0: continue if "dt_live_instagram_" in test_name or 'dt_augmented_reality_' in test_name: continue for hostname in hostnames: td = Tag(name='td') s = Tag(name='span') d = summary.get_data(test_name=test_name, hostname=hostname) if d is None: td.attrs['class'] = 'n-a' td.append('-') else: td.attrs['class'] = d['status'] out_short = d['out_short'] out_long = d['out_long'] # # out_short = out_short.encode('utf8', 'ignore') # out_long = out_long.encode('utf8', 'ignore') # print out_short.__repr__() # print out_long.__repr__() out = u" ||| ".join([out_short, out_long]) td.attrs['title'] = out upload_date = d['upload_event_date'] elapsedTime = datetime.datetime.now() - upload_date when = duration_compact(elapsedTime.total_seconds()) # td.append(d['status']) # td.append(Tag(name='br')) # td.append('(%s)' % when) # s.append(when) td.append(s) tr.append(td) table.append(tr) css = """ td.passed { background-color: green; color: white; } td.invalid { background-color: purple; color: white; } td.skipped { background-color: yellow; color: black; } td.failed { background-color: red; color: white; } td.hostname { font-family: monospace; } td.test_name { width: 30em; max-width: 30em; } body { font-size: 8pt; } td { overflow-wrap: break-word; min-width: 4em; max-width: 4em; padding-bottom: 3px; } td { border-bottom: solid 1px black; } """ style = Tag(name='style') style.append(css) body.append(style) body.append(table) html = Tag(name='html') head = Tag(name='head') meta = Tag(name='meta') meta.attrs['charset'] = 'UTF-8' head.append(meta) html.append(head) html.append(body) return html
def do_bib(soup, bibhere): """ find used bibliography entries put them there """ used = [] unused = set() for a in soup.find_all('a'): href = a.attrs.get('href', '') if href.startswith('#bib:'): used.append(href[1:]) # no "#" logger.debug('I found %d references, to these: %s' % (len(used), used)) # collect all the <cite> id2cite = {} for c in soup.find_all('cite'): ID = c.attrs.get('id', None) id2cite[ID] = c if ID in used: add_class(c, 'used') else: unused.add(ID) add_class(c, 'unused') # divide in found and not found found = [] notfound = [] for ID in used: if not ID in id2cite: if not ID in notfound: notfound.append(ID) else: found.append(ID) # now create additional <cite> for the ones that are not found for ID in notfound: cite = Tag(name='cite') s = 'Reference %s not found.' % ID cite.append(NavigableString(s)) cite.attrs['class'] = ['errored', 'error'] # XXX soup.append(cite) id2cite[ID] = cite # now number the cites n = 1 id2number = {} for ID in used: if not ID in id2number: id2number[ID] = n n += 1 # now add the attributes for cross-referencing for ID in used: number = id2number[ID] cite = id2cite[ID] cite.attrs[LABEL_NAME] = '[%s]' % number cite.attrs[LABEL_SELF] = '[%s]' % number cite.attrs[LABEL_NUMBER] = number cite.attrs[LABEL_WHAT] = 'Reference' cite.attrs[LABEL_WHAT_NUMBER_NAME] = '[%s]' % number cite.attrs[LABEL_WHAT_NUMBER] = '[%s]' % number # now put the cites at the end of the document for ID in used: c = id2cite[ID] # remove it from parent c.extract() # add to bibliography bibhere.append(c) s = ("Bib cites: %d\nBib used: %s\nfound: %s\nnot found: %s\nunused: %d" % (len(id2cite), len(used), len(found), len(notfound), len(unused))) logger.info(s)
def get_links_from_artefacts(artefacts, branch=None, build_num=None): links = Tag(name='span') links.attrs['class'] = 'links' groups = sorted(set([art.group for art in artefacts])) for j, g in enumerate(groups): if j > 0: links.append(br()) tag_g = Tag(name='span') tag_g.attrs['class'] = 'group' tag_g.append(' ') tag_g.append(g) tag_g.append(br()) links.append(tag_g) links.append('[') arts = [_ for _ in artefacts if _.group == g] div = get_links2(arts, branch, build_num) links.append(div) links.append(']') return links