def __build_content_text(base_url, all_docs, node, github_url, domain_cats): """ Build an HTML text /!\ Links to refs broken by Markdown transformation NEED to replace refs in md with good links BEFORE transformation. """ raw_content = "" #### XML.xpath_plain(node, "../raw_content/text()") if raw_content > "" : ref = XML.xpath_plain(node, "@ref") doc = XML.xpath_node(node, "ancestor::document") src_path = os.path.join(XML.xpath_plain(doc, "@dir"), XML.xpath_plain(doc, "@file")) src_url = __to_github(github_url, src_path) github = f"""<a href="{src_url}" target="_blank">Github</a>""" md_html = MD.markdown(raw_content) ret = f""" <div class="sectionTitle smTitle"> <h6>{ref} ({github})</h6> <div class="titleBar"></div> </div> {md_html} """ else: ret = __build_content_chapter(base_url, all_docs, 2, node, github_url, domain_cats) return ret
def __build_content_gallery(base_url, all_docs, item): """ Build an HTML image gallery for given ref. """ try: root_doc = XML.xpath_node(item, "ancestor::document") root_dir = XML.xpath_plain(root_doc, "@dir") exts = LIB.DOC_EXT["img"] images = XML.xpath_list(root_doc, "_meta/attachment[contains('"+exts+"', @ext)]") inner_html = "" for image in images: img_path = XML.xpath_plain(image, "@to_path") img_file = XML.xpath_plain(image, "@file") rel_path = os.path.relpath(img_path, start=root_dir) link = f""" <span class="icon"><a title="{img_file}" href="{rel_path}"><img alt="[image]" src="{rel_path}"/></a></span> """ inner_html += link ret = f""" <h2>Image gallery</h2> <p>{inner_html}</p> """ if inner_html > "" else "" except: ret = "" return __comment_html(ret)
def __build_content_attachments(base_url, all_docs, item): """ Build an HTML attached documents list for given ref. """ try: root_doc = XML.xpath_node(item, "ancestor::document") root_dir = XML.xpath_plain(root_doc, "@dir") exts = "|".join([LIB.DOC_EXT[t] for t in ["txt","doc","slide","data"]]) attachments = XML.xpath_list(root_doc, "_meta/attachment[contains('"+exts+"', @ext)]") inner_html = "" for att in attachments: att_path = XML.xpath_plain(att, "@to_path") att_title = XML.xpath_plain(att, "@file") att_ext = XML.xpath_plain(att, "@ext") rel_path = os.path.relpath(att_path, start=root_dir) link = f""" <a title="{att_title}" href='{rel_path}' target="_blank" class="link_for_{att_ext}">{att_title}</a> """ inner_html += link ret = f""" <h2>Attached documents</h2> <p>{inner_html}</p> """ if inner_html > "" else "" except: ret = "" return __comment_html(ret)
def __store_with_attachments(dst_dir, filename, text, all_docs, ref, root_dir): """ Store text in dst_dir/filename, create dir if necessary and copy attachments. """ path = __store_file(dst_dir, filename, text) root_doc = XML.xpath_node(all_docs, ".//document[.//*/@ref='"+ref+"']") rel_dir = XML.xpath_plain(root_doc, "@dir") src_dir = os.path.join(root_dir, rel_dir) attachments = XML.xpath_list(root_doc, "_meta/attachment") for att in attachments: att_path = XML.xpath_plain(att, "@to_path") att_rel_path = os.path.relpath(att_path, start=rel_dir) src_att = os.path.join(src_dir, att_rel_path) dst_att = os.path.join(dst_dir, att_rel_path) dst_dir_att = os.path.dirname(dst_att) if not os.path.exists(dst_dir_att): os.makedirs(dst_dir_att) try: if not os.path.exists(dst_att): print("Storing attachment : " + dst_att) shutil.copyfile(src_att, dst_att) except: #LIB.debug_error() meta = XML.xpath_node(all_docs, ".//document[.//*/@ref='"+ref+"']/_meta") XML.add_error(meta, "__store_with_attachments()", "warning", "missing_file", "can't copy attachment, missing file", 0, src_att) return path
def doc(root_dir, name): """ Parse a Markdown or CSV document file and return DOM. """ full_path = os.path.join(root_dir, name) input_file = codecs.open(full_path, mode="r", encoding="utf-8") dir, file = os.path.split(name) forget, ext = os.path.splitext(name) ext = ext[1:].lower() xml_doc = ET.Element('document', { "path": name, "dir": dir, "file": file, "ext": ext }) meta = ET.SubElement(xml_doc, "_meta") if (ext == LIB.CSV_EXT): try: dom = __csv_parse(xml_doc, input_file.read()) except: dom = __bin_parse(xml_doc, None) elif (ext == LIB.MD_EXT): #try: dom = __md_parse(xml_doc, input_file.read()) #except: dom = __bin_parse(xml_doc, None) else: # other parsed as binary dom = __bin_parse(xml_doc, None) #build @ref, @title, @icon, @cat attributes for root_node in XML.xpath_list(dom, "*[name][not(@is_template)]"): ref = __extract_ref(XML.xpath_plain(root_node, "name/text()")) if (ref > ""): title = LIB.sanitize(XML.xpath_plain(root_node, "title/text()")) icon = __extract_image(XML.xpath_plain(root_node, "text()")) cat = "CAT-" + root_node.tag root_node.set("ref", ref) root_node.set("title", title) root_node.set("icon", icon) root_node.set("cat", cat) #find all distinct ref patterns text = " ".join(root_node.xpath(".//text()")) ext_refs = set(re.findall('\[`([A-Z]{2,6}-[\w_]+)`\]', text)) for ext_ref in ext_refs: if (ext_ref != ref): XML.add_external_ref(meta, ext_ref, ref) else: XML.add_error(meta, "doc()", "warning", "ref_error", "document reference syntax error", 0, ref) return dom
def store_content_page(www_path, all_docs, item_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors): """ Store content page for item_ref entity in www_path/cat_ref/item_ref/content.html file. """ base_url = "../" item_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+item_ref+"']/@cat") entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat") html_menu = __build_menu(base_url+"../", all_docs, item_cat_ref, domain_cats) html_content = __build_content_section(base_url+"../", all_docs, item_ref, github_url, domain_cats, domain_refs) html_view = __build_view(base_url+"../", html_menu, html_content, doc_entry_ref, entry_cat_ref, "doc", with_errors) dst_dir = www_path + item_ref + "/" return __store_with_attachments(dst_dir, "content.html", html_view, all_docs, item_ref, root_dir)
def __build_index_section(base_url, all_docs, cat_ref, github_url, domain_refs): """ Build an HTML section for ref entity in all_docs. """ cat = XML.xpath_node(all_docs, (".//*[@ref='"+cat_ref+"']")) cat_title = XML.xpath_plain(cat, "title/text()") cat_about_html = XML.xpath_md(cat, "about/text()") all_items = XML.xpath_list(all_docs, ".//*[@cat='"+cat_ref+"']") rows = "" for itm in all_items: itm_ref = XML.xpath_plain(itm, "@ref") if (not (itm_ref in domain_refs)) : continue itm_icon = XML.xpath_plain(itm, "@icon") doc = XML.xpath_node(itm, "ancestor::document") itm_src_path = os.path.join(XML.xpath_plain(doc, "@dir"), XML.xpath_plain(doc, "@file")) itm_ttl_html = XML.xpath_md(itm, "@title") itm_about_html = XML.xpath_md(itm, "about/text()") itm_src_url = __to_github(github_url, itm_src_path) link_ref = __build_ref_link(base_url, all_docs, itm_ref) rows += f""" <tr> <td class="icon"><img class="icon" alt="[icon]" src='{itm_ref}/{itm_icon}'/></td> <td>{link_ref}</td> <td>{itm_ttl_html}</td> <td>(<a href="{itm_src_url}" target="_blank">Github</a>)</td> </tr>\n """ ret = f""" <div class="MenuContent"> <div id="primary"> <section class="comonSection singlefoliodetails"> <div class="container-fluid"> <div class="col-lg-12 noPadding"> <section class="col-md-10"> <div class="sectionTitle smTitle"> <h2>{cat_title} index</h2> <div class="titleBar"></div> </div> <p>{cat_about_html}</p> <table class="table table-bordered"> {rows} </table> </section> </div> </div> </section> </div> </div> """ return __comment_html(ret)
def __build_content_chapter(base_url, all_docs, level, node, github_url, domain_cats): """ Build recursively an HTML chapter at level for given node. TODO : if level 2 produce : <div class="sectionTitle smTitle"> <h2>configuration</h2> <h6>CFG-sweeping_probe (<a href="https://github.com/Olivierff/PRJ-medtec_kit//tree/master/configurations/CFG-sweeping_probe" target="_blank">Github</a>)</h6> <div class="titleBar"></div> </div> """ try: children = XML.xpath_list(node, "node()") children_html = "" for child in children: if (ET.iselement(child)): children_html += __build_content_chapter(base_url, all_docs, level+1, child, github_url, domain_cats) else: child = __update_refs(base_url, all_docs, child, node, domain_cats) children_html += MD.markdown(child) title = node.tag if (level==2): ref = XML.xpath_plain(node, "@ref") doc = XML.xpath_node(node, "ancestor::document") src_path = os.path.join(XML.xpath_plain(doc, "@dir"), XML.xpath_plain(doc, "@file")) src_url = __to_github(github_url, src_path) github = f"""<a href="{src_url}" target="_blank">Github</a>""" ret = f""" <div class="sectionTitle smTitle"> <h2>{title}</h2> <h6>{ref} ({github})</h6> <div class="titleBar"></div> </div> {children_html} """ else: ret = f""" <h{level}>{title}</h{level}> {children_html} """ except: ret = """ <p><strong>Error here !!! </strong></p> """ return ret
def __build_one_site(root_dir, all_docs, domain_refs, doc_entry_ref, www_root_dir, site_name, github_url, user_group): """ Build one autonomous static web site centered on doc_entry_ref and restricted on domain_refs which is a set of authorized entities. Allowed user_groups: #public, #devs, #core_team, #admin """ www_build_dir = www_root_dir + site_name + "/" print("Reset build directory : " + www_build_dir) shutil.rmtree(www_build_dir, ignore_errors=True) try: shutil.copytree("src/default_www/", www_build_dir) except: # strange behaviour on Windows, try again ... shutil.copytree("src/default_www/", www_build_dir) all_domain_cats = { XML.xpath_plain(all_docs, ".//*[@ref='"+itm_ref+"']/@cat") for itm_ref in domain_refs } - {""} domain_cats = [] for cat_ref in all_domain_cats: cat_restrict = XML.xpath_plain(all_docs, ".//*[@ref='"+cat_ref+"']/@restricted_to") if (cat_restrict == "") or (user_group in cat_restrict): domain_cats += [cat_ref] with_errors = user_group in "#devs,#admin" path = HTM.store_home_page(www_build_dir, all_docs, "", root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors) for cat_ref in domain_cats: try: path = HTM.store_index_page(www_build_dir, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors) except: LIB.debug_error() LIB.debug("### Error for category index : ", cat_ref) for itm_ref in domain_refs: cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+itm_ref+"']/@cat") if (cat_ref in domain_cats): try: path = HTM.store_content_page(www_build_dir+cat_ref+"/", all_docs, itm_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors) except: LIB.debug_error() LIB.debug("### Error for file : ", itm) path = HTM.store_glossary_page(www_build_dir, all_docs, "", root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors) if with_errors: path = HTM.store_error_page(www_build_dir, all_docs, "", root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors)
def store_glossary_page(www_path, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors): """ Store error page for ref category in www_path/error_log.html file. """ base_url = "" entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat") html_menu = __build_empty_menu() html_glossary = __build_glossary_section(base_url, all_docs, github_url, domain_cats, domain_refs) html_view = __build_view(base_url, html_menu, html_glossary, doc_entry_ref, entry_cat_ref, "gloss", with_errors) dst_dir = www_path return __store_with_attachments(dst_dir, "glossary.html", html_view, all_docs, "", root_dir)
def store_home_page(www_path, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors): """ Store index page for ref category in www_path/index.html file. """ base_url = "" entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat") html_menu = __build_empty_menu() html_home = "<p>Landing page would be here !</p>" html_view = __build_view(base_url, html_menu, html_home, doc_entry_ref, entry_cat_ref, "home", with_errors) dst_dir = www_path return __store_with_attachments(dst_dir, "index.html", html_view, all_docs, "", root_dir)
def store_index_page(www_path, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors): """ Store index page for ref category in www_path/cat_ref/index.html file. """ base_url = "../" entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat") html_menu = __build_menu(base_url, all_docs, cat_ref, domain_cats) html_index = __build_index_section(base_url, all_docs, cat_ref, github_url, domain_refs) html_view = __build_view(base_url, html_menu, html_index, doc_entry_ref, entry_cat_ref, "doc", with_errors) dst_dir = www_path + cat_ref + "/" return __store_with_attachments(dst_dir, "index.html", html_view, all_docs, "", root_dir)
def __build_menu(base_url, all_docs, cur_ref, domain_cats): """ Build an HTML menu part for ref entity in all_docs. """ sorted_cats = XML.xpath_num_sorted(all_docs, "//category", "menu/@position") menu_items = "" for cat in sorted_cats: cat_ref = XML.xpath_plain(cat, "@ref") if (not (cat_ref in domain_cats)) : continue cat_menu = XML.xpath_plain(cat, "menu/text()") is_active = ' class="active"' if cat_ref==cur_ref else "" menu_items += f""" <li{is_active}><a class="Menu" href='{base_url}{cat_ref}/index.html'>{cat_menu}</a></li> """ ret = f""" <ul class="Menu"> {menu_items} </ul> """ return __comment_html(ret)
def register_attachments(all_docs): """ register any attached "binary" document to its master document. Based on the file path """ attachments = XML.xpath_list(all_docs, "//document[count(*) <= 1]") for att in attachments: att_dir = XML.xpath_plain(att, "@dir") att_path = XML.xpath_plain(att, "@path") att_name = XML.xpath_plain(att, "@file") att_ext = XML.xpath_plain( att, "@ext" ) # vvvv Not very smart, but it's late and it works .... vvvv root_pathes = XML.xpath_list( all_docs, "//document[count(*) > 1][@file='readme.md' or @file='Readme.md' or @file='README.md' or @file='readme.MD' or @file='Readme.MD' or @file='README.MD'][starts-with('" + att_dir + "', @dir)]/@path") max_root_path = max(root_pathes, key=len, default="") meta = XML.xpath_node( all_docs, "//document[@path='" + max_root_path + "']/_meta") XML.add_attachment(meta, att_path, att_ext, att_name) return all_docs
def __build_ref_link(base_url, all_docs, ref): """ Build an HTML link for ref entity. """ try: itm = XML.xpath_node(all_docs, ".//*[@ref='"+ref+"']") cat_ref = "CAT-" + itm.tag url = base_url + cat_ref + "/" + ref + "/content.html" title = XML.xpath_plain(itm, "@title") except: url="" title="Not found" ret = f""" <a href="{url}" title="{title}"><code>{ref}</code></a> """ return ret
def __update_refs(base_url, all_docs, text, node, domain_cats): """ Replace all Markdown link pattern [`CAT-xxxxxx`](../../interfaces/ITF-A_gnd) ) with correct url relative to base_url. """ #find all distinct ref patterns refs = re.findall('\[`([A-Z]{2,6}-[\w_]+)`\]', text) for ref in refs: try: itm = XML.xpath_node(all_docs, ".//*[@ref='"+ref+"']") cat_ref = "CAT-" + itm.tag if cat_ref in domain_cats: url = base_url + cat_ref + "/" + ref + "/content.html" else: url="" ttl = XML.xpath_plain(itm, "@title") md_link = f"""[`{ref}`]({url} "{ttl}")""" text = re.sub("\[`" + ref + "`\] *(\([^)]*\)|)", md_link, text) except: meta = XML.xpath_node(node, "ancestor::document/_meta") XML.add_error(meta, "__update_refs()", "warning", "unknown_ref", "unknown reference", 0, ref) return text
def __build_glossary_section(base_url, all_docs, github_url, domain_cats, domain_refs): """ Build an HTML section for glossary """ all_cats = XML.xpath_list(all_docs, "//category") cat_links={} for cat in all_cats: cat_ref = XML.xpath_plain(cat, "@ref") if (not(cat_ref in domain_cats)) : continue cat_title = XML.xpath_plain(cat, "@title") cat_about = XML.xpath_plain(cat, "about/text()") cat_links[cat_ref] = f""" <a href='{base_url}{cat_ref}/index.html' title="{cat_about}">{cat_title}</a> """ sorted_items = XML.xpath_alpha_sorted(all_docs, "//*[@ref]", "@title") rows="" for itm in sorted_items: itm_ref = XML.xpath_plain(itm, "@ref") if (not(itm_ref in domain_refs)) : continue itm_title = XML.xpath_plain(itm, "@title") itm_about_html = XML.xpath_md(itm, "about/text()") link_ref = __build_ref_link(base_url, all_docs, itm_ref) cat_ref = "CAT-" + itm.tag cat_link = cat_links.get(cat_ref,"") rows += f""" <tr> <td>{itm_title}</td> <td>{cat_link}</td> <td>{itm_about_html}</td> <td>{link_ref}</td> </tr> """ ret = f""" <div class="MenuContent"> <div id="primary"> <section class="comonSection singlefoliodetails"> <div class="container-fluid"> <div class="col-lg-12 noPadding"> <section class="col-md-10"> <div class="sectionTitle smTitle"> <h2>Glossary</h2> <div class="titleBar"></div> </div> <p>Alphabetical list ist all terms used in echOpen project.</p> <table class="table table-bordered"> <tr> <th>Term</th> <th>Category</th> <th>About</th> <th>Symbol</th> </tr> {rows} </table> </section> </div> </div> </section> </div> </div> """ return __comment_html(ret)
def __build_index_errors_section(base_url, all_docs, github_url): """ Build an HTML section for errors in all_docs. """ docs = XML.xpath_list(all_docs, "//document") nb_docs = len(docs) docs_error = XML.xpath_list(all_docs, "//document[_meta/error]") nb_docs_error = len(docs_error) rows = "" for doc_err in docs_error: src_path = os.path.join(XML.xpath_plain(doc_err, "@dir"), XML.xpath_plain(doc_err, "@file")) src_url = __to_github(github_url, src_path) all_errors = XML.xpath_list(doc_err, "_meta/error") rows += f""" <tr> <th colspan="6">{src_path} (<a href="{src_url}" target="_blank">Github</a>)</th> </tr> """ for err in all_errors: fct = XML.xpath_plain(err, "@fct") errlevel = XML.xpath_plain(err, "@errlevel") errno = XML.xpath_plain(err, "@errno") errmsg = XML.xpath_plain(err, "@errmsg") line = XML.xpath_plain(err, "@line") text = XML.xpath_plain(err, "text()") levelclass = {"fatal":"danger", "warning":"warning", "info":"info"}[errlevel] rows += f""" <tr class="{levelclass}"> <td>{errlevel}</td> <td>{errno}</td> <td>{errmsg}</td> <td>{line}</td> <td>{text}</td> <td>{fct}</td> </tr> """ ret = f""" <div class="MenuContent"> <div id="primary"> <section class="comonSection singlefoliodetails"> <div class="container-fluid"> <div class="col-lg-12 noPadding"> <section class="col-md-10"> <div class="sectionTitle smTitle"> <h2>Generation report</h2> <div class="titleBar"></div> </div> <h3>Overview</h3> <table class="table table-bordered"> <tr> <th></th> <th>With errors</th> <th>Total</th> </tr> <tr> <th>Parsed documents</th> <td>{nb_docs_error}</td> <td>{nb_docs}</td> </tr> </table> <h3>Error index</h3> <p>List all errors found during static site generation.</p> <table class="table table-bordered"> <tr> <th>errlevel</th> <th>errno</th> <th>errmsg</th> <th>line</th> <th>text</th> <th>fct</th> </tr> {rows} </table> </section> </div> </div> </section> </div> </div> """ return __comment_html(ret)