def __build_content_text(base_url, all_docs, node, github_url, domain_cats):
"""
Build an HTML text
/!\ Links to refs broken by Markdown transformation NEED to replace refs in md with good
links BEFORE transformation.
"""
raw_content = "" #### XML.xpath_plain(node, "../raw_content/text()")
if raw_content > "" :
ref = XML.xpath_plain(node, "@ref")
doc = XML.xpath_node(node, "ancestor::document")
src_path = os.path.join(XML.xpath_plain(doc, "@dir"), XML.xpath_plain(doc, "@file"))
src_url = __to_github(github_url, src_path)
github = f"""<a href="{src_url}" target="_blank">Github</a>"""
md_html = MD.markdown(raw_content)
ret = f"""
<div class="sectionTitle smTitle">
<h6>{ref} ({github})</h6>
<div class="titleBar"></div>
</div>
{md_html}
"""
else:
ret = __build_content_chapter(base_url, all_docs, 2, node, github_url, domain_cats)
return ret
def __build_content_gallery(base_url, all_docs, item):
"""
Build an HTML image gallery for given ref.
"""
try:
root_doc = XML.xpath_node(item, "ancestor::document")
root_dir = XML.xpath_plain(root_doc, "@dir")
exts = LIB.DOC_EXT["img"]
images = XML.xpath_list(root_doc, "_meta/attachment[contains('"+exts+"', @ext)]")
inner_html = ""
for image in images:
img_path = XML.xpath_plain(image, "@to_path")
img_file = XML.xpath_plain(image, "@file")
rel_path = os.path.relpath(img_path, start=root_dir)
link = f"""
<span class="icon"><a title="{img_file}" href="{rel_path}"><img alt="[image]" src="{rel_path}"/></a></span>
"""
inner_html += link
ret = f"""
<h2>Image gallery</h2>
<p>{inner_html}</p>
""" if inner_html > "" else ""
except:
ret = ""
return __comment_html(ret)
def __build_content_attachments(base_url, all_docs, item):
"""
Build an HTML attached documents list for given ref.
"""
try:
root_doc = XML.xpath_node(item, "ancestor::document")
root_dir = XML.xpath_plain(root_doc, "@dir")
exts = "|".join([LIB.DOC_EXT[t] for t in ["txt","doc","slide","data"]])
attachments = XML.xpath_list(root_doc, "_meta/attachment[contains('"+exts+"', @ext)]")
inner_html = ""
for att in attachments:
att_path = XML.xpath_plain(att, "@to_path")
att_title = XML.xpath_plain(att, "@file")
att_ext = XML.xpath_plain(att, "@ext")
rel_path = os.path.relpath(att_path, start=root_dir)
link = f"""
<a title="{att_title}" href='{rel_path}' target="_blank" class="link_for_{att_ext}">{att_title}</a>
"""
inner_html += link
ret = f"""
<h2>Attached documents</h2>
<p>{inner_html}</p>
""" if inner_html > "" else ""
except:
ret = ""
return __comment_html(ret)
def __store_with_attachments(dst_dir, filename, text, all_docs, ref, root_dir):
"""
Store text in dst_dir/filename, create dir if necessary and copy attachments.
"""
path = __store_file(dst_dir, filename, text)
root_doc = XML.xpath_node(all_docs, ".//document[.//*/@ref='"+ref+"']")
rel_dir = XML.xpath_plain(root_doc, "@dir")
src_dir = os.path.join(root_dir, rel_dir)
attachments = XML.xpath_list(root_doc, "_meta/attachment")
for att in attachments:
att_path = XML.xpath_plain(att, "@to_path")
att_rel_path = os.path.relpath(att_path, start=rel_dir)
src_att = os.path.join(src_dir, att_rel_path)
dst_att = os.path.join(dst_dir, att_rel_path)
dst_dir_att = os.path.dirname(dst_att)
if not os.path.exists(dst_dir_att):
os.makedirs(dst_dir_att)
try:
if not os.path.exists(dst_att):
print("Storing attachment : " + dst_att)
shutil.copyfile(src_att, dst_att)
except:
#LIB.debug_error()
meta = XML.xpath_node(all_docs, ".//document[.//*/@ref='"+ref+"']/_meta")
XML.add_error(meta, "__store_with_attachments()", "warning", "missing_file", "can't copy attachment, missing file", 0, src_att)
return path
def doc(root_dir, name):
"""
Parse a Markdown or CSV document file and return DOM.
"""
full_path = os.path.join(root_dir, name)
input_file = codecs.open(full_path, mode="r", encoding="utf-8")
dir, file = os.path.split(name)
forget, ext = os.path.splitext(name)
ext = ext[1:].lower()
xml_doc = ET.Element('document', {
"path": name,
"dir": dir,
"file": file,
"ext": ext
})
meta = ET.SubElement(xml_doc, "_meta")
if (ext == LIB.CSV_EXT):
try:
dom = __csv_parse(xml_doc, input_file.read())
except:
dom = __bin_parse(xml_doc, None)
elif (ext == LIB.MD_EXT):
#try:
dom = __md_parse(xml_doc, input_file.read())
#except:
dom = __bin_parse(xml_doc, None)
else: # other parsed as binary
dom = __bin_parse(xml_doc, None)
#build @ref, @title, @icon, @cat attributes
for root_node in XML.xpath_list(dom, "*[name][not(@is_template)]"):
ref = __extract_ref(XML.xpath_plain(root_node, "name/text()"))
if (ref > ""):
title = LIB.sanitize(XML.xpath_plain(root_node, "title/text()"))
icon = __extract_image(XML.xpath_plain(root_node, "text()"))
cat = "CAT-" + root_node.tag
root_node.set("ref", ref)
root_node.set("title", title)
root_node.set("icon", icon)
root_node.set("cat", cat)
#find all distinct ref patterns
text = " ".join(root_node.xpath(".//text()"))
ext_refs = set(re.findall('\[`([A-Z]{2,6}-[\w_]+)`\]', text))
for ext_ref in ext_refs:
if (ext_ref != ref):
XML.add_external_ref(meta, ext_ref, ref)
else:
XML.add_error(meta, "doc()", "warning", "ref_error",
"document reference syntax error", 0, ref)
return dom
def store_content_page(www_path, all_docs, item_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors):
"""
Store content page for item_ref entity in www_path/cat_ref/item_ref/content.html file.
"""
base_url = "../"
item_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+item_ref+"']/@cat")
entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat")
html_menu = __build_menu(base_url+"../", all_docs, item_cat_ref, domain_cats)
html_content = __build_content_section(base_url+"../", all_docs, item_ref, github_url, domain_cats, domain_refs)
html_view = __build_view(base_url+"../", html_menu, html_content, doc_entry_ref, entry_cat_ref, "doc", with_errors)
dst_dir = www_path + item_ref + "/"
return __store_with_attachments(dst_dir, "content.html", html_view, all_docs, item_ref, root_dir)
def __build_index_section(base_url, all_docs, cat_ref, github_url, domain_refs):
"""
Build an HTML section for ref entity in all_docs.
"""
cat = XML.xpath_node(all_docs, (".//*[@ref='"+cat_ref+"']"))
cat_title = XML.xpath_plain(cat, "title/text()")
cat_about_html = XML.xpath_md(cat, "about/text()")
all_items = XML.xpath_list(all_docs, ".//*[@cat='"+cat_ref+"']")
rows = ""
for itm in all_items:
itm_ref = XML.xpath_plain(itm, "@ref")
if (not (itm_ref in domain_refs)) : continue
itm_icon = XML.xpath_plain(itm, "@icon")
doc = XML.xpath_node(itm, "ancestor::document")
itm_src_path = os.path.join(XML.xpath_plain(doc, "@dir"), XML.xpath_plain(doc, "@file"))
itm_ttl_html = XML.xpath_md(itm, "@title")
itm_about_html = XML.xpath_md(itm, "about/text()")
itm_src_url = __to_github(github_url, itm_src_path)
link_ref = __build_ref_link(base_url, all_docs, itm_ref)
rows += f"""
<tr>
<td class="icon"><img class="icon" alt="[icon]" src='{itm_ref}/{itm_icon}'/></td>
<td>{link_ref}</td>
<td>{itm_ttl_html}</td>
<td>(<a href="{itm_src_url}" target="_blank">Github</a>)</td>
</tr>\n
"""
ret = f"""
<div class="MenuContent">
<div id="primary">
<section class="comonSection singlefoliodetails">
<div class="container-fluid">
<div class="col-lg-12 noPadding">
<section class="col-md-10">
<div class="sectionTitle smTitle">
<h2>{cat_title} index</h2>
<div class="titleBar"></div>
</div>
<p>{cat_about_html}</p>
<table class="table table-bordered">
{rows}
</table>
</section>
</div>
</div>
</section>
</div>
</div>
"""
return __comment_html(ret)
def __build_content_chapter(base_url, all_docs, level, node, github_url, domain_cats):
"""
Build recursively an HTML chapter at level for given node.
TODO : if level 2 produce :
<div class="sectionTitle smTitle">
<h2>configuration</h2>
<h6>CFG-sweeping_probe (<a href="https://github.com/Olivierff/PRJ-medtec_kit//tree/master/configurations/CFG-sweeping_probe" target="_blank">Github</a>)</h6>
<div class="titleBar"></div>
</div>
"""
try:
children = XML.xpath_list(node, "node()")
children_html = ""
for child in children:
if (ET.iselement(child)):
children_html += __build_content_chapter(base_url, all_docs, level+1, child, github_url, domain_cats)
else:
child = __update_refs(base_url, all_docs, child, node, domain_cats)
children_html += MD.markdown(child)
title = node.tag
if (level==2):
ref = XML.xpath_plain(node, "@ref")
doc = XML.xpath_node(node, "ancestor::document")
src_path = os.path.join(XML.xpath_plain(doc, "@dir"), XML.xpath_plain(doc, "@file"))
src_url = __to_github(github_url, src_path)
github = f"""<a href="{src_url}" target="_blank">Github</a>"""
ret = f"""
<div class="sectionTitle smTitle">
<h2>{title}</h2>
<h6>{ref} ({github})</h6>
<div class="titleBar"></div>
</div>
{children_html}
"""
else:
ret = f"""
<h{level}>{title}</h{level}>
{children_html}
"""
except:
ret = """
<p><strong>Error here !!! </strong></p>
"""
return ret
def __build_one_site(root_dir, all_docs, domain_refs, doc_entry_ref, www_root_dir, site_name, github_url, user_group):
"""
Build one autonomous static web site centered on doc_entry_ref and restricted
on domain_refs which is a set of authorized entities.
Allowed user_groups: #public, #devs, #core_team, #admin
"""
www_build_dir = www_root_dir + site_name + "/"
print("Reset build directory : " + www_build_dir)
shutil.rmtree(www_build_dir, ignore_errors=True)
try:
shutil.copytree("src/default_www/", www_build_dir)
except: # strange behaviour on Windows, try again ...
shutil.copytree("src/default_www/", www_build_dir)
all_domain_cats = { XML.xpath_plain(all_docs, ".//*[@ref='"+itm_ref+"']/@cat") for itm_ref in domain_refs } - {""}
domain_cats = []
for cat_ref in all_domain_cats:
cat_restrict = XML.xpath_plain(all_docs, ".//*[@ref='"+cat_ref+"']/@restricted_to")
if (cat_restrict == "") or (user_group in cat_restrict):
domain_cats += [cat_ref]
with_errors = user_group in "#devs,#admin"
path = HTM.store_home_page(www_build_dir, all_docs, "", root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors)
for cat_ref in domain_cats:
try:
path = HTM.store_index_page(www_build_dir, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors)
except:
LIB.debug_error()
LIB.debug("### Error for category index : ", cat_ref)
for itm_ref in domain_refs:
cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+itm_ref+"']/@cat")
if (cat_ref in domain_cats):
try:
path = HTM.store_content_page(www_build_dir+cat_ref+"/", all_docs, itm_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors)
except:
LIB.debug_error()
LIB.debug("### Error for file : ", itm)
path = HTM.store_glossary_page(www_build_dir, all_docs, "", root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors)
if with_errors:
path = HTM.store_error_page(www_build_dir, all_docs, "", root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors)
def store_glossary_page(www_path, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors):
"""
Store error page for ref category in www_path/error_log.html file.
"""
base_url = ""
entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat")
html_menu = __build_empty_menu()
html_glossary = __build_glossary_section(base_url, all_docs, github_url, domain_cats, domain_refs)
html_view = __build_view(base_url, html_menu, html_glossary, doc_entry_ref, entry_cat_ref, "gloss", with_errors)
dst_dir = www_path
return __store_with_attachments(dst_dir, "glossary.html", html_view, all_docs, "", root_dir)
def store_home_page(www_path, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors):
"""
Store index page for ref category in www_path/index.html file.
"""
base_url = ""
entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat")
html_menu = __build_empty_menu()
html_home = "<p>Landing page would be here !</p>"
html_view = __build_view(base_url, html_menu, html_home, doc_entry_ref, entry_cat_ref, "home", with_errors)
dst_dir = www_path
return __store_with_attachments(dst_dir, "index.html", html_view, all_docs, "", root_dir)
def store_index_page(www_path, all_docs, cat_ref, root_dir, github_url, doc_entry_ref, domain_cats, domain_refs, with_errors):
"""
Store index page for ref category in www_path/cat_ref/index.html file.
"""
base_url = "../"
entry_cat_ref = XML.xpath_plain(all_docs, ".//*[@ref='"+doc_entry_ref+"']/@cat")
html_menu = __build_menu(base_url, all_docs, cat_ref, domain_cats)
html_index = __build_index_section(base_url, all_docs, cat_ref, github_url, domain_refs)
html_view = __build_view(base_url, html_menu, html_index, doc_entry_ref, entry_cat_ref, "doc", with_errors)
dst_dir = www_path + cat_ref + "/"
return __store_with_attachments(dst_dir, "index.html", html_view, all_docs, "", root_dir)
def __build_menu(base_url, all_docs, cur_ref, domain_cats):
"""
Build an HTML menu part for ref entity in all_docs.
"""
sorted_cats = XML.xpath_num_sorted(all_docs, "//category", "menu/@position")
menu_items = ""
for cat in sorted_cats:
cat_ref = XML.xpath_plain(cat, "@ref")
if (not (cat_ref in domain_cats)) : continue
cat_menu = XML.xpath_plain(cat, "menu/text()")
is_active = ' class="active"' if cat_ref==cur_ref else ""
menu_items += f"""
<li{is_active}><a class="Menu" href='{base_url}{cat_ref}/index.html'>{cat_menu}</a></li>
"""
ret = f"""
<ul class="Menu">
{menu_items}
</ul>
"""
return __comment_html(ret)
def register_attachments(all_docs):
"""
register any attached "binary" document to its master document. Based on the file path
"""
attachments = XML.xpath_list(all_docs, "//document[count(*) <= 1]")
for att in attachments:
att_dir = XML.xpath_plain(att, "@dir")
att_path = XML.xpath_plain(att, "@path")
att_name = XML.xpath_plain(att, "@file")
att_ext = XML.xpath_plain(
att, "@ext"
) # vvvv Not very smart, but it's late and it works .... vvvv
root_pathes = XML.xpath_list(
all_docs,
"//document[count(*) > 1][@file='readme.md' or @file='Readme.md' or @file='README.md' or @file='readme.MD' or @file='Readme.MD' or @file='README.MD'][starts-with('"
+ att_dir + "', @dir)]/@path")
max_root_path = max(root_pathes, key=len, default="")
meta = XML.xpath_node(
all_docs, "//document[@path='" + max_root_path + "']/_meta")
XML.add_attachment(meta, att_path, att_ext, att_name)
return all_docs
def __build_ref_link(base_url, all_docs, ref):
"""
Build an HTML link for ref entity.
"""
try:
itm = XML.xpath_node(all_docs, ".//*[@ref='"+ref+"']")
cat_ref = "CAT-" + itm.tag
url = base_url + cat_ref + "/" + ref + "/content.html"
title = XML.xpath_plain(itm, "@title")
except:
url=""
title="Not found"
ret = f"""
<a href="{url}" title="{title}"><code>{ref}</code></a>
"""
return ret
def __update_refs(base_url, all_docs, text, node, domain_cats):
"""
Replace all Markdown link pattern [`CAT-xxxxxx`](../../interfaces/ITF-A_gnd) ) with correct url relative to base_url.
"""
#find all distinct ref patterns
refs = re.findall('\[`([A-Z]{2,6}-[\w_]+)`\]', text)
for ref in refs:
try:
itm = XML.xpath_node(all_docs, ".//*[@ref='"+ref+"']")
cat_ref = "CAT-" + itm.tag
if cat_ref in domain_cats:
url = base_url + cat_ref + "/" + ref + "/content.html"
else:
url=""
ttl = XML.xpath_plain(itm, "@title")
md_link = f"""[`{ref}`]({url} "{ttl}")"""
text = re.sub("\[`" + ref + "`\] *(\([^)]*\)|)", md_link, text)
except:
meta = XML.xpath_node(node, "ancestor::document/_meta")
XML.add_error(meta, "__update_refs()", "warning", "unknown_ref", "unknown reference", 0, ref)
return text
def __build_glossary_section(base_url, all_docs, github_url, domain_cats, domain_refs):
"""
Build an HTML section for glossary
"""
all_cats = XML.xpath_list(all_docs, "//category")
cat_links={}
for cat in all_cats:
cat_ref = XML.xpath_plain(cat, "@ref")
if (not(cat_ref in domain_cats)) : continue
cat_title = XML.xpath_plain(cat, "@title")
cat_about = XML.xpath_plain(cat, "about/text()")
cat_links[cat_ref] = f"""
<a href='{base_url}{cat_ref}/index.html' title="{cat_about}">{cat_title}</a>
"""
sorted_items = XML.xpath_alpha_sorted(all_docs, "//*[@ref]", "@title")
rows=""
for itm in sorted_items:
itm_ref = XML.xpath_plain(itm, "@ref")
if (not(itm_ref in domain_refs)) : continue
itm_title = XML.xpath_plain(itm, "@title")
itm_about_html = XML.xpath_md(itm, "about/text()")
link_ref = __build_ref_link(base_url, all_docs, itm_ref)
cat_ref = "CAT-" + itm.tag
cat_link = cat_links.get(cat_ref,"")
rows += f"""
<tr>
<td>{itm_title}</td>
<td>{cat_link}</td>
<td>{itm_about_html}</td>
<td>{link_ref}</td>
</tr>
"""
ret = f"""
<div class="MenuContent">
<div id="primary">
<section class="comonSection singlefoliodetails">
<div class="container-fluid">
<div class="col-lg-12 noPadding">
<section class="col-md-10">
<div class="sectionTitle smTitle">
<h2>Glossary</h2>
<div class="titleBar"></div>
</div>
<p>Alphabetical list ist all terms used in echOpen project.</p>
<table class="table table-bordered">
<tr>
<th>Term</th>
<th>Category</th>
<th>About</th>
<th>Symbol</th>
</tr>
{rows}
</table>
</section>
</div>
</div>
</section>
</div>
</div>
"""
return __comment_html(ret)
def __build_index_errors_section(base_url, all_docs, github_url):
"""
Build an HTML section for errors in all_docs.
"""
docs = XML.xpath_list(all_docs, "//document")
nb_docs = len(docs)
docs_error = XML.xpath_list(all_docs, "//document[_meta/error]")
nb_docs_error = len(docs_error)
rows = ""
for doc_err in docs_error:
src_path = os.path.join(XML.xpath_plain(doc_err, "@dir"), XML.xpath_plain(doc_err, "@file"))
src_url = __to_github(github_url, src_path)
all_errors = XML.xpath_list(doc_err, "_meta/error")
rows += f"""
<tr>
<th colspan="6">{src_path} (<a href="{src_url}" target="_blank">Github</a>)</th>
</tr>
"""
for err in all_errors:
fct = XML.xpath_plain(err, "@fct")
errlevel = XML.xpath_plain(err, "@errlevel")
errno = XML.xpath_plain(err, "@errno")
errmsg = XML.xpath_plain(err, "@errmsg")
line = XML.xpath_plain(err, "@line")
text = XML.xpath_plain(err, "text()")
levelclass = {"fatal":"danger", "warning":"warning", "info":"info"}[errlevel]
rows += f"""
<tr class="{levelclass}">
<td>{errlevel}</td>
<td>{errno}</td>
<td>{errmsg}</td>
<td>{line}</td>
<td>{text}</td>
<td>{fct}</td>
</tr>
"""
ret = f"""
<div class="MenuContent">
<div id="primary">
<section class="comonSection singlefoliodetails">
<div class="container-fluid">
<div class="col-lg-12 noPadding">
<section class="col-md-10">
<div class="sectionTitle smTitle">
<h2>Generation report</h2>
<div class="titleBar"></div>
</div>
<h3>Overview</h3>
<table class="table table-bordered">
<tr>
<th></th>
<th>With errors</th>
<th>Total</th>
</tr>
<tr>
<th>Parsed documents</th>
<td>{nb_docs_error}</td>
<td>{nb_docs}</td>
</tr>
</table>
<h3>Error index</h3>
<p>List all errors found during static site generation.</p>
<table class="table table-bordered">
<tr>
<th>errlevel</th>
<th>errno</th>
<th>errmsg</th>
<th>line</th>
<th>text</th>
<th>fct</th>
</tr>
{rows}
</table>
</section>
</div>
</div>
</section>
</div>
</div>
"""
return __comment_html(ret)
