def process_md_file(title, idx, original_path, proper_path):
proper_md_path = '/'.join(proper_path + [original_path.rsplit('/', 1)[-1]])
if proper_md_path == 'introduction/index.md':
proper_md_path = 'index.md'
print(locals())
if original_path != proper_md_path:
redirects[original_path] = proper_md_path
original_path = os.path.join(docs_dir, original_path)
proper_md_path = os.path.join(docs_dir, proper_md_path)
if os.path.exists(original_path):
meta, content = util.read_md_file(original_path)
else:
meta, content = util.read_md_file(proper_md_path)
meta['toc_title'] = title
meta['toc_priority'] = idx
if title == 'hidden':
meta['toc_hidden'] = True
for src, dst in redirects.items():
content = content.replace('(' + src, '(' + dst)
content = content.replace('../' + src, '../' + dst)
util.write_md_file(proper_md_path, meta, content)
if original_path != proper_md_path:
subprocess.check_call(f'git add {proper_md_path}', shell=True)
if os.path.exists(original_path):
subprocess.check_call(f'rm {original_path}', shell=True)
def build_nav_entry(root, args):
if root.endswith('images'):
return None, None, None
result_items = []
index_meta, index_content = util.read_md_file(
os.path.join(root, 'index.md'))
current_title = index_meta.get(
'toc_folder_title',
index_meta.get('toc_title', find_first_header(index_content)))
for filename in os.listdir(root):
path = os.path.join(root, filename)
if os.path.isdir(path):
prio, title, payload = build_nav_entry(path, args)
if title and payload:
result_items.append((prio, title, payload))
elif filename.endswith('.md'):
path = os.path.join(root, filename)
meta, content = util.read_md_file(path)
path = path.split('/', 2)[-1]
title = meta.get('toc_title', find_first_header(content))
if title:
title = title.strip().rstrip('.')
else:
title = meta.get('toc_folder_title', 'hidden')
prio = meta.get('toc_priority', 9999)
logging.debug(f'Nav entry: {prio}, {title}, {path}')
if not content.strip():
title = 'hidden'
if args.nav_limit and len(result_items) >= args.nav_limit:
break
result_items.append((prio, title, path))
result_items = sorted(result_items, key=lambda x: (x[0], x[1]))
result = collections.OrderedDict([(item[1], item[2])
for item in result_items])
return index_meta.get('toc_priority', 10000), current_title, result
def process_toc_entry(entry, path, idx):
if isinstance(entry, list):
for e in entry:
idx = process_toc_entry(e, path, idx)
elif isinstance(entry, dict):
for key, value in entry.items():
next_path = path + [make_key(key)]
index_md_idx = idx
idx += 1
if isinstance(value, list):
for v in value:
process_toc_entry(v, next_path, idx)
idx += 1
else:
process_md_file(key, idx, value, path)
idx += 1
index_md_path = os.path.join(docs_dir, '/'.join(next_path),
'index.md')
if os.path.exists(os.path.dirname(index_md_path)):
index_meta, index_content = util.read_md_file(index_md_path)
if not index_meta.get('toc_folder_title'):
index_meta['toc_folder_title'] = key
index_meta['toc_priority'] = index_md_idx
util.write_md_file(index_md_path, index_meta, index_content)
subprocess.check_call(f'git add {index_md_path}', shell=True)
return idx
def build_nav_entry(root, args):
if root.endswith("images"):
return None, None, None
result_items = []
index_meta, index_content = util.read_md_file(os.path.join(root, "index.md"))
current_title = index_meta.get("toc_folder_title", index_meta.get("toc_title"))
current_title = current_title or index_meta.get(
"title", find_first_header(index_content)
)
for filename in os.listdir(root):
path = os.path.join(root, filename)
if os.path.isdir(path):
prio, title, payload = build_nav_entry(path, args)
if title and payload:
result_items.append((prio, title, payload))
elif filename.endswith(".md"):
path = os.path.join(root, filename)
meta = ""
content = ""
try:
meta, content = util.read_md_file(path)
except:
print("Error in file: {}".format(path))
raise
path = path.split("/", 2)[-1]
title = meta.get("toc_title", find_first_header(content))
if title:
title = title.strip().rstrip(".")
else:
title = meta.get("toc_folder_title", "hidden")
prio = meta.get("toc_priority", 9999)
logging.debug(f"Nav entry: {prio}, {title}, {path}")
if meta.get("toc_hidden") or not content.strip():
title = "hidden"
if title == "hidden":
title = "hidden-" + hashlib.sha1(content.encode("utf-8")).hexdigest()
if args.nav_limit and len(result_items) >= args.nav_limit:
break
result_items.append((prio, title, path))
result_items = sorted(result_items, key=lambda x: (x[0], x[1]))
result = collections.OrderedDict([(item[1], item[2]) for item in result_items])
if index_meta.get("toc_hidden_folder"):
current_title += "|hidden-folder"
return index_meta.get("toc_priority", 10000), current_title, result
def build_blog_nav(lang, args):
blog_dir = os.path.join(args.blog_dir, lang)
years = sorted(os.listdir(blog_dir), reverse=True)
result_nav = [{"hidden": "index.md"}]
post_meta = collections.OrderedDict()
for year in years:
year_dir = os.path.join(blog_dir, year)
if not os.path.isdir(year_dir):
continue
result_nav.append({year: collections.OrderedDict()})
posts = []
post_meta_items = []
for post in os.listdir(year_dir):
post_path = os.path.join(year_dir, post)
if not post.endswith(".md"):
raise RuntimeError(
f"Unexpected non-md file in posts folder: {post_path}"
)
meta, _ = util.read_md_file(post_path)
post_date = meta["date"]
post_title = meta["title"]
if datetime.date.fromisoformat(post_date) > datetime.date.today():
continue
posts.append(
(
post_date,
post_title,
os.path.join(year, post),
)
)
if post_title in post_meta:
raise RuntimeError(f"Duplicate post title: {post_title}")
if not post_date.startswith(f"{year}-"):
raise RuntimeError(
f"Post date {post_date} doesn't match the folder year {year}: {post_title}"
)
post_url_part = post.replace(".md", "")
post_meta_items.append(
(
post_date,
{
"date": post_date,
"title": post_title,
"image": meta.get("image"),
"url": f"/blog/{lang}/{year}/{post_url_part}/",
},
)
)
for _, title, path in sorted(posts, reverse=True):
result_nav[-1][year][title] = path
for _, post_meta_item in sorted(
post_meta_items, reverse=True, key=lambda item: item[0]
):
post_meta[post_meta_item["title"]] = post_meta_item
return result_nav, post_meta
def sync_translation():
init_redirects()
for src, dst in redirects.items():
en_src = os.path.join(en_dir, src)
lang_src = os.path.join(docs_dir, src)
lang_dst = os.path.join(docs_dir, dst)
if os.path.exists(lang_src):
if os.path.islink(lang_src):
pass
else:
en_meta, en_content = util.read_md_file(en_src)
lang_meta, lang_content = util.read_md_file(lang_src)
en_meta.update(lang_meta)
for src_link, dst_link in redirects.items():
lang_content = lang_content.replace('(' + src_link, '(' + dst)
lang_content = lang_content.replace('../' + src_link, '../' + dst)
util.write_md_file(lang_dst, en_meta, lang_content)
subprocess.check_call(f'git add {lang_dst}', shell=True)
subprocess.check_call(f'rm {lang_src}', shell=True)
def build_blog_nav(lang, args):
blog_dir = os.path.join(args.blog_dir, lang)
years = sorted(os.listdir(blog_dir), reverse=True)
result_nav = [{'hidden': 'index.md'}]
post_meta = collections.OrderedDict()
for year in years:
year_dir = os.path.join(blog_dir, year)
if not os.path.isdir(year_dir):
continue
result_nav.append({year: collections.OrderedDict()})
posts = []
post_meta_items = []
for post in os.listdir(year_dir):
post_path = os.path.join(year_dir, post)
if not post.endswith('.md'):
raise RuntimeError(
f'Unexpected non-md file in posts folder: {post_path}')
meta, _ = util.read_md_file(post_path)
post_date = meta['date']
post_title = meta['title']
if datetime.date.fromisoformat(post_date) > datetime.date.today():
continue
posts.append((
post_date,
post_title,
os.path.join(year, post),
))
if post_title in post_meta:
raise RuntimeError(f'Duplicate post title: {post_title}')
if not post_date.startswith(f'{year}-'):
raise RuntimeError(
f'Post date {post_date} doesn\'t match the folder year {year}: {post_title}'
)
post_url_part = post.replace('.md', '')
post_meta_items.append((
post_date,
{
'date': post_date,
'title': post_title,
'image': meta.get('image'),
'url': f'/blog/{lang}/{year}/{post_url_part}/'
},
))
for _, title, path in sorted(posts, reverse=True):
result_nav[-1][year][title] = path
for _, post_meta_item in sorted(post_meta_items,
reverse=True,
key=lambda item: item[0]):
post_meta[post_meta_item['title']] = post_meta_item
return result_nav, post_meta
#!/usr/bin/env python3
import os
import subprocess
import sys
import util
if __name__ == '__main__':
path = sys.argv[1]
content_path = f'{path}.content'
meta_path = f'{path}.meta'
meta, content = util.read_md_file(path)
target_language = os.getenv('TARGET_LANGUAGE')
if target_language is not None and target_language != 'en':
rev = subprocess.check_output('git rev-parse HEAD',
shell=True).decode('utf-8').strip()
meta['machine_translated'] = True
meta['machine_translated_rev'] = rev
with open(content_path, 'w') as f:
print(content, file=f)
util.write_md_file(meta_path, meta, '')
