def formatted(self):
val = self.content
if val is not None:
extensions = [
tables.TableExtension(),
markdown_extensions.TocExtension(pod=self._doc.pod),
markdown_extensions.CodeBlockExtension(self._doc.pod),
markdown_extensions.IncludeExtension(self._doc.pod),
markdown_extensions.UrlExtension(self._doc.pod),
'markdown.extensions.fenced_code',
'markdown.extensions.codehilite',
]
config = self.get_markdown_config('markdown.extensions.codehilite')
codehilite_config = {
'pygments_style': 'default',
'noclasses': True,
'css_class': 'code',
}
if 'theme' in config:
codehilite_config['pygments_style'] = config.theme
if 'classes' in config:
codehilite_config['noclasses'] = not config.classes
if 'class_name' in config:
codehilite_config['css_class'] = config.class_name
extension_configs = {
'markdown.extensions.codehilite': codehilite_config,
}
val = markdown.markdown(
val.decode('utf-8'), extensions=extensions, extension_configs=extension_configs)
return val
def write_output(args):
converter = markdown.Markdown(
extensions=[
mdx_tables.TableExtension(),
mdx_sane_lists.SaneListExtension(),
mdx_smarty.SmartyExtension(),
mdx_toc.TocExtension(),
],
output_format='html5',
)
with args.input_path.open(encoding=args.encoding) as src_file:
body = converter.convert(src_file.read())
with tempfile.NamedTemporaryFile(
'w',
encoding=args.encoding,
dir=args.git_output.dir_path.as_posix(),
suffix='.html',
delete=False,
) as tmp_out:
try:
tmp_out.write(TEMPLATE_HEADER)
tmp_out.write(body)
tmp_out.write(TEMPLATE_FOOTER)
tmp_out.flush()
os.rename(tmp_out.name, str(args.output_file_path))
except BaseException:
os.unlink(tmp_out.name)
raise
if args.output_link_path.is_symlink():
args.output_link_path.unlink()
args.output_link_path.symlink_to(args.output_file_path.name)
def _md(x):
o = markdown.markdown(
x,
extensions=[
tables.TableExtension(),
fenced_code.FencedCodeExtension(),
sane_lists.SaneListExtension(),
markdown_math.MathExtension(),
],
)
return o
def html(self):
val = self.body
if val is not None:
extensions = [
tables.TableExtension(),
toc.TocExtension(),
markdown_extensions.CodeBlockExtension(),
markdown_extensions.IncludeExtension(self.doc.pod),
markdown_extensions.UrlExtension(self.doc.pod),
]
val = markdown.markdown(val.decode('utf-8'), extensions=extensions)
return val
def formatted(self):
val = self.content
if val is not None:
extensions = [
tables.TableExtension(),
markdown_extensions.TocExtension(pod=self._doc.pod),
markdown_extensions.CodeBlockExtension(self._doc.pod),
markdown_extensions.IncludeExtension(self._doc.pod),
markdown_extensions.UrlExtension(self._doc.pod),
]
val = markdown.markdown(val.decode('utf-8'), extensions=extensions)
return val
def render_to_html_string(self):
"""
Render the watched document to html, returning the html string
:return: The rendered HTML of the markdown file
"""
self.logger.debug('Rendering markdown (%s)', self.filename)
fake_file = BytesIO()
markdown.markdownFromFile(input=self.filename,
output=fake_file,
extensions=[abbr.AbbrExtension(), admonition.AdmonitionExtension(),
attr_list.AttrListExtension(), codehilite.CodeHiliteExtension(),
def_list.DefListExtension(), extra.ExtraExtension(),
fenced_code.FencedCodeExtension(), footnotes.FootnoteExtension(),
legacy_attrs.LegacyAttrExtension(), legacy_em.LegacyEmExtension(),
meta.MetaExtension(), nl2br.Nl2BrExtension(),
sane_lists.SaneListExtension(), smarty.SmartyExtension(),
tables.TableExtension(), toc.TocExtension(),
wikilinks.WikiLinkExtension()])
fake_file.seek(0)
return str(fake_file.read(), 'utf-8')
def extensions(self):
"""List of enabled extensions for the pod."""
# Do not cache property so that the extensions are created fresh.
extensions = [
tables.TableExtension(),
markdown_extensions.TocExtension(pod=self.pod),
markdown_extensions.CodeBlockExtension(self.pod),
markdown_extensions.IncludeExtension(self.pod),
markdown_extensions.UrlExtension(self.pod),
'markdown.extensions.fenced_code',
'markdown.extensions.codehilite',
]
for config in self.markdown_config:
if config['kind'] in extensions:
continue
if config['kind'].startswith('markdown.extensions'):
extensions.append(config['kind'])
return extensions
def update_articles(self):
'''
Updates articles, folders and categories
'''
# Find all characters that are not the os dir separator
base_depth = self.article_dir.count(os.sep)
# Loop through articles directory to find categories, folders and
# articles.
for cat in os.walk(self.article_dir, topdown=False):
cat_string = str(cat[0])
# Don't worry about the top level directory
if cat_string == self.article_dir:
continue
# Check if the category already has an ID
current_depth = cat_string.count(os.sep) - base_depth
matches = self.docid_check.search(cat_string)
if matches:
# We have an ID, parse out title and ID
parsed_name = self.docid_re.search(cat_string)
if current_depth == 1:
# Check the DOCID exists
category_info = self.categories.get(
int(parsed_name.group('docid')))
if category_info:
# Change the title if there is a discrepancy
if category_info['title'] != parsed_name.group(
'title'):
category_info['title'] = parsed_name.group('title')
# NOTE = Any consumer should change title
# to new 'title'
category_info['action'] = {
'action': 'TITLE_CHANGE'
}
else:
# Unknown DOCID - add it in
self.categories[parsed_name.group('docid')] = {
'title': parsed_name.group('title'),
'action': {
'action': 'TITLE_CHANGE'
}
}
elif current_depth == 2:
# Folder
# Check the DOCID exists
folder_info = self.folders.get(
int(parsed_name.group('docid')))
if folder_info:
# Change the title if there is a discrepancy
if folder_info['title'] != parsed_name.group('title'):
folder_info['title'] = parsed_name.group('title')
# NOTE = Any consumer should change title
# to new 'title'
folder_info['action'] = {'action': 'TITLE_CHANGE'}
else:
# Unknown DOCID - add it in
self.folders[int(parsed_name.group('docid'))] = {
'title': parsed_name.group('title'),
'action': {
'action': 'TITLE_CHANGE'
}
}
# Now, get the files in this folder, and check their names
for listing in os.walk(cat_string, topdown=False):
directory = cat_string
articles = listing[2]
for article in articles:
got_id = self.docid_check.search(article)
if got_id:
# We have an ID, parse out title and ID
article_info = self.docid_re.search(article)
# Check the DOCID exists
article_dict = self.articles.get(
int(article_info.group('docid')))
if article_dict:
# Change the title if there is a discrepancy
if article_dict[
'title'] != article_info.group(
'title'):
article_dict[
'title'] = article_info.group(
'title')
# NOTE = Any consumer should change title to new 'title'
article_dict['action'] = {
'action': 'TITLE_CHANGE'
}
else:
# Unknown DOCID - add it in
self.articles[int(
parsed_name.group('docid'))] = {
'title':
article_info.group('title'),
'action': {
'action': 'TITLE_CHANGE'
}
}
else:
# No ID, need a new one
# Flag that we need to update
article_info = self.title_re.search(article)
# Ignore any files that aren'd Markdown
if article_info.group('extension'):
# Update the latest article ID
self.counters['article'] += 1
# Flag renaming Category
old_name = '{dir}{sep}{title}{extension}'.format(
dir=directory,
sep=os.sep,
title=article_info.group('title'),
extension=article_info.group(
'extension'))
new_name = '{dir}{sep}{title}--DOCID{docid}{extension}'\
.format(
dir=directory,
sep=os.sep,
title=article_info.group('title'),
docid=self.counters['article'],
extension=article_info.group('extension')
)
# Add information about this article
self.articles[self.counters['article']] = {
'title': article_info.group('title'),
'action': {
'action': 'CREATE',
'from': old_name,
'to': new_name
}
}
else:
# No ID, just a title
# XXX Debugging
# Get the current relative depth
# 1 = Category
# 2 = Folder
parsed_name = self.title_re.search(cat_string)
if current_depth == 1:
# Category
# Flag that we need to update
# Update the latest cat ID
self.counters['category'] += 1
# Flag renaming Category
new_name = '{oldname}--DOCID{docid}'.format(
oldname=cat_string, docid=self.counters['category'])
# Add information about this category
self.categories[self.counters['category']] = {
'title': parsed_name.group('title'),
'action': {
'action': 'CREATE',
'from': cat_string,
'to': new_name
}
}
elif current_depth == 2:
# Folder
# Flag that we need to update
# Update the latest folder ID
self.counters['folder'] += 1
# Flag renaming Category
new_name = '{oldname}--DOCID{docid}'.format(
oldname=cat_string, docid=self.counters['folder'])
# Add information about this folder
self.folders[self.counters['folder']] = {
'title': parsed_name.group('title'),
'action': {
'action': 'CREATE',
'from': cat_string,
'to': new_name
}
}
# Now, get the files in this folder, and check their names
for listing in os.walk(cat_string, topdown=False):
directory = cat_string
articles = listing[2]
for article in articles:
got_id = self.docid_check.search(article)
if got_id:
# We have an ID, parse out title and ID
article_info = self.docid_re.search(article)
# Check the DOCID exists
article_dict = self.articles.get(
int(article_info.group('docid')))
if article_dict:
# Change the title if there is a discrepancy
if article_dict[
'title'] != article_info.group(
'title'):
article_dict[
'title'] = article_info.group(
'title')
# NOTE = Any consumer should change title to
# new 'title'
article_dict['action'] = {
'action': 'TITLE_CHANGE'
}
else:
# Unknown DOCID - add it in
self.articles[article_info.group(
'docid')] = {
'title':
article_info.group('title'),
'action': {
'action': 'TITLE_CHANGE'
}
}
else:
# No ID, need a new one
# Flag that we need to update
article_info = self.title_re.search(article)
if article_info.group('extension'):
# Update the latest article ID
self.counters['article'] += 1
# Flag renaming Article
oldname = '{dir}{sep}{title}'.format(
dir=directory,
sep=os.sep,
title=article_info.group('title'))
article_name =\
'{oldname}--DOCID{docid}{extension}'\
.format(
oldname=oldname,
docid=self.counters['article'],
extension=article_info.group('extension')
)
# Add information about this article
self.articles[self.counters['article']] = {
'title': article_info.group('title'),
'action': {
'action':
'CREATE',
'from':
'{name}{extension}'.format(
name=oldname,
extension=article_info.group(
'extension')),
'to':
article_name
}
}
else:
# Too deep, ignore
pass
# We now have all the information required to rename files and folders
# Start with articles
for aid, article in self.articles.items():
# Check if there is an action
action = article.get('action')
if action:
# Need to create a branch
self.require_change = True
# If it is a new file, we need to rename it with the new
# DOCID
if action['action'] == 'CREATE':
os.rename(action['from'], action['to'])
self.article_creations[aid] = True
self.require_change = True
elif action['action'] == 'TITLE_CHANGE':
self.article_updates[aid] = True
self.require_change = True
# Delete the actions
for i in self.article_creations:
del (self.articles[i]['action'])
for i in self.article_updates:
del (self.articles[i]['action'])
# Then to folders
for fid, folder in self.folders.items():
# Check if there is an action
action = folder.get('action')
if action:
# Need to create a branch
self.require_change = True
# If it is a new file, we need to rename it with the new
# DOCID
if action['action'] == 'CREATE':
os.rename(action['from'], action['to'])
self.folder_creations[fid] = True
self.require_change = True
if action['action'] == 'TITLE_CHANGE':
self.folder_updates[fid] = True
self.require_change = True
# Delete the actions
for i in self.folder_creations:
del (self.folders[i]['action'])
for i in self.folder_updates:
del (self.folders[i]['action'])
# Finally Categories
category_creations = []
for cid, category in self.categories.items():
# Check if there is an action
action = category.get('action')
if action:
# Need to create a branch
self.require_change = True
# If it is a new file, we need to rename it with the new
# DOCID
if action['action'] == 'CREATE':
os.rename(action['from'], action['to'])
self.category_creations[cid] = True
if action['action'] == 'TITLE_CHANGE':
self.category_updates[cid] = True
# Delete the actions
for i in self.category_creations:
del (self.categories[i]['action'])
for i in self.category_updates:
del (self.categories[i]['action'])
# Now that all IDS have been assigned, we can map parent IDS properly
for cat in os.walk(self.article_dir, topdown=False):
cat_string = str(cat[0])
# Don't worry about the top level directory
if cat_string == self.article_dir:
continue
# Check if the category already has an ID
current_depth = cat_string.count(os.sep) - base_depth
if current_depth == 1:
# Categories don't have parents, but we need to record that
# we found it for deletions.
matches = self.docid_re.search(cat_string)
#if not found, need to do something
cat_id = int(matches.group('docid'))
if cat_id in self.categories:
self.categories[cat_id]['found'] = True
elif current_depth == 2:
# Folder
matches = self.parentid_re.search(cat_string)
# Set the folder parent
folder_info = self.folders.get(int(matches.group('docid')))
folder_info['parent'] = int(matches.group('parent_docid'))
folder_info['found'] = True
# Now, get the files in this folder, and check their names
for listing in os.walk(cat_string, topdown=False):
directory = cat_string
articles = listing[2]
for article in articles:
article_info = self.docid_re.search(article)
# Only process if it is actually an article
# COULD be a .gitignore file/other file that
# we don't handle
if article_info:
# Add the parent folder...
tmp_article = self.articles[int(
article_info.group('docid'))]
tmp_article['parent'] = int(matches.group('docid'))
tmp_article['found'] = True
# Add a sha1sum of the file
with open(
'{directory}{sep}{name}'.format(
directory=cat_string,
sep=os.sep,
name=article), 'r') as f:
# Set up our extension
# Need to convert the file system path to an encoded URL
image_url = directory.replace(
self.article_dir, 'articles')
image_ext = imagelinkrewrite.ImageLinkRewriteExtension(
image_file_path=image_url)
table_ext = tables.TableExtension()
temp = f.read()
tmp_article['html'] =\
markdown(
temp,
extensions=[image_ext, table_ext],
output_format='html5'
)
tmp_article['sha1'] =\
sha1(tmp_article['html'].encode('utf-8')).hexdigest()
# Find the deleted and updated items
# Categories
for cid, cat in self.categories.items():
if cat.get('found'):
del (cat['found'])
if not cid in self.category_creations:
# Check for updates
if cat['title'] != self.orig_categories[cid]['title']:
self.category_updates[cid] = True
self.require_change = True
else:
self.category_deletions[cid] = True
self.require_change = True
# Folders
for fid, folder in self.folders.items():
if folder.get('found'):
del (folder['found'])
if not fid in self.folder_creations and fid in self.orig_folders:
# Check title and parent change
if\
folder['title'] != self.orig_folders[fid]['title']\
or\
folder['parent'] != self.orig_folders[fid]['parent']:
self.folder_updates[fid] = True
self.require_change = True
else:
# Deletion
self.folder_deletions[fid] = True
self.require_change = True
# Articles
for aid, article in self.articles.items():
if article.get('found'):
del (article['found'])
if not aid in self.article_creations:
log.debug('Got to here.... 1')
# Check for updates to content, title and parent
if\
article['sha1'] != self.orig_articles[int(aid)]['sha1']\
or\
article['title'] != self.orig_articles[int(aid)]['title']\
or\
article['parent'] != self.orig_articles[int(aid)]['parent']:
log.debug('Got to here.... 2')
self.article_updates[aid] = True
self.require_change = True
else:
log.debug('Got to here.... 3')
else:
self.article_deletions[aid] = True
self.require_change = True