def scan(self):
"""Create list of posts from POSTS and PAGES options."""
seen = set([])
if not self.site.quiet:
print("Scanning posts", end='', file=sys.stderr)
timeline = []
for wildcard, destination, template_name, use_in_feeds in \
self.site.config['post_pages']:
if not self.site.quiet:
print(".", end='', file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname, followlinks=True):
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname))) # output/destination/foo/
# Get all the untranslated paths
dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst
untranslated = glob.glob(dir_glob)
# And now get all the translated paths
translated = set([])
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == self.site.config['DEFAULT_LANG']:
continue
lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst
translated = translated.union(set(glob.glob(lang_glob)))
# untranslated globs like *.rst often match translated paths too, so remove them
# and ensure x.rst is not in the translated set
untranslated = set(untranslated) - translated
# also remove from translated paths that are translations of
# paths in untranslated_list, so x.es.rst is not in the untranslated set
for p in untranslated:
translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()])
full_list = list(translated) + list(untranslated)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.site.config,
dest_dir,
use_in_feeds,
self.site.MESSAGES,
template_name,
self.site.get_compiler(base_path)
)
timeline.append(post)
return timeline
def scan(self):
"""Create list of posts from POSTS and PAGES options."""
seen = set([])
if not self.site.quiet:
print("Scanning posts", end='', file=sys.stderr)
timeline = []
for wildcard, destination, template_name, use_in_feeds in \
self.site.config['post_pages']:
if not self.site.quiet:
print(".", end='', file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname, followlinks=True):
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname))) # output/destination/foo/
# Get all the untranslated paths
dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst
untranslated = glob.glob(dir_glob)
# And now get all the translated paths
translated = set([])
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == self.site.config['DEFAULT_LANG']:
continue
lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst
translated = translated.union(set(glob.glob(lang_glob)))
# untranslated globs like *.rst often match translated paths too, so remove them
# and ensure x.rst is not in the translated set
untranslated = set(untranslated) - translated
# also remove from translated paths that are translations of
# paths in untranslated_list, so x.es.rst is not in the untranslated set
for p in untranslated:
translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()])
full_list = list(translated) + list(untranslated)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.site.config,
dest_dir,
use_in_feeds,
self.site.MESSAGES,
template_name,
self.site.get_compiler(base_path)
)
timeline.append(post)
return timeline
def find_metadata(self, gallery, lang):
"""Search for a gallery metadata file.
If there is an metadata file for the gallery, use that to determine
captions and the order in which images shall be displayed in the
gallery. You only need to list the images if a specific ordering or
caption is required. The metadata file is YAML-formatted, with field
names of
#
name:
caption:
order:
#
If a numeric order value is specified, we use that directly, otherwise
we depend on how PyYAML returns the information - which may or may not
be in the same order as in the file itself. Non-numeric ordering is not
supported. If no caption is specified, then we return an empty string.
Returns a string (l18n'd filename), list (ordering), dict (captions),
dict (image metadata).
"""
base_meta_path = os.path.join(gallery, "metadata.yml")
localized_meta_path = utils.get_translation_candidate(self.site.config,
base_meta_path, lang)
order = []
captions = {}
custom_metadata = {}
used_path = ""
if os.path.isfile(localized_meta_path):
used_path = localized_meta_path
elif os.path.isfile(base_meta_path):
used_path = base_meta_path
else:
return "", [], {}, {}
self.logger.debug("Using {0} for gallery {1}".format(
used_path, gallery))
with open(used_path, "r", encoding='utf-8-sig') as meta_file:
if yaml is None:
utils.req_missing(['PyYAML'], 'use metadata.yml files for galleries')
meta = yaml.safe_load_all(meta_file)
for img in meta:
# load_all and safe_load_all both return None as their
# final element, so skip it
if not img:
continue
if 'name' in img:
img_name = img.pop('name')
if 'caption' in img and img['caption']:
captions[img_name] = img.pop('caption')
if 'order' in img and img['order'] is not None:
order.insert(img.pop('order'), img_name)
else:
order.append(img_name)
custom_metadata[img_name] = img
else:
self.logger.error("no 'name:' for ({0}) in {1}".format(
img, used_path))
return used_path, order, captions, custom_metadata
def find_metadata(self, gallery, lang):
"""Search for a gallery metadata file.
If there is an metadata file for the gallery, use that to determine
captions and the order in which images shall be displayed in the
gallery. You only need to list the images if a specific ordering or
caption is required. The metadata file is YAML-formatted, with field
names of
#
name:
caption:
order:
#
If a numeric order value is specified, we use that directly, otherwise
we depend on how PyYAML returns the information - which may or may not
be in the same order as in the file itself. Non-numeric ordering is not
supported. If no caption is specified, then we return an empty string.
Returns a string (l18n'd filename), list (ordering), dict (captions),
dict (image metadata).
"""
base_meta_path = os.path.join(gallery, "metadata.yml")
localized_meta_path = utils.get_translation_candidate(self.site.config,
base_meta_path, lang)
order = []
captions = {}
custom_metadata = {}
used_path = ""
if os.path.isfile(localized_meta_path):
used_path = localized_meta_path
elif os.path.isfile(base_meta_path):
used_path = base_meta_path
else:
return "", [], {}, {}
self.logger.debug("Using {0} for gallery {1}".format(
used_path, gallery))
with open(used_path, "r") as meta_file:
if yaml is None:
utils.req_missing(['PyYAML'], 'use metadata.yml files for galleries')
meta = yaml.safe_load_all(meta_file)
for img in meta:
# load_all and safe_load_all both return None as their
# final element, so skip it
if not img:
continue
if 'name' in img:
img_name = img.pop('name')
if 'caption' in img and img['caption']:
captions[img_name] = img.pop('caption')
if 'order' in img and img['order'] is not None:
order.insert(img.pop('order'), img_name)
else:
order.append(img_name)
custom_metadata[img_name] = img
else:
self.logger.error("no 'name:' for ({0}) in {1}".format(
img, used_path))
return used_path, order, captions, custom_metadata
def gen_tasks(self):
"""Publish the page sources into the output.
Required keyword arguments:
translations
default_lang
post_pages
output_folder
"""
kw = {
"translations": self.site.config["TRANSLATIONS"],
"output_folder": self.site.config["OUTPUT_FOLDER"],
"default_lang": self.site.config["DEFAULT_LANG"],
"show_untranslated_posts":
self.site.config['SHOW_UNTRANSLATED_POSTS'],
}
self.site.scan_posts()
yield self.group_task()
if self.site.config['COPY_SOURCES']:
for lang in kw["translations"]:
for post in self.site.timeline:
if not kw[
"show_untranslated_posts"] and lang not in post.translated_to:
continue
if post.meta('password'):
continue
output_name = os.path.join(
kw['output_folder'],
post.destination_path(lang, post.source_ext(True)))
# do not publish PHP sources
if post.source_ext(True) == post.compiler.extension():
continue
source = post.source_path
if lang != kw["default_lang"]:
source_lang = utils.get_translation_candidate(
self.site.config, source, lang)
if os.path.exists(source_lang):
source = source_lang
if os.path.isfile(source):
yield {
'basename':
'render_sources',
'name':
os.path.normpath(output_name),
'file_dep': [source],
'targets': [output_name],
'actions':
[(utils.copy_file, (source, output_name))],
'clean':
True,
'uptodate': [
utils.config_changed(
kw, 'nikola.plugins.task.sources')
],
}
def test_get_translation_candidate(pattern, path, lang, expected_path):
config = {
"TRANSLATIONS_PATTERN": pattern,
"DEFAULT_LANG": "en",
"TRANSLATIONS": {
"es": "1",
"en": 1
},
}
assert get_translation_candidate(config, path, lang) == expected_path
def _execute(self, options, args):
L = utils.get_logger('upgrade_metadata', utils.STDERR_HANDLER)
nikola.post._UPGRADE_METADATA_ADVERTISED = True
# scan posts
self.site.scan_posts()
flagged = []
for post in self.site.timeline:
if not post.newstylemeta:
flagged.append(post)
if flagged:
if len(flagged) == 1:
L.info(
'1 post (and/or its translations) contains old-style metadata:'
)
else:
L.info(
'{0} posts (and/or their translations) contain old-style metadata:'
.format(len(flagged)))
for post in flagged:
L.info(' ' + post.metadata_path)
if not options['yes']:
yesno = utils.ask_yesno("Proceed with metadata upgrade?")
if options['yes'] or yesno:
for post in flagged:
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == post.default_lang:
fname = post.metadata_path
else:
meta_path = os.path.splitext(
post.source_path)[0] + '.meta'
fname = utils.get_translation_candidate(
post.config, meta_path, lang)
if os.path.exists(fname):
with io.open(fname, 'r', encoding='utf-8') as fh:
meta = fh.readlines()
if not meta[min(1,
len(meta) - 1)].startswith('.. '):
# check if we’re dealing with old style metadata
with io.open(fname, 'w',
encoding='utf-8') as fh:
for k, v in zip(self.fields, meta):
fh.write('.. {0}: {1}'.format(k, v))
L.debug(fname)
L.info('{0} posts upgraded.'.format(len(flagged)))
else:
L.info('Metadata not upgraded.')
else:
L.info(
'No old-style metadata posts found. No action is required.')
def gen_tasks(self):
"""Publish the page sources into the output.
Required keyword arguments:
translations
default_lang
post_pages
output_folder
"""
kw = {
"translations": self.site.config["TRANSLATIONS"],
"output_folder": self.site.config["OUTPUT_FOLDER"],
"default_lang": self.site.config["DEFAULT_LANG"],
"show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
}
self.site.scan_posts()
yield self.group_task()
if self.site.config['COPY_SOURCES']:
for lang in kw["translations"]:
for post in self.site.timeline:
if not kw["show_untranslated_posts"] and lang not in post.translated_to:
continue
if post.meta('password'):
continue
output_name = os.path.join(
kw['output_folder'], post.destination_path(
lang, post.source_ext(True)))
# do not publish PHP sources
if post.source_ext(True) == post.compiler.extension():
continue
source = post.source_path
if lang != kw["default_lang"]:
source_lang = utils.get_translation_candidate(self.site.config, source, lang)
if os.path.exists(source_lang):
source = source_lang
if os.path.isfile(source):
yield {
'basename': 'render_sources',
'name': os.path.normpath(output_name),
'file_dep': [source],
'targets': [output_name],
'actions': [(utils.copy_file, (source, output_name))],
'clean': True,
'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.sources')],
}
def _execute(self, options, args):
L = utils.get_logger('upgrade_metadata', utils.STDERR_HANDLER)
nikola.post._UPGRADE_METADATA_ADVERTISED = True
# scan posts
self.site.scan_posts()
flagged = []
for post in self.site.timeline:
if not post.newstylemeta:
flagged.append(post)
if flagged:
if len(flagged) == 1:
L.info('1 post (and/or its translations) contains old-style metadata:')
else:
L.info('{0} posts (and/or their translations) contain old-style metadata:'.format(len(flagged)))
for post in flagged:
L.info(' ' + post.metadata_path)
if not options['yes']:
yesno = utils.ask_yesno("Proceed with metadata upgrade?")
if options['yes'] or yesno:
for post in flagged:
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == post.default_lang:
fname = post.metadata_path
else:
meta_path = os.path.splitext(post.source_path)[0] + '.meta'
fname = utils.get_translation_candidate(post.config, meta_path, lang)
if os.path.exists(fname):
with io.open(fname, 'r', encoding='utf-8') as fh:
meta = fh.readlines()
if not meta[min(1, len(meta) - 1)].startswith('.. '):
# check if we’re dealing with old style metadata
with io.open(fname, 'w', encoding='utf-8') as fh:
for k, v in zip(self.fields, meta):
fh.write('.. {0}: {1}'.format(k, v))
L.debug(fname)
L.info('{0} posts upgraded.'.format(len(flagged)))
else:
L.info('Metadata not upgraded.')
else:
L.info('No old-style metadata posts found. No action is required.')
def import_item(self, item, wordpress_namespace, out_folder=None):
"""Takes an item from the feed and creates a post file."""
if out_folder is None:
out_folder = 'posts'
title = get_text_tag(item, 'title', 'NO TITLE')
# link is something like http://foo.com/2012/09/01/hello-world/
# So, take the path, utils.slugify it, and that's our slug
link = get_text_tag(item, 'link', None)
path = unquote(urlparse(link).path.strip('/'))
# In python 2, path is a str. slug requires a unicode
# object. According to wikipedia, unquoted strings will
# usually be UTF8
if isinstance(path, utils.bytes_str):
path = path.decode('utf8')
pathlist = path.split('/')
if len(pathlist) > 1:
out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
slug = utils.slugify(pathlist[-1])
if not slug: # it happens if the post has no "nice" URL
slug = get_text_tag(
item, '{{{0}}}post_name'.format(wordpress_namespace), None)
if not slug: # it *may* happen
slug = get_text_tag(
item, '{{{0}}}post_id'.format(wordpress_namespace), None)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return
description = get_text_tag(item, 'description', '')
post_date = get_text_tag(
item, '{{{0}}}post_date'.format(wordpress_namespace), None)
dt = utils.to_datetime(post_date)
if dt.tzinfo and self.timezone is None:
self.timezone = utils.get_tzname(dt)
status = get_text_tag(
item, '{{{0}}}status'.format(wordpress_namespace), 'publish')
content = get_text_tag(
item, '{http://purl.org/rss/1.0/modules/content/}encoded', '')
tags = []
if status == 'trash':
LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title))
return
elif status != 'publish':
tags.append('draft')
is_draft = True
else:
is_draft = False
for tag in item.findall('category'):
text = tag.text
if text == 'Uncategorized':
continue
tags.append(text)
if '$latex' in content:
tags.append('mathjax')
if is_draft and self.exclude_drafts:
LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
elif content.strip():
# If no content is found, no files are written.
self.url_map[link] = (self.context['SITE_URL'] + out_folder + '/'
+ slug + '.html')
if hasattr(self, "separate_qtranslate_content") \
and self.separate_qtranslate_content:
content_translations = separate_qtranslate_content(content)
else:
content_translations = {"": content}
default_language = self.context["DEFAULT_LANG"]
for lang, content in content_translations.items():
if lang:
out_meta_filename = slug + '.meta'
if lang == default_language:
out_content_filename = slug + '.wp'
else:
out_content_filename \
= utils.get_translation_candidate(self.context,
slug + ".wp", lang)
self.extra_languages.add(lang)
meta_slug = slug
else:
out_meta_filename = slug + '.meta'
out_content_filename = slug + '.wp'
meta_slug = slug
content = self.transform_content(content)
self.write_metadata(os.path.join(self.output_folder, out_folder,
out_meta_filename),
title, meta_slug, post_date, description, tags)
self.write_content(
os.path.join(self.output_folder,
out_folder, out_content_filename),
content)
else:
LOGGER.warn('Not going to import "{0}" because it seems to contain'
' no content.'.format(title))
def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None):
"""Take an item from the feed and creates a post file."""
if out_folder is None:
out_folder = "posts"
title = get_text_tag(item, "title", "NO TITLE")
# link is something like http://foo.com/2012/09/01/hello-world/
# So, take the path, utils.slugify it, and that's our slug
link = get_text_tag(item, "link", None)
parsed = urlparse(link)
path = unquote(parsed.path.strip("/"))
try:
if isinstance(path, utils.bytes_str):
path = path.decode("utf8", "replace")
else:
path = path
except AttributeError:
pass
# Cut out the base directory.
if path.startswith(self.base_dir.strip("/")):
path = path.replace(self.base_dir.strip("/"), "", 1)
pathlist = path.split("/")
if parsed.query: # if there are no nice URLs and query strings are used
out_folder = os.path.join(*([out_folder] + pathlist))
slug = get_text_tag(item, "{{{0}}}post_name".format(wordpress_namespace), None)
if not slug: # it *may* happen
slug = get_text_tag(item, "{{{0}}}post_id".format(wordpress_namespace), None)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return False
else:
if len(pathlist) > 1:
out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
slug = utils.slugify(pathlist[-1])
description = get_text_tag(item, "description", "")
post_date = get_text_tag(item, "{{{0}}}post_date".format(wordpress_namespace), None)
try:
dt = utils.to_datetime(post_date)
except ValueError:
dt = datetime.datetime(1970, 1, 1, 0, 0, 0)
LOGGER.error(
'Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(
post_date, title, slug
)
)
post_date = dt.strftime("%Y-%m-%d %H:%M:%S")
if dt.tzinfo and self.timezone is None:
self.timezone = utils.get_tzname(dt)
status = get_text_tag(item, "{{{0}}}status".format(wordpress_namespace), "publish")
content = get_text_tag(item, "{http://purl.org/rss/1.0/modules/content/}encoded", "")
excerpt = get_text_tag(item, "{http://wordpress.org/export/1.2/excerpt/}encoded", None)
if excerpt is not None:
if len(excerpt) == 0:
excerpt = None
tags = []
categories = []
if status == "trash":
LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title))
return False
elif status == "private":
tags.append("private")
is_draft = False
is_private = True
elif status != "publish":
tags.append("draft")
is_draft = True
is_private = False
else:
is_draft = False
is_private = False
for tag in item.findall("category"):
text = tag.text
type = "category"
if "domain" in tag.attrib:
type = tag.attrib["domain"]
if text == "Uncategorized" and type == "category":
continue
if type == "category":
categories.append(text)
else:
tags.append(text)
if "$latex" in content:
tags.append("mathjax")
for i, cat in enumerate(categories[:]):
cat = self._sanitize(cat, True)
categories[i] = cat
self.all_tags.add(cat)
for i, tag in enumerate(tags[:]):
tag = self._sanitize(tag, False)
tags[i] = tag
self.all_tags.add(tag)
# Find post format if it's there
post_format = "wp"
format_tag = [x for x in item.findall("*//{%s}meta_key" % wordpress_namespace) if x.text == "_tc_post_format"]
if format_tag:
post_format = format_tag[0].getparent().find("{%s}meta_value" % wordpress_namespace).text
if post_format == "wpautop":
post_format = "wp"
if is_draft and self.exclude_drafts:
LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
return False
elif is_private and self.exclude_privates:
LOGGER.notice('Private post "{0}" will not be imported.'.format(title))
return False
elif content.strip() or self.import_empty_items:
# If no content is found, no files are written.
self.url_map[link] = (self.context["SITE_URL"] + out_folder.rstrip("/") + "/" + slug + ".html").replace(
os.sep, "/"
)
if hasattr(self, "separate_qtranslate_content") and self.separate_qtranslate_content:
content_translations = separate_qtranslate_content(content)
else:
content_translations = {"": content}
default_language = self.context["DEFAULT_LANG"]
for lang, content in content_translations.items():
try:
content, extension, rewrite_html = self.transform_content(content, post_format, attachments)
except:
LOGGER.error(
('Cannot interpret post "{0}" (language {1}) with post ' + "format {2}!").format(
os.path.join(out_folder, slug), lang, post_format
)
)
return False
if lang:
out_meta_filename = slug + ".meta"
if lang == default_language:
out_content_filename = slug + "." + extension
else:
out_content_filename = utils.get_translation_candidate(
self.context, slug + "." + extension, lang
)
self.extra_languages.add(lang)
meta_slug = slug
else:
out_meta_filename = slug + ".meta"
out_content_filename = slug + "." + extension
meta_slug = slug
tags, other_meta = self._create_metadata(
status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug)
)
self.write_metadata(
os.path.join(self.output_folder, out_folder, out_meta_filename),
title,
meta_slug,
post_date,
description,
tags,
**other_meta
)
self.write_content(
os.path.join(self.output_folder, out_folder, out_content_filename), content, rewrite_html
)
if self.export_comments:
comments = []
for tag in item.findall("{{{0}}}comment".format(wordpress_namespace)):
comment = self._extract_comment(tag, wordpress_namespace)
if comment is not None:
comments.append(comment)
for comment in comments:
comment_filename = "{0}.{1}.wpcomment".format(slug, comment["id"])
self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment)
return (out_folder, slug)
else:
LOGGER.warn(('Not going to import "{0}" because it seems to contain' " no content.").format(title))
return False
def scan(self):
"""Create list of posts from HIERARCHICAL_PAGES options."""
seen = set([])
if not self.site.quiet:
print("Scanning hierarchical pages", end='', file=sys.stderr)
timeline = []
for wildcard, destination, template_name in self.site.config.get('HIERARCHICAL_PAGES', []):
if not self.site.quiet:
print(".", end='', file=sys.stderr)
root = Node(slugs=_spread(destination, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG']))
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname, followlinks=True):
# Get all the untranslated paths
dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst
untranslated = glob.glob(dir_glob)
# And now get all the translated paths
translated = set([])
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == self.site.config['DEFAULT_LANG']:
continue
lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst
translated = translated.union(set(glob.glob(lang_glob)))
# untranslated globs like *.rst often match translated paths too, so remove them
# and ensure x.rst is not in the translated set
untranslated = set(untranslated) - translated
# also remove from translated paths that are translations of
# paths in untranslated_list, so x.es.rst is not in the untranslated set
for p in untranslated:
translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()])
full_list = list(translated) + list(untranslated)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
# Extract path
path = utils.os_path_split(os.path.relpath(base_path, dirname))
path[-1] = os.path.splitext(path[-1])[0]
if path[-1] == 'index':
path = path[:-1]
# Find node
node = root
for path_elt in path:
if path_elt not in node.children:
node.children[path_elt] = Node(path_elt)
node = node.children[path_elt]
node.post_source = base_path
# Add posts
def crawl(node, destinations_so_far, root=True):
if node.post_source is not None:
try:
post = Post(
node.post_source,
self.site.config,
'',
False,
self.site.MESSAGES,
template_name,
self.site.get_compiler(node.post_source),
destination_base=utils.TranslatableSetting('destinations', destinations_so_far, self.site.config['TRANSLATIONS']),
metadata_extractors_by=self.site.metadata_extractors_by
)
timeline.append(post)
except Exception as err:
LOGGER.error('Error reading post {}'.format(base_path))
raise err
# Compute slugs
slugs = {}
for lang in self.site.config['TRANSLATIONS']:
slug = post.meta('slug', lang=lang)
if slug:
slugs[lang] = slug
if not slugs:
slugs[self.site.config['DEFAULT_LANG']] = node.name
node.slugs = _spread(slugs, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG'])
# Update destinations_so_far
if not root:
if node.slugs is not None:
destinations_so_far = {lang: os.path.join(dest, node.slugs[lang]) for lang, dest in destinations_so_far.items()}
else:
destinations_so_far = {lang: os.path.join(dest, node.name) for lang, dest in destinations_so_far.items()}
for p, n in node.children.items():
crawl(n, destinations_so_far, root=False)
crawl(root, root.slugs)
return timeline
def scan(self):
"""Create list of posts from HIERARCHICAL_PAGES options."""
seen = set([])
if not self.site.quiet:
print("Scanning hierarchical pages", end='', file=sys.stderr)
timeline = []
for wildcard, destination, template_name in self.site.config.get(
'HIERARCHICAL_PAGES', []):
if not self.site.quiet:
print(".", end='', file=sys.stderr)
root = Node(
slugs=_spread(destination, self.site.config['TRANSLATIONS'],
self.site.config['DEFAULT_LANG']))
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname, followlinks=True):
# Get all the untranslated paths
dir_glob = os.path.join(
dirpath, os.path.basename(wildcard)) # posts/foo/*.rst
untranslated = glob.glob(dir_glob)
# And now get all the translated paths
translated = set([])
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == self.site.config['DEFAULT_LANG']:
continue
lang_glob = utils.get_translation_candidate(
self.site.config, dir_glob,
lang) # posts/foo/*.LANG.rst
translated = translated.union(set(glob.glob(lang_glob)))
# untranslated globs like *.rst often match translated paths too, so remove them
# and ensure x.rst is not in the translated set
untranslated = set(untranslated) - translated
# also remove from translated paths that are translations of
# paths in untranslated_list, so x.es.rst is not in the untranslated set
for p in untranslated:
translated = translated - set([
utils.get_translation_candidate(
self.site.config, p, l)
for l in self.site.config['TRANSLATIONS'].keys()
])
full_list = list(translated) + list(untranslated)
# We eliminate from the list the files inside any .ipynb folder
full_list = [
p for p in full_list
if not any([x.startswith('.') for x in p.split(os.sep)])
]
for base_path in sorted(full_list):
if base_path in seen:
continue
for lang in self.site.config['TRANSLATIONS'].keys():
seen.add(
utils.get_translation_candidate(
self.site.config, base_path, lang))
# Extract path
path = utils.os_path_split(
os.path.relpath(base_path, dirname))
path[-1] = os.path.splitext(path[-1])[0]
if path[-1] == 'index':
path = path[:-1]
# Find node
node = root
for path_elt in path:
if path_elt not in node.children:
node.children[path_elt] = Node(path_elt)
node = node.children[path_elt]
node.post_source = base_path
# Add posts
def crawl(node, destinations_so_far, root=True):
if node.post_source is not None:
try:
post = Post(node.post_source,
self.site.config,
'',
False,
self.site.MESSAGES,
template_name,
self.site.get_compiler(node.post_source),
destination_base=utils.TranslatableSetting(
'destinations', destinations_so_far,
self.site.config['TRANSLATIONS']))
timeline.append(post)
except Exception as err:
LOGGER.error('Error reading post {}'.format(base_path))
raise err
# Compute slugs
slugs = {}
for lang in self.site.config['TRANSLATIONS']:
slug = post.meta('slug', lang=lang)
if slug:
slugs[lang] = slug
if not slugs:
slugs[self.site.config['DEFAULT_LANG']] = node.name
node.slugs = _spread(slugs,
self.site.config['TRANSLATIONS'],
self.site.config['DEFAULT_LANG'])
# Update destinations_so_far
if not root:
if node.slugs is not None:
destinations_so_far = {
lang: os.path.join(dest, node.slugs[lang])
for lang, dest in destinations_so_far.items()
}
else:
destinations_so_far = {
lang: os.path.join(dest, node.name)
for lang, dest in destinations_so_far.items()
}
for p, n in node.children.items():
crawl(n, destinations_so_far, root=False)
crawl(root, root.slugs)
return timeline
def gen_tasks(self):
self.kw = {
'strip_indexes': self.site.config['STRIP_INDEXES'],
'output_folder': self.site.config['OUTPUT_FOLDER'],
'cache_folder': self.site.config['CACHE_FOLDER'],
'default_lang': self.site.config['DEFAULT_LANG'],
'filters': self.site.config['FILTERS'],
'translations': self.site.config['TRANSLATIONS'],
'global_context': self.site.GLOBAL_CONTEXT,
'tzinfo': self.site.tzinfo,
}
tasks = {}
classes = []
folder_name = "learning"
directory = os.path.join(self.site.original_cwd, folder_name)
template_name = "tutorials.mako"
categories = []
dirs = os.listdir(directory)
dirs.sort()
files = []
for catfolder in dirs:
if not os.path.isdir(os.path.join(directory,catfolder)):
continue
articles = []
translations = []
category = catfolder[catfolder.find("_")+1:]
articlesfiles = os.listdir(os.path.join(directory,catfolder));
articlesfiles.sort()
for article in articlesfiles:
file_split = os.path.splitext(article)
extension = file_split[1]
lang_split = article.split(".")
lang = self.site.config['DEFAULT_LANG'] if len(lang_split)<3 else lang_split[1]
is_translation = lang != self.site.config['DEFAULT_LANG']
folder = os.path.join(directory,catfolder,article)
if extension=='.markdown':
path = os.path.join(directory,catfolder,article)
files += [path]
articleobj = MarkdownArticle(path, directory, lang, is_translation)
if not is_translation:
articles.append(articleobj)
else:
translations.append(articleobj)
elif extension=='.asciidoc':
path = os.path.join(directory,catfolder,article)
files += [path]
articleobj = AsciidocArticle(path, directory, lang, is_translation)
if not is_translation:
articles.append(articleobj)
else:
translations.append(articleobj)
elif os.path.isdir(folder):
for lang in self.kw['translations']:
if lang == self.site.config['DEFAULT_LANG']:
out_folder = os.path.join(self.site.original_cwd, 'output',folder_name,catfolder,article.lower())
else:
out_folder = os.path.join(self.site.original_cwd, 'output',lang,folder_name,catfolder,article.lower())
for root, dirs, file_ins in os.walk(folder):
for f in file_ins:
in_path = os.path.join(root,f)
out_path = os.path.join(out_folder, f)
yield utils.apply_filters({
'basename': self.name,
'name': in_path + "." + lang,
'file_dep': [in_path, __file__],
'targets': [out_path],
'actions': [
(create_file, (in_path, out_path))
],
'clean': True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])
def find_translations(article):
article_file_name = os.path.splitext(article.file)[0]
it = filter((lambda possible_translation: os.path.splitext(os.path.splitext(possible_translation.file)[0])[0] == article_file_name), translations)
article_translations = {}
for translation in it:
if article.modification_date > translation.modification_date:
translation.original_newer = True
article_translations[translation.lang] = translation
return article_translations
def collect_translations(article):
article.translations = find_translations(article)
return article
articles = list(map(collect_translations, articles))
categories.append({'category': category, 'articles': articles});
for lang in self.kw['translations']:
### -----------------------------------
### 3) BOTTOM SECTION: GUIDES FROM OF-BOOK ###
of_book_path = os.path.join(directory, "of_book.md")
if lang != self.site.config['DEFAULT_LANG']:
of_book_lang_path = utils.get_translation_candidate(self.site.config, of_book_path, lang)
p = pathlib.Path(of_book_lang_path)
if p.exists():
of_book_path = of_book_lang_path
of_book = open(of_book_path).read()
### -----------------------------------
context = {}
context["lang"] = lang
if lang == self.site.config['DEFAULT_LANG']:
context["permalink"] = '/' + folder_name + '/'
else:
context["permalink"] = '/' + lang + '/' + folder_name + '/'
context["of_book"] = of_book
context["title"] = "learning"
context['categories'] = categories
short_tdst = os.path.join(self.kw['translations'][lang], folder_name, "index.html")
tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst))
template_dep = self.site.template_system.template_deps(template_name)
template_dep += files
template_dep += [__file__]
template_dep += [os.path.join(self.site.original_cwd, "messages", "of_messages_" + lang + ".py")]
yield utils.apply_filters({
'basename': self.name,
'name': tdst,
'file_dep': template_dep,
'targets': [tdst],
'actions': [
(self.site.render_template, (template_name, tdst, context))
],
'clean': True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])
def _execute(self, options, args):
L = utils.get_logger('upgrade_metadata_v8', utils.STDERR_HANDLER)
if not self.site.config['USE_TAG_METADATA']:
L.error('This plugin can only be used if USE_TAG_METADATA is set to True.')
sys.exit(-1)
self.site.config['WARN_ABOUT_TAG_METADATA'] = False
# scan posts
self.site.scan_posts()
flagged = []
for post in self.site.timeline:
flag = False
if post.has_oldstyle_metadata_tags:
flag = True
for lang in self.site.config['TRANSLATIONS'].keys():
if 'section' in post.meta[lang]:
flag = True
if flag:
flagged.append(post)
if flagged:
if len(flagged) == 1:
L.info('1 post (and/or its translations) contains old-style metadata or has section metadata:')
else:
L.info('{0} posts (and/or their translations) contain old-style metadata or have section metadata:'.format(len(flagged)))
for post in flagged:
L.info(' ' + (post.metadata_path if post.is_two_file else post.source_path))
L.warn('Please make a backup before running this plugin. It might eat your data.')
if not options['yes']:
yesno = utils.ask_yesno("Proceed with metadata upgrade?")
if options['yes'] or yesno:
number_converted = 0
number_converted_partial = 0
for post in flagged:
converted = False
fully_converted = True
for lang in self.site.config['TRANSLATIONS'].keys():
# Get file names and extractor
extractor = post.used_extractor[lang]
is_two_file = post.is_two_file
if lang == post.default_lang:
fname = post.metadata_path if is_two_file else post.source_path
else:
meta_path = os.path.splitext(post.source_path)[0] + '.meta' if is_two_file else post.source_path
fname = utils.get_translation_candidate(post.config, meta_path, lang)
# We don't handle compilers which extract metadata for now
if post.compiler is extractor:
L.warn('Cannot convert {0} (language {1}), as metadata was extracted by compiler.'.format(fname, lang))
fully_converted = False
continue
# Read metadata and text from post file
if not os.path.exists(fname):
L.debug("File {0} does not exist, skipping.".format(fname))
continue
with io.open(fname, "r", encoding="utf-8-sig") as meta_file:
source_text = meta_file.read()
if not is_two_file:
_, content_str = extractor.split_metadata_from_text(source_text)
meta = extractor.extract_text(source_text)
# Consider metadata mappings
sources = {}
for m in ('tags', 'status', 'has_math', 'section', 'category'):
sources[m] = m
for foreign, ours in self.site.config.get('METADATA_MAPPING', {}).get(extractor.map_from, {}).items():
if ours in sources:
sources[ours] = foreign
for meta_key, hook in self.site.config.get('METADATA_VALUE_MAPPING', {}).get(extractor.map_from, {}).items():
if meta_key in sources.values():
L.warn('Cannot convert {0} (language {1}): a metadata value mapping is defined for "{2}"!'.format(fname, lang, meta_key))
# Update metadata
updated = False
tags = meta.get(sources['tags'], [])
tags_are_string = False
if not isinstance(tags, list):
tags_are_string = True
tags = [tag.strip() for tag in tags.split(',') if tag.strip()]
if 'draft' in [_.lower() for _ in tags]:
tags.remove('draft')
meta[sources['status']] = 'draft'
updated = True
if 'private' in tags:
tags.remove('private')
meta[sources['status']] = 'private'
updated = True
if 'mathjax' in tags:
tags.remove('mathjax')
meta[sources['has_math']] = 'yes'
updated = True
if meta.get(sources['section']):
if meta.get(sources['category']):
L.warn('Cannot completely {0} (language {1}): both section and category are specified. Please determine the correct category to use yourself!'.format(fname, lang))
fully_converted = False
else:
meta[sources['category']] = meta[sources['section']]
del meta[sources['section']]
updated = True
if tags_are_string:
meta[sources['tags']] = ', '.join(tags)
if not updated:
# Nothing to do (but successful)!
converted = True
continue
# Recombine metadata with post text if necessary, and write back to file
meta_str = utils.write_metadata(meta, metadata_format=extractor.name, compiler=post.compiler,
comment_wrap=(post.compiler.name != 'rest'), site=self.site)
final_str = meta_str if is_two_file else (meta_str + content_str)
with io.open(fname, "w", encoding="utf-8") as meta_file:
meta_file.write(final_str)
converted = True
if converted:
if fully_converted:
number_converted += 1
else:
number_converted_partial += 1
L.info('{0} out of {2} posts upgraded; {1} only converted partially '
'(see above output).'.format(number_converted + number_converted_partial, number_converted_partial, len(flagged)))
else:
L.info('Metadata not upgraded.')
else:
L.info('No posts found with special tags or section metadata. No action is required.')
L.info('You can safely set the USE_TAG_METADATA and the WARN_ABOUT_TAG_METADATA settings to False.')
def import_item(self, item, wordpress_namespace, out_folder=None):
"""Takes an item from the feed and creates a post file."""
if out_folder is None:
out_folder = "posts"
title = get_text_tag(item, "title", "NO TITLE")
# link is something like http://foo.com/2012/09/01/hello-world/
# So, take the path, utils.slugify it, and that's our slug
link = get_text_tag(item, "link", None)
parsed = urlparse(link)
path = unquote(parsed.path.strip("/"))
# In python 2, path is a str. slug requires a unicode
# object. According to wikipedia, unquoted strings will
# usually be UTF8
if isinstance(path, utils.bytes_str):
path = path.decode("utf8")
# Cut out the base directory.
if path.startswith(self.base_dir.strip("/")):
path = path.replace(self.base_dir.strip("/"), "", 1)
pathlist = path.split("/")
if parsed.query: # if there are no nice URLs and query strings are used
out_folder = os.path.join(*([out_folder] + pathlist))
slug = get_text_tag(item, "{{{0}}}post_name".format(wordpress_namespace), None)
if not slug: # it *may* happen
slug = get_text_tag(item, "{{{0}}}post_id".format(wordpress_namespace), None)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return
else:
if len(pathlist) > 1:
out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
slug = utils.slugify(pathlist[-1])
description = get_text_tag(item, "description", "")
post_date = get_text_tag(item, "{{{0}}}post_date".format(wordpress_namespace), None)
try:
dt = utils.to_datetime(post_date)
except ValueError:
dt = datetime.datetime(1970, 1, 1, 0, 0, 0)
LOGGER.error(
'Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(
post_date, title, slug
)
)
post_date = dt.strftime("%Y-%m-%d %H:%M:%S")
if dt.tzinfo and self.timezone is None:
self.timezone = utils.get_tzname(dt)
status = get_text_tag(item, "{{{0}}}status".format(wordpress_namespace), "publish")
content = get_text_tag(item, "{http://purl.org/rss/1.0/modules/content/}encoded", "")
tags = []
if status == "trash":
LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title))
return
elif status != "publish":
tags.append("draft")
is_draft = True
else:
is_draft = False
for tag in item.findall("category"):
text = tag.text
if text == "Uncategorized":
continue
tags.append(text)
if "$latex" in content:
tags.append("mathjax")
if is_draft and self.exclude_drafts:
LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
elif content.strip():
# If no content is found, no files are written.
self.url_map[link] = (self.context["SITE_URL"] + out_folder.rstrip("/") + "/" + slug + ".html").replace(
os.sep, "/"
)
if hasattr(self, "separate_qtranslate_content") and self.separate_qtranslate_content:
content_translations = separate_qtranslate_content(content)
else:
content_translations = {"": content}
default_language = self.context["DEFAULT_LANG"]
for lang, content in content_translations.items():
if lang:
out_meta_filename = slug + ".meta"
if lang == default_language:
out_content_filename = slug + ".wp"
else:
out_content_filename = utils.get_translation_candidate(self.context, slug + ".wp", lang)
self.extra_languages.add(lang)
meta_slug = slug
else:
out_meta_filename = slug + ".meta"
out_content_filename = slug + ".wp"
meta_slug = slug
content = self.transform_content(content)
self.write_metadata(
os.path.join(self.output_folder, out_folder, out_meta_filename),
title,
meta_slug,
post_date,
description,
tags,
)
self.write_content(os.path.join(self.output_folder, out_folder, out_content_filename), content)
else:
LOGGER.warn('Not going to import "{0}" because it seems to contain' " no content.".format(title))
def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None):
"""Take an item from the feed and creates a post file."""
if out_folder is None:
out_folder = 'posts'
title = get_text_tag(item, 'title', 'NO TITLE')
# link is something like http://foo.com/2012/09/01/hello-world/
# So, take the path, utils.slugify it, and that's our slug
link = get_text_tag(item, 'link', None)
parsed = urlparse(link)
path = unquote(parsed.path.strip('/'))
try:
path = path.decode('utf8')
except AttributeError:
pass
# Cut out the base directory.
if path.startswith(self.base_dir.strip('/')):
path = path.replace(self.base_dir.strip('/'), '', 1)
pathlist = path.split('/')
if parsed.query: # if there are no nice URLs and query strings are used
out_folder = os.path.join(*([out_folder] + pathlist))
slug = get_text_tag(
item, '{{{0}}}post_name'.format(wordpress_namespace), None)
if not slug: # it *may* happen
slug = get_text_tag(
item, '{{{0}}}post_id'.format(wordpress_namespace), None)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return False
else:
if len(pathlist) > 1:
out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
slug = utils.slugify(pathlist[-1])
description = get_text_tag(item, 'description', '')
post_date = get_text_tag(
item, '{{{0}}}post_date'.format(wordpress_namespace), None)
try:
dt = utils.to_datetime(post_date)
except ValueError:
dt = datetime.datetime(1970, 1, 1, 0, 0, 0)
LOGGER.error('Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(post_date, title, slug))
post_date = dt.strftime('%Y-%m-%d %H:%M:%S')
if dt.tzinfo and self.timezone is None:
self.timezone = utils.get_tzname(dt)
status = get_text_tag(
item, '{{{0}}}status'.format(wordpress_namespace), 'publish')
content = get_text_tag(
item, '{http://purl.org/rss/1.0/modules/content/}encoded', '')
excerpt = get_text_tag(
item, '{http://wordpress.org/export/1.2/excerpt/}encoded', None)
if excerpt is not None:
if len(excerpt) == 0:
excerpt = None
tags = []
categories = []
if status == 'trash':
LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title))
return False
elif status == 'private':
tags.append('private')
is_draft = False
is_private = True
elif status != 'publish':
tags.append('draft')
is_draft = True
is_private = False
else:
is_draft = False
is_private = False
for tag in item.findall('category'):
text = tag.text
type = 'category'
if 'domain' in tag.attrib:
type = tag.attrib['domain']
if text == 'Uncategorized' and type == 'category':
continue
self.all_tags.add(text)
if type == 'category':
categories.append(type)
else:
tags.append(text)
if '$latex' in content:
tags.append('mathjax')
# Find post format if it's there
post_format = 'wp'
format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format']
if format_tag:
post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text
if post_format == 'wpautop':
post_format = 'wp'
if is_draft and self.exclude_drafts:
LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
return False
elif is_private and self.exclude_privates:
LOGGER.notice('Private post "{0}" will not be imported.'.format(title))
return False
elif content.strip() or self.import_empty_items:
# If no content is found, no files are written.
self.url_map[link] = (self.context['SITE_URL'] +
out_folder.rstrip('/') + '/' + slug +
'.html').replace(os.sep, '/')
if hasattr(self, "separate_qtranslate_content") \
and self.separate_qtranslate_content:
content_translations = separate_qtranslate_content(content)
else:
content_translations = {"": content}
default_language = self.context["DEFAULT_LANG"]
for lang, content in content_translations.items():
try:
content, extension, rewrite_html = self.transform_content(content, post_format, attachments)
except:
LOGGER.error(('Cannot interpret post "{0}" (language {1}) with post ' +
'format {2}!').format(os.path.join(out_folder, slug), lang, post_format))
return False
if lang:
out_meta_filename = slug + '.meta'
if lang == default_language:
out_content_filename = slug + '.' + extension
else:
out_content_filename \
= utils.get_translation_candidate(self.context,
slug + "." + extension, lang)
self.extra_languages.add(lang)
meta_slug = slug
else:
out_meta_filename = slug + '.meta'
out_content_filename = slug + '.' + extension
meta_slug = slug
tags, other_meta = self._create_metadata(status, excerpt, tags, categories,
post_name=os.path.join(out_folder, slug))
self.write_metadata(os.path.join(self.output_folder, out_folder,
out_meta_filename),
title, meta_slug, post_date, description, tags, **other_meta)
self.write_content(
os.path.join(self.output_folder,
out_folder, out_content_filename),
content,
rewrite_html)
if self.export_comments:
comments = []
for tag in item.findall('{{{0}}}comment'.format(wordpress_namespace)):
comment = self._extract_comment(tag, wordpress_namespace)
if comment is not None:
comments.append(comment)
for comment in comments:
comment_filename = slug + "." + str(comment['id']) + ".wpcomment"
self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment)
return (out_folder, slug)
else:
LOGGER.warn(('Not going to import "{0}" because it seems to contain'
' no content.').format(title))
return False
def create_docs(self):
tasks = {}
classes = []
directory = "documentation"
classes = markdown_file.getclass_list()
classes_simple_name = markdown_file.getclass_list(False)
addon_classes = markdown_file.list_all_addons()
module_lookup = dict()
core_index = dict()
addons_index = dict()
module_subtitles = dict()
docs_dir = os.path.join(self.site.original_cwd, "documentation")
md_extensions = self.site.config.get("MARKDOWN_EXTENSIONS")
# Create an index of which module each class is in for generated links to other classes
for class_name in classes:
clazz = markdown_file.getclass(class_name)
if clazz.istemplated:
module_lookup[class_name[:-1]] = clazz.module
else:
module_lookup[class_name] = clazz.module
for clazz_name in classes:
clazz = markdown_file.getclass(clazz_name)
if clazz.istemplated:
clazz.name = clazz.name[:-1]
clazz.detailed_inline_description = markdown(clazz.detailed_inline_description, md_extensions)
# clazz.description = str(markdown(clazz.description, md_extensions).encode('ascii', 'ignore'))
for class_name in classes_simple_name:
rep = class_name + "[\s]"
clazz.detailed_inline_description = re.sub(
rep,
'<a href="/documentation/'
+ module_lookup[class_name]
+ "/"
+ class_name
+ '" class="docs_class" >'
+ class_name
+ "</a> ",
clazz.detailed_inline_description,
)
rep = class_name + "[(]"
clazz.detailed_inline_description = re.sub(
rep,
'<a href="/documentation/'
+ module_lookup[class_name]
+ "/"
+ class_name
+ '" class="docs_class" >'
+ class_name
+ "</a>(",
clazz.detailed_inline_description,
)
clazz.reference = markdown(clazz.reference, md_extensions)
for class_name in classes_simple_name:
rep = class_name + "[\s]"
clazz.reference = re.sub(
rep,
'<a href="/documentation/'
+ module_lookup[class_name]
+ "/"
+ class_name
+ '" class="docs_class" >'
+ class_name
+ "</a> ",
clazz.reference,
)
rep = class_name + "[(]"
clazz.reference = re.sub(
rep,
'<a href="/documentation/'
+ module_lookup[class_name]
+ "/"
+ class_name
+ '" class="docs_class" >'
+ class_name
+ "</a>(",
clazz.reference,
)
for function in clazz.function_list:
function.description = markdown(function.description, md_extensions)
function.inlined_description = markdown(function.inlined_description, md_extensions)
def gen_link(class_name):
return (
'<a href="/documentation/'
+ module_lookup[class_name]
+ "/"
+ class_name
+ '" class="docs_class" >'
+ class_name
+ "</a> "
if class_name in module_lookup
else ""
)
def filter_out_empty(class_name):
return class_name != ""
clazz.extends = list(filter(filter_out_empty, map(gen_link, clazz.extends)))
functions_file = markdown_file.getfunctionsfile(clazz.name)
for function in functions_file.function_list:
function.description = markdown(function.description, md_extensions)
function.inlined_description = markdown(function.inlined_description, md_extensions)
# print clazz.name
# print clazz.function_list
env = {
"modulename": clazz.name,
"clazz": clazz,
"functions": functions_file,
"classes_list": classes,
"is_addon": (clazz.name in addon_classes),
}
# print("class " + clazz_name)
template_name = "documentation_class.mako"
for lang in self.kw["translations"]:
env["lang"] = lang
env["title"] = clazz.name
env["permalink"] = (
self.kw["translations"][lang] + "/documentation/" + clazz.module + "/" + clazz.name + "/"
)
short_tdst = os.path.join(
self.kw["translations"][lang], "documentation", clazz.module, clazz.name, "index.html"
)
tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst))
self.site.render_template(template_name, tdst, env)
if not clazz.module in addon_classes:
if not clazz.module in core_index.keys():
core_index[clazz.module] = []
if functions_file != None:
for function in functions_file.function_list:
clazz.function_list.append(function)
core_index[clazz.module].append(clazz)
else:
if not clazz.module in addons_index.keys():
addons_index[clazz.module] = []
if functions_file != None:
for function in functions_file.function_list:
clazz.function_list.append(function)
addons_index[clazz.module].append(clazz)
function_files = markdown_file.getfunctionsfiles_list()
for functionfile_name in function_files:
if functionfile_name in classes_simple_name:
continue
functions_file = markdown_file.getfunctionsfile(functionfile_name)
# might be needed at some point?
# functions_file.reference = str(functions_file.reference)
# for func in function_files:
# functions_file.reference = str.replace(functions_file.reference, class_name, "<a href=\"../"+clazz.module+"/"+class_name+".html\">"+class_name+"</a>")
for function in functions_file.function_list:
function.description = markdown(function.description, md_extensions)
function.inlined_description = markdown(function.inlined_description, md_extensions)
env = {
"modulename": functions_file.name,
"clazz": None,
"functions": functions_file,
"is_addon": (functions_file.name in addon_classes),
}
template_name = "documentation_class.mako"
for lang in self.kw["translations"]:
env["lang"] = lang
env["title"] = clazz.name
env["permalink"] = (
self.kw["translations"][lang]
+ "/documentation/"
+ functions_file.module
+ "/"
+ functions_file.name
+ "/"
)
short_tdst = os.path.join(
self.kw["translations"][lang],
"documentation",
functions_file.module,
functions_file.name,
"index.html",
)
tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst))
self.site.render_template(template_name, tdst, env)
if not functions_file.module in addon_classes:
if not functions_file.module in core_index:
core_index[functions_file.module] = []
core_index[functions_file.module].append(functions_file)
else:
if not functions_file.module in addons_index:
addons_index[functions_file.module] = []
addons_index[functions_file.module].append(functions_file)
for root, dirs, files in os.walk(directory):
""" copy images to their folders """
for name in files:
file_split = os.path.splitext(name)
if (
file_split[1] == ".jpeg"
or file_split[1] == ".jpg"
or file_split[1] == ".gif"
or file_split[1] == ".png"
):
try:
os.mkdir(os.path.join("_site", "documentation", os.path.basename(root)))
except:
pass
shutil.copyfile(
os.path.join(root, name), os.path.join("output", "documentation", os.path.basename(root), name)
)
""" create module introductions """
for module in dirs:
if module != "addons":
module_intro = os.path.join(root, module, "introduction.markdown")
if os.path.isfile(module_intro):
module_intro_file = open(module_intro)
module_intro_content = module_intro_file.read()
module_subtitles[module] = module_intro_content.splitlines()[0].strip("##").strip(" ")
module_intro_content = markdown(module_intro_content, md_extensions)
template_name = "documentation_module_intro.mako"
for lang in self.kw["translations"]:
context = {}
context["lang"] = lang
context["title"] = clazz.name
context["module"] = module
context["intro_content"] = module_intro_content
context["permalink"] = (
self.kw["translations"][lang] + "/documentation/" + module + "/introduction.html"
)
short_tdst = os.path.join(
self.kw["translations"][lang], "documentation", module, "introduction.html"
)
tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst))
if module.find("ofx") == 0:
context["classes"] = addons_index[module]
self.site.render_template(template_name, tdst, context)
else:
context["classes"] = core_index[module]
self.site.render_template(template_name, tdst, context)
else:
module_subtitles[module] = None
print("couldn't find " + module_intro)
# process index file
template_name = "documentation.mako"
for lang in self.kw["translations"]:
# lang_suffix = self.kw['translations'][lang]
docs_intro_path = os.path.join(docs_dir, "index.md")
if lang != self.site.config["DEFAULT_LANG"]:
docs_intro_lang_path = utils.get_translation_candidate(self.site.config, docs_intro_path, lang)
p = pathlib.Path(docs_intro_lang_path)
if p.exists():
docs_intro_path = docs_intro_lang_path
docs_intro = open(docs_intro_path).read()
for key in self.site.GLOBAL_CONTEXT.keys():
if isinstance(self.site.GLOBAL_CONTEXT[key], str):
docs_intro = docs_intro.replace("${" + key + "}", self.site.GLOBAL_CONTEXT[key])
docs_intro = markdown(docs_intro, md_extensions)
context = {}
context["lang"] = lang
context["title"] = "documentation"
context["docs_intro"] = docs_intro
context["core"] = core_index
context["addons"] = addons_index
context["module_subtitles"] = module_subtitles
context["permalink"] = self.kw["translations"][lang] + "/documentation/"
short_tdst = os.path.join(self.kw["translations"][lang], "documentation", "index.html")
tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst))
self.site.render_template(template_name, tdst, context)
def test_get_translation_candidate():
config = {'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'DEFAULT_LANG': 'en', 'TRANSLATIONS': {'es': '1', 'en': 1}}
assert get_translation_candidate(config, '*.rst', 'es') == '*.es.rst'
assert get_translation_candidate(
config, 'fancy.post.rst', 'es') == 'fancy.post.es.rst'
assert get_translation_candidate(config, '*.es.rst', 'es') == '*.es.rst'
assert get_translation_candidate(config, '*.es.rst', 'en') == '*.rst'
assert get_translation_candidate(
config, 'cache/posts/fancy.post.es.html', 'en') == 'cache/posts/fancy.post.html'
assert get_translation_candidate(
config, 'cache/posts/fancy.post.html', 'es') == 'cache/posts/fancy.post.es.html'
assert get_translation_candidate(
config, 'cache/pages/charts.html', 'es') == 'cache/pages/charts.es.html'
assert get_translation_candidate(
config, 'cache/pages/charts.html', 'en') == 'cache/pages/charts.html'
config = {'TRANSLATIONS_PATTERN': '{path}.{ext}.{lang}',
'DEFAULT_LANG': 'en', 'TRANSLATIONS': {'es': '1', 'en': 1}}
assert get_translation_candidate(config, '*.rst', 'es') == '*.rst.es'
assert get_translation_candidate(config, '*.rst.es', 'es') == '*.rst.es'
assert get_translation_candidate(config, '*.rst.es', 'en') == '*.rst'
assert get_translation_candidate(
config, 'cache/posts/fancy.post.html.es', 'en') == 'cache/posts/fancy.post.html'
assert get_translation_candidate(
config, 'cache/posts/fancy.post.html', 'es') == 'cache/posts/fancy.post.html.es'
def create_docs(self):
tasks = {}
classes = []
directory = "documentation"
classes = markdown_file.getclass_list()
classes_simple_name = markdown_file.getclass_list(False)
addon_classes = markdown_file.list_all_addons()
module_lookup = dict()
core_index = dict()
addons_index = dict()
module_subtitles = dict()
docs_dir = os.path.join(self.site.original_cwd, "documentation")
md_extensions = self.site.config.get("MARKDOWN_EXTENSIONS")
content_js = {}
class_template = "documentation_class.mako"
class_template_dep = self.site.template_system.template_deps(
class_template)
module_template = "documentation_module_intro.mako"
# start js string for docs search
for lang in self.kw['translations']:
content_js[lang] = 'var tipuesearch = {"pages": ['
# Create an index of which module each class is in for generated links to other classes
for class_name in classes:
clazz = markdown_file.getclass(class_name)
if clazz.istemplated:
module_lookup[class_name[:-1]] = clazz.module
else:
module_lookup[class_name] = clazz.module
# classes docs
for clazz_name in classes:
clazz = markdown_file.getclass(clazz_name)
if clazz.istemplated:
clazz.name = clazz.name[:-1]
clazz.detailed_inline_description = relative_urls(
clazz.detailed_inline_description)
clazz.detailed_inline_description = markdown(
clazz.detailed_inline_description, md_extensions)
clazz.detailed_inline_description = of_classes_to_links(
clazz.detailed_inline_description, classes_simple_name,
module_lookup)
clazz.reference = relative_urls(clazz.reference)
clazz.reference = markdown(clazz.reference, md_extensions)
clazz.reference = of_classes_to_links(clazz.reference,
classes_simple_name,
module_lookup)
# methods in class
for function in clazz.function_list:
function.description = relative_urls(function.description)
function.description = markdown(function.description,
md_extensions)
function.description = of_classes_to_links(
function.description, classes_simple_name, module_lookup)
function.inlined_description = relative_urls(
function.inlined_description)
function.inlined_description = markdown(
function.inlined_description, md_extensions)
function.inlined_description = of_classes_to_links(
function.inlined_description, classes_simple_name,
module_lookup)
for lang in self.kw['translations']:
content_js[lang] += method_to_js(function, clazz,
self.site, lang)
# inheritance
def gen_link(class_name):
return "<a href=\"/documentation/" + module_lookup[
class_name] + "/" + class_name + "\" class=\"docs_class\" >" + class_name + "</a> " if class_name in module_lookup else ""
def filter_out_empty(class_name):
return class_name != ""
clazz.extends = list(
filter(filter_out_empty, map(gen_link, clazz.extends)))
# c functions in the class file
functions_file = markdown_file.getfunctionsfile(clazz.name)
for function in functions_file.function_list:
function.description = relative_urls(function.description)
function.description = markdown(function.description,
md_extensions)
function.description = of_classes_to_links(
function.description, classes_simple_name, module_lookup)
function.inlined_description = relative_urls(
function.inlined_description)
function.inlined_description = markdown(
function.inlined_description, md_extensions)
function.inlined_description = of_classes_to_links(
function.inlined_description, classes_simple_name,
module_lookup)
for lang in self.kw['translations']:
content_js[lang] += function_to_js(function,
functions_file,
self.site, lang)
# render template + js for search
env = {
"modulename": clazz.name,
"clazz": clazz,
"functions": functions_file,
"classes_list": classes,
"is_addon": (clazz.name in addon_classes)
}
md_file = "documentation/" + module_lookup[
class_name] + "/" + class_name + ".markdown"
for lang in self.kw['translations']:
env["lang"] = lang
env["title"] = clazz.name
env["permalink"] = self.kw['translations'][
lang] + '/documentation/' + clazz.module + "/" + clazz.name + "/"
short_tdst = os.path.join(self.kw['translations'][lang],
'documentation', clazz.module,
clazz.name, "index.html")
tdst = os.path.normpath(
os.path.join(self.kw['output_folder'], short_tdst))
"""yield utils.apply_filters({
'basename': self.name,
'name': clazz.name,
'file_dep': class_template_dep + md_file,
'targets': tdst,
'actions': [
(self.site.render_template, (template_name, tdst, env))
],
'clean': True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])"""
self.site.render_template(class_template, tdst, env)
content_js[lang] += class_to_js(clazz, self.site, lang)
# add to index core or addons
if not clazz.module in addon_classes:
if not clazz.module in core_index.keys():
core_index[clazz.module] = []
if functions_file != None:
for function in functions_file.function_list:
clazz.function_list.append(function)
core_index[clazz.module].append(clazz)
else:
if not clazz.module in addons_index.keys():
addons_index[clazz.module] = []
if functions_file != None:
for function in functions_file.function_list:
clazz.function_list.append(function)
addons_index[clazz.module].append(clazz)
# generate c functions docs
function_files = markdown_file.getfunctionsfiles_list()
for functionfile_name in function_files:
if functionfile_name in classes_simple_name:
continue
functions_file = markdown_file.getfunctionsfile(functionfile_name)
# might be needed at some point?
# functions_file.reference = str(functions_file.reference)
# for func in function_files:
# functions_file.reference = str.replace(functions_file.reference, class_name, "<a href=\"../"+clazz.module+"/"+class_name+".html\">"+class_name+"</a>")
for function in functions_file.function_list:
function.description = relative_urls(function.description)
function.description = markdown(function.description,
md_extensions)
function.description = of_classes_to_links(
function.description, classes_simple_name, module_lookup)
function.inlined_description = relative_urls(
function.inlined_description)
function.inlined_description = markdown(
function.inlined_description, md_extensions)
function.inlined_description = of_classes_to_links(
function.inlined_description, classes_simple_name,
module_lookup)
for lang in self.kw['translations']:
content_js[lang] += function_to_js(function,
functions_file,
self.site, lang)
# render template + js for search
env = {
"modulename": functions_file.name,
"clazz": None,
"functions": functions_file,
"is_addon": (functions_file.name in addon_classes)
}
for lang in self.kw['translations']:
env["lang"] = lang
env["title"] = clazz.name
env["permalink"] = self.kw['translations'][
lang] + '/documentation/' + functions_file.module + "/" + functions_file.name + "/"
short_tdst = os.path.join(self.kw['translations'][lang],
'documentation',
functions_file.module,
functions_file.name, "index.html")
tdst = os.path.normpath(
os.path.join(self.kw['output_folder'], short_tdst))
self.site.render_template(class_template, tdst, env)
content_js[lang] += functions_file_to_js(
functions_file, self.site, lang)
# add to index core or addons
if not functions_file.module in addon_classes:
if not functions_file.module in core_index:
core_index[functions_file.module] = []
core_index[functions_file.module].append(functions_file)
else:
if not functions_file.module in addons_index:
addons_index[functions_file.module] = []
addons_index[functions_file.module].append(functions_file)
# copy images and render intros
for root, dirs, files in os.walk(directory):
""" copy images to their folders """
for name in files:
file_split = os.path.splitext(name)
if file_split[1] == ".jpeg" or file_split[
1] == ".jpg" or file_split[1] == ".gif" or file_split[
1] == ".png":
try:
os.mkdir(
os.path.join('_site', 'documentation',
os.path.basename(root)))
except:
pass
shutil.copyfile(
os.path.join(root, name),
os.path.join('output', 'documentation',
os.path.basename(root), name))
""" create module introductions """
for module in dirs:
if module != "addons":
module_intro = os.path.join(root, module,
"introduction.markdown")
if os.path.isfile(module_intro):
module_intro_file = open(module_intro)
module_intro_content = module_intro_file.read()
module_subtitles[
module] = module_intro_content.splitlines(
)[0].strip('##').strip(' ')
module_intro_content = markdown(
module_intro_content, md_extensions)
for lang in self.kw['translations']:
context = {}
context["lang"] = lang
context["title"] = module
context["module"] = module
context["intro_content"] = module_intro_content
context["permalink"] = self.kw['translations'][
lang] + '/documentation/' + module + "/"
if lang == self.site.config['DEFAULT_LANG']:
short_tdst = os.path.join(
'documentation', module, "index.html")
else:
short_tdst = os.path.join(
self.kw['translations'][lang],
'documentation', module, "index.html")
tdst = os.path.normpath(
os.path.join(self.kw['output_folder'],
short_tdst))
if module.find("ofx") == 0:
context["classes"] = addons_index[module]
self.site.render_template(
module_template, tdst, context)
else:
context["classes"] = core_index[module]
self.site.render_template(
module_template, tdst, context)
content_js[lang] += module_to_js(
module, module_intro_content, self.site, lang)
else:
module_subtitles[module] = None
# close js for docs search and save per language
for lang in self.kw['translations']:
content_js[lang] += ']};'
content_js_file = open(
"output" + lang_prefix(lang, self.site) +
"/tipuesearch_content.js", "w")
content_js_file.write(content_js[lang])
content_js_file.close()
# render index file
template_name = "documentation.mako"
for lang in self.kw['translations']:
#lang_suffix = self.kw['translations'][lang]
docs_intro_path = os.path.join(docs_dir, "index.md")
if lang != self.site.config['DEFAULT_LANG']:
docs_intro_lang_path = utils.get_translation_candidate(
self.site.config, docs_intro_path, lang)
p = pathlib.Path(docs_intro_lang_path)
if p.exists():
docs_intro_path = docs_intro_lang_path
docs_intro = open(docs_intro_path).read()
for key in self.site.GLOBAL_CONTEXT.keys():
if isinstance(self.site.GLOBAL_CONTEXT[key], str):
docs_intro = docs_intro.replace(
'${' + key + '}', self.site.GLOBAL_CONTEXT[key])
docs_intro = markdown(docs_intro, md_extensions)
context = {}
context["lang"] = lang
context["title"] = "documentation"
context["docs_intro"] = docs_intro
context['core'] = core_index
context['addons'] = addons_index
context['module_subtitles'] = module_subtitles
context["permalink"] = self.kw['translations'][
lang] + '/documentation/'
short_tdst = os.path.join(self.kw['translations'][lang],
"documentation", "index.html")
tdst = os.path.normpath(
os.path.join(self.kw['output_folder'], short_tdst))
self.site.render_template(template_name, tdst, context)
def import_item(self, item, wordpress_namespace, out_folder=None):
"""Takes an item from the feed and creates a post file."""
if out_folder is None:
out_folder = 'posts'
title = get_text_tag(item, 'title', 'NO TITLE')
# link is something like http://foo.com/2012/09/01/hello-world/
# So, take the path, utils.slugify it, and that's our slug
link = get_text_tag(item, 'link', None)
parsed = urlparse(link)
path = unquote(parsed.path.strip('/'))
# In python 2, path is a str. slug requires a unicode
# object. According to wikipedia, unquoted strings will
# usually be UTF8
if isinstance(path, utils.bytes_str):
path = path.decode('utf8')
# Cut out the base directory.
if path.startswith(self.base_dir.strip('/')):
path = path.replace(self.base_dir.strip('/'), '', 1)
pathlist = path.split('/')
if parsed.query: # if there are no nice URLs and query strings are used
out_folder = os.path.join(*([out_folder] + pathlist))
slug = get_text_tag(item,
'{{{0}}}post_name'.format(wordpress_namespace),
None)
if not slug: # it *may* happen
slug = get_text_tag(
item, '{{{0}}}post_id'.format(wordpress_namespace), None)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return
else:
if len(pathlist) > 1:
out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
slug = utils.slugify(pathlist[-1])
description = get_text_tag(item, 'description', '')
post_date = get_text_tag(
item, '{{{0}}}post_date'.format(wordpress_namespace), None)
try:
dt = utils.to_datetime(post_date)
except ValueError:
dt = datetime.datetime(1970, 1, 1, 0, 0, 0)
LOGGER.error(
'Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'
.format(post_date, title, slug))
post_date = dt.strftime('%Y-%m-%d %H:%M:%S')
if dt.tzinfo and self.timezone is None:
self.timezone = utils.get_tzname(dt)
status = get_text_tag(item,
'{{{0}}}status'.format(wordpress_namespace),
'publish')
content = get_text_tag(
item, '{http://purl.org/rss/1.0/modules/content/}encoded', '')
tags = []
if status == 'trash':
LOGGER.warn(
'Trashed post "{0}" will not be imported.'.format(title))
return
elif status != 'publish':
tags.append('draft')
is_draft = True
else:
is_draft = False
for tag in item.findall('category'):
text = tag.text
if text == 'Uncategorized':
continue
tags.append(text)
if '$latex' in content:
tags.append('mathjax')
if is_draft and self.exclude_drafts:
LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
elif content.strip():
# If no content is found, no files are written.
self.url_map[link] = (self.context['SITE_URL'] +
out_folder.rstrip('/') + '/' + slug +
'.html').replace(os.sep, '/')
if hasattr(self, "separate_qtranslate_content") \
and self.separate_qtranslate_content:
content_translations = separate_qtranslate_content(content)
else:
content_translations = {"": content}
default_language = self.context["DEFAULT_LANG"]
for lang, content in content_translations.items():
if lang:
out_meta_filename = slug + '.meta'
if lang == default_language:
out_content_filename = slug + '.wp'
else:
out_content_filename \
= utils.get_translation_candidate(self.context,
slug + ".wp", lang)
self.extra_languages.add(lang)
meta_slug = slug
else:
out_meta_filename = slug + '.meta'
out_content_filename = slug + '.wp'
meta_slug = slug
content = self.transform_content(content)
self.write_metadata(
os.path.join(self.output_folder, out_folder,
out_meta_filename), title, meta_slug,
post_date, description, tags)
self.write_content(
os.path.join(self.output_folder, out_folder,
out_content_filename), content)
else:
LOGGER.warn('Not going to import "{0}" because it seems to contain'
' no content.'.format(title))
def import_item(self, item, wordpress_namespace, out_folder=None):
"""Takes an item from the feed and creates a post file."""
if out_folder is None:
out_folder = 'posts'
title = get_text_tag(item, 'title', 'NO TITLE')
# link is something like http://foo.com/2012/09/01/hello-world/
# So, take the path, utils.slugify it, and that's our slug
link = get_text_tag(item, 'link', None)
parsed = urlparse(link)
path = unquote(parsed.path.strip('/'))
# In python 2, path is a str. slug requires a unicode
# object. According to wikipedia, unquoted strings will
# usually be UTF8
if isinstance(path, utils.bytes_str):
path = path.decode('utf8')
# Cut out the base directory.
if path.startswith(self.base_dir.strip('/')):
path = path.replace(self.base_dir.strip('/'), '', 1)
pathlist = path.split('/')
if parsed.query: # if there are no nice URLs and query strings are used
out_folder = os.path.join(*([out_folder] + pathlist))
slug = get_text_tag(
item, '{{{0}}}post_name'.format(wordpress_namespace), None)
if not slug: # it *may* happen
slug = get_text_tag(
item, '{{{0}}}post_id'.format(wordpress_namespace), None)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return
else:
if len(pathlist) > 1:
out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
slug = utils.slugify(pathlist[-1])
description = get_text_tag(item, 'description', '')
post_date = get_text_tag(
item, '{{{0}}}post_date'.format(wordpress_namespace), None)
try:
dt = utils.to_datetime(post_date)
except ValueError:
dt = datetime.datetime(1970, 1, 1, 0, 0, 0)
LOGGER.error('Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(post_date, title, slug))
post_date = dt.strftime('%Y-%m-%d %H:%M:%S')
if dt.tzinfo and self.timezone is None:
self.timezone = utils.get_tzname(dt)
status = get_text_tag(
item, '{{{0}}}status'.format(wordpress_namespace), 'publish')
content = get_text_tag(
item, '{http://purl.org/rss/1.0/modules/content/}encoded', '')
tags = []
if status == 'trash':
LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title))
return
elif status != 'publish':
tags.append('draft')
is_draft = True
else:
is_draft = False
for tag in item.findall('category'):
text = tag.text
if text == 'Uncategorized':
continue
tags.append(text)
if '$latex' in content:
tags.append('mathjax')
# Find post format if it's there
post_format = 'wp'
format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format']
if format_tag:
post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text
if is_draft and self.exclude_drafts:
LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
elif content.strip():
# If no content is found, no files are written.
self.url_map[link] = (self.context['SITE_URL'] +
out_folder.rstrip('/') + '/' + slug +
'.html').replace(os.sep, '/')
if hasattr(self, "separate_qtranslate_content") \
and self.separate_qtranslate_content:
content_translations = separate_qtranslate_content(content)
else:
content_translations = {"": content}
default_language = self.context["DEFAULT_LANG"]
for lang, content in content_translations.items():
if lang:
out_meta_filename = slug + '.meta'
if lang == default_language:
out_content_filename = slug + '.wp'
else:
out_content_filename \
= utils.get_translation_candidate(self.context,
slug + ".wp", lang)
self.extra_languages.add(lang)
meta_slug = slug
else:
out_meta_filename = slug + '.meta'
out_content_filename = slug + '.wp'
meta_slug = slug
if post_format == 'wp':
content = self.transform_content(content)
self.write_metadata(os.path.join(self.output_folder, out_folder,
out_meta_filename),
title, meta_slug, post_date, description, tags)
self.write_content(
os.path.join(self.output_folder,
out_folder, out_content_filename),
content)
else:
LOGGER.warn('Not going to import "{0}" because it seems to contain'
' no content.'.format(title))
def gen_tasks(self):
self.kw = {
'strip_indexes': self.site.config['STRIP_INDEXES'],
'output_folder': self.site.config['OUTPUT_FOLDER'],
'cache_folder': self.site.config['CACHE_FOLDER'],
'default_lang': self.site.config['DEFAULT_LANG'],
'filters': self.site.config['FILTERS'],
'translations': self.site.config['TRANSLATIONS'],
'global_context': self.site.GLOBAL_CONTEXT,
'tzinfo': self.site.tzinfo,
}
tasks = {}
classes = []
directory = os.path.join(self.site.original_cwd, "tutorials")
template_name = "tutorials.mako"
categories = []
dirs = os.listdir(directory)
dirs.sort()
files = []
for catfolder in dirs:
if not os.path.isdir(os.path.join(directory, catfolder)):
continue
articles = []
translations = []
category = catfolder[catfolder.find("_") + 1:]
articlesfiles = os.listdir(os.path.join(directory, catfolder))
articlesfiles.sort()
for article in articlesfiles:
file_split = os.path.splitext(article)
extension = file_split[1]
lang_split = article.split(".")
lang = self.site.config['DEFAULT_LANG'] if len(
lang_split) < 3 else lang_split[1]
is_translation = lang != self.site.config['DEFAULT_LANG']
folder = os.path.join(directory, catfolder, article)
if extension == '.markdown':
path = os.path.join(directory, catfolder, article)
files += [path]
articleobj = MarkdownArticle(path, directory, lang,
is_translation)
if not is_translation:
articles.append(articleobj)
else:
translations.append(articleobj)
elif extension == '.asciidoc':
path = os.path.join(directory, catfolder, article)
files += [path]
articleobj = AsciidocArticle(path, directory, lang,
is_translation)
if not is_translation:
articles.append(articleobj)
else:
translations.append(articleobj)
elif os.path.isdir(folder):
for lang in self.kw['translations']:
if lang == self.site.config['DEFAULT_LANG']:
out_folder = os.path.join(self.site.original_cwd,
'output',
'tutorials', catfolder,
article.lower())
else:
out_folder = os.path.join(self.site.original_cwd,
'output', lang,
'tutorials', catfolder,
article.lower())
for root, dirs, file_ins in os.walk(folder):
for f in file_ins:
in_path = os.path.join(root, f)
out_path = os.path.join(out_folder, f)
yield utils.apply_filters(
{
'basename':
self.name,
'name':
in_path + "." + lang,
'file_dep': [in_path, __file__],
'targets': [out_path],
'actions': [(create_file,
(in_path, out_path))],
'clean':
True,
'uptodate':
[utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])
def find_translations(article):
article_file_name = os.path.splitext(article.file)[0]
it = filter((lambda possible_translation: os.path.splitext(
os.path.splitext(possible_translation.file)[0])[0] ==
article_file_name), translations)
article_translations = {}
for translation in it:
article_translations[translation.lang] = translation
return article_translations
def collect_translations(article):
article.translations = find_translations(article)
return article
articles = list(map(collect_translations, articles))
categories.append({
'category': category,
'articles': articles
})
for lang in self.kw['translations']:
tutorials_intro_path = os.path.join(directory, "index.md")
if lang != self.site.config['DEFAULT_LANG']:
tutorials_intro_lang_path = utils.get_translation_candidate(
self.site.config, tutorials_intro_path, lang)
p = pathlib.Path(tutorials_intro_lang_path)
if p.exists():
tutorials_intro_path = tutorials_intro_lang_path
tutorials_intro = open(tutorials_intro_path).read()
context = {}
context["lang"] = lang
if lang == self.site.config['DEFAULT_LANG']:
context["permalink"] = '/tutorials/'
else:
context["permalink"] = '/' + lang + '/tutorials/'
context["tutorials_intro"] = tutorials_intro
context["title"] = "tutorials"
context['categories'] = categories
short_tdst = os.path.join(self.kw['translations'][lang],
"tutorials", "index.html")
tdst = os.path.normpath(
os.path.join(self.kw['output_folder'], short_tdst))
template_dep = self.site.template_system.template_deps(
template_name)
template_dep += files
template_dep += [__file__]
yield utils.apply_filters(
{
'basename':
self.name,
'name':
tdst,
'file_dep':
template_dep,
'targets': [tdst],
'actions': [(self.site.render_template,
(template_name, tdst, context))],
'clean':
True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])
def create_docs(self):
tasks = {}
classes = []
directory = "documentation"
classes = markdown_file.getclass_list()
classes_simple_name = markdown_file.getclass_list(False)
addon_classes = markdown_file.list_all_addons()
module_lookup = dict()
core_index = dict()
addons_index = dict()
module_subtitles = dict()
docs_dir = os.path.join(self.site.original_cwd, "documentation")
md_extensions = self.site.config.get("MARKDOWN_EXTENSIONS")
content_js = {}
class_template = "documentation_class.mako"
class_template_dep = self.site.template_system.template_deps(class_template)
module_template = "documentation_module_intro.mako"
# start js string for docs search
for lang in self.kw['translations']:
content_js[lang] = 'var tipuesearch = {"pages": ['
# Create an index of which module each class is in for generated links to other classes
for class_name in classes:
clazz = markdown_file.getclass(class_name)
if clazz.istemplated:
module_lookup[class_name[:-1]] = clazz.module
else:
module_lookup[class_name] = clazz.module
# classes docs
for clazz_name in classes:
clazz = markdown_file.getclass(clazz_name)
if clazz.istemplated:
clazz.name = clazz.name[:-1]
clazz.detailed_inline_description = relative_urls(clazz.detailed_inline_description)
clazz.detailed_inline_description = markdown(clazz.detailed_inline_description, md_extensions)
clazz.detailed_inline_description = of_classes_to_links(clazz.detailed_inline_description, classes_simple_name, module_lookup)
clazz.reference = relative_urls(clazz.reference)
clazz.reference = markdown(clazz.reference, md_extensions)
clazz.reference = of_classes_to_links(clazz.reference, classes_simple_name, module_lookup)
# methods in class
for function in clazz.function_list:
function.description = relative_urls(function.description)
function.description = of_classes_to_links(function.description, classes_simple_name, module_lookup)
function.description = markdown(function.description, md_extensions)
function.inlined_description = relative_urls(function.inlined_description)
function.inlined_description = of_classes_to_links(function.inlined_description, classes_simple_name, module_lookup)
function.inlined_description = markdown(function.inlined_description, md_extensions)
for lang in self.kw['translations']:
content_js[lang] += method_to_js(function, clazz, self.site, lang)
# inheritance
def gen_link(class_name):
return "<a href=\"/documentation/" + module_lookup[class_name] + "/" + class_name + "\" class=\"docs_class\" >"+class_name+"</a> " if class_name in module_lookup else ""
def filter_out_empty(class_name):
return class_name!=""
clazz.extends = list(filter(filter_out_empty, map(gen_link, clazz.extends)))
# c functions in the class file
functions_file = markdown_file.getfunctionsfile(clazz.name)
for function in functions_file.function_list:
function.description = relative_urls(function.description)
function.description = of_classes_to_links(function.description, classes_simple_name, module_lookup)
function.description = markdown(function.description, md_extensions)
function.inlined_description = relative_urls(function.inlined_description)
function.inlined_description = of_classes_to_links(function.inlined_description, classes_simple_name, module_lookup)
function.inlined_description = markdown(function.inlined_description, md_extensions)
for lang in self.kw['translations']:
content_js[lang] += function_to_js(function, functions_file, self.site, lang)
# render template + js for search
env = {
"modulename": clazz.name,
"clazz": clazz,
"functions": functions_file,
"classes_list": classes,
"is_addon": (clazz.name in addon_classes)
}
md_file = "documentation/" + module_lookup[class_name] + "/" + class_name + ".markdown"
for lang in self.kw['translations']:
env["lang"] = lang
env["title"] = clazz.name
env["permalink"] = self.kw['translations'][lang] + '/documentation/' + clazz.module + "/" + clazz.name + "/"
short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', clazz.module, clazz.name,"index.html")
tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst))
"""yield utils.apply_filters({
'basename': self.name,
'name': clazz.name,
'file_dep': class_template_dep + md_file,
'targets': tdst,
'actions': [
(self.site.render_template, (template_name, tdst, env))
],
'clean': True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])"""
self.site.render_template(class_template, tdst, env)
content_js[lang] += class_to_js(clazz, self.site, lang)
# add to index core or addons
if not clazz.module in addon_classes:
if not clazz.module in core_index.keys():
core_index[clazz.module] = []
if functions_file!=None:
for function in functions_file.function_list:
clazz.function_list.append(function)
core_index[clazz.module].append(clazz)
else:
if not clazz.module in addons_index.keys():
addons_index[clazz.module] = []
if functions_file!=None:
for function in functions_file.function_list:
clazz.function_list.append(function)
addons_index[clazz.module].append(clazz)
# generate c functions docs
function_files = markdown_file.getfunctionsfiles_list()
for functionfile_name in function_files:
if functionfile_name in classes_simple_name:
continue
functions_file = markdown_file.getfunctionsfile(functionfile_name)
# might be needed at some point?
# functions_file.reference = str(functions_file.reference)
# for func in function_files:
# functions_file.reference = str.replace(functions_file.reference, class_name, "<a href=\"../"+clazz.module+"/"+class_name+".html\">"+class_name+"</a>")
for function in functions_file.function_list:
function.description = relative_urls(function.description)
function.description = of_classes_to_links(function.description, classes_simple_name, module_lookup)
function.description = markdown(function.description, md_extensions)
function.inlined_description = relative_urls(function.inlined_description)
function.inlined_description = of_classes_to_links(function.inlined_description, classes_simple_name, module_lookup)
function.inlined_description = markdown(function.inlined_description, md_extensions)
for lang in self.kw['translations']:
content_js[lang] += function_to_js(function, functions_file, self.site, lang)
# render template + js for search
env = {
"modulename": functions_file.name,
"clazz": None,
"functions": functions_file,
"is_addon": (functions_file.name in addon_classes)
}
for lang in self.kw['translations']:
env["lang"] = lang
env["title"] = clazz.name
env["permalink"] = self.kw['translations'][lang] + '/documentation/' + functions_file.module + "/" + functions_file.name + "/"
short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', functions_file.module, functions_file.name,"index.html")
tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst))
self.site.render_template(class_template, tdst, env)
content_js[lang] += functions_file_to_js(functions_file, self.site, lang)
# add to index core or addons
if not functions_file.module in addon_classes:
if not functions_file.module in core_index:
core_index[functions_file.module] = []
core_index[functions_file.module].append(functions_file)
else:
if not functions_file.module in addons_index:
addons_index[functions_file.module] = []
addons_index[functions_file.module].append(functions_file)
# copy images and render intros
for root, dirs, files in os.walk(directory):
""" copy images to their folders """
for name in files:
file_split = os.path.splitext(name)
if file_split[1]==".jpeg" or file_split[1]==".jpg" or file_split[1]==".gif" or file_split[1]==".png":
try:
os.mkdir(os.path.join('_site','documentation',os.path.basename(root)))
except:
pass
shutil.copyfile(os.path.join(root,name), os.path.join('output','documentation',os.path.basename(root),name))
""" create module introductions """
for module in dirs:
if module!="addons":
module_intro = os.path.join(root,module,"introduction.markdown")
if os.path.isfile(module_intro):
module_intro_file = open(module_intro)
module_intro_content = module_intro_file.read()
module_subtitles[module] = module_intro_content.splitlines()[0].strip('##').strip(' ')
module_intro_content = markdown(module_intro_content, md_extensions)
for lang in self.kw['translations']:
context = {}
context["lang"] = lang
context["title"] = module
context["module"] = module
context["intro_content"] = module_intro_content
context["permalink"] = self.kw['translations'][lang] + '/documentation/' + module + "/"
if lang == self.site.config['DEFAULT_LANG']:
short_tdst = os.path.join('documentation', module, "index.html")
else:
short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', module, "index.html")
tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst))
if module.find("ofx") == 0:
context["classes"] = addons_index[module]
self.site.render_template(module_template, tdst, context)
else:
context["classes"] = core_index[module]
self.site.render_template(module_template, tdst, context)
content_js[lang] += module_to_js(module, module_intro_content, self.site, lang)
else:
module_subtitles[module] = None
# close js for docs search and save per language
for lang in self.kw['translations']:
content_js[lang] += ']};'
content_js_file = open("output" + lang_prefix(lang, self.site) + "/tipuesearch_content.js","w")
content_js_file.write(content_js[lang])
content_js_file.close()
# render index file
template_name = "documentation.mako"
for lang in self.kw['translations']:
#lang_suffix = self.kw['translations'][lang]
docs_intro_path = os.path.join(docs_dir, "index.md")
if lang != self.site.config['DEFAULT_LANG']:
docs_intro_lang_path = utils.get_translation_candidate(self.site.config, docs_intro_path, lang)
p = pathlib.Path(docs_intro_lang_path)
if p.exists():
docs_intro_path = docs_intro_lang_path
docs_intro = open(docs_intro_path).read()
for key in self.site.GLOBAL_CONTEXT.keys():
if isinstance(self.site.GLOBAL_CONTEXT[key], str):
docs_intro = docs_intro.replace('${' + key + '}', self.site.GLOBAL_CONTEXT[key])
docs_intro = markdown(docs_intro, md_extensions)
context = {}
context["lang"] = lang
context["title"] = "documentation"
context["docs_intro"] = docs_intro
context['core'] = core_index
context['addons'] = addons_index
context['module_subtitles'] = module_subtitles
context["permalink"] = self.kw['translations'][lang] + '/documentation/'
short_tdst = os.path.join(self.kw['translations'][lang], "documentation", "index.html")
tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst))
self.site.render_template(template_name, tdst, context)
def test_get_translation_candidate():
config = {
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'DEFAULT_LANG': 'en',
'TRANSLATIONS': {
'es': '1',
'en': 1
}
}
assert get_translation_candidate(config, '*.rst', 'es') == '*.es.rst'
assert get_translation_candidate(config, 'fancy.post.rst',
'es') == 'fancy.post.es.rst'
assert get_translation_candidate(config, '*.es.rst', 'es') == '*.es.rst'
assert get_translation_candidate(config, '*.es.rst', 'en') == '*.rst'
assert get_translation_candidate(config, 'cache/posts/fancy.post.es.html',
'en') == 'cache/posts/fancy.post.html'
assert get_translation_candidate(config, 'cache/posts/fancy.post.html',
'es') == 'cache/posts/fancy.post.es.html'
assert get_translation_candidate(config, 'cache/pages/charts.html',
'es') == 'cache/pages/charts.es.html'
assert get_translation_candidate(config, 'cache/pages/charts.html',
'en') == 'cache/pages/charts.html'
config = {
'TRANSLATIONS_PATTERN': '{path}.{ext}.{lang}',
'DEFAULT_LANG': 'en',
'TRANSLATIONS': {
'es': '1',
'en': 1
}
}
assert get_translation_candidate(config, '*.rst', 'es') == '*.rst.es'
assert get_translation_candidate(config, '*.rst.es', 'es') == '*.rst.es'
assert get_translation_candidate(config, '*.rst.es', 'en') == '*.rst'
assert get_translation_candidate(config, 'cache/posts/fancy.post.html.es',
'en') == 'cache/posts/fancy.post.html'
assert get_translation_candidate(config, 'cache/posts/fancy.post.html',
'es') == 'cache/posts/fancy.post.html.es'
def gen_tasks(self):
self.kw = {
'strip_indexes': self.site.config['STRIP_INDEXES'],
'output_folder': self.site.config['OUTPUT_FOLDER'],
'cache_folder': self.site.config['CACHE_FOLDER'],
'default_lang': self.site.config['DEFAULT_LANG'],
'filters': self.site.config['FILTERS'],
'translations': self.site.config['TRANSLATIONS'],
'global_context': self.site.GLOBAL_CONTEXT,
'tzinfo': self.site.tzinfo,
}
tasks = {}
classes = []
directory = os.path.join(self.site.original_cwd, "tutorials")
template_name = "tutorials.mako"
categories = []
dirs = os.listdir(directory)
dirs.sort()
files = []
for catfolder in dirs:
if not os.path.isdir(os.path.join(directory,catfolder)):
continue
articles = []
category = catfolder[catfolder.find("_")+1:]
articlesfiles = os.listdir(os.path.join(directory,catfolder));
articlesfiles.sort()
for article in articlesfiles:
file_split = os.path.splitext(article)
folder = os.path.join(directory,catfolder,article)
if file_split[1]=='.markdown':
path = os.path.join(directory,catfolder,article)
files += [path]
articleobj = MarkdownArticle(path, directory)
articles.append(articleobj)
elif file_split[1]=='.asciidoc':
path = os.path.join(directory,catfolder,article)
files += [path]
articleobj = AsciidocArticle(path, directory)
articles.append(articleobj)
elif os.path.isdir(folder):
out_folder = os.path.join(self.site.original_cwd, 'output','tutorials',catfolder,article.lower())
for root, dirs, file_ins in os.walk(folder):
for f in file_ins:
in_path = os.path.join(root,f)
out_path = os.path.join(out_folder, f)
yield utils.apply_filters({
'basename': self.name,
'name': in_path,
'file_dep': [in_path],
'targets': [out_path],
'actions': [
(create_file, (in_path, out_path))
],
'clean': True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])
categories.append({'category': category, 'articles': articles});
for lang in self.kw['translations']:
tutorials_intro_path = os.path.join(directory, "index.md")
if lang != self.site.config['DEFAULT_LANG']:
tutorials_intro_lang_path = utils.get_translation_candidate(self.site.config, tutorials_intro_path, lang)
p = pathlib.Path(tutorials_intro_lang_path)
if p.exists():
tutorials_intro_path = tutorials_intro_lang_path
tutorials_intro = open(tutorials_intro_path).read()
context = {}
context["lang"] = lang
if lang == self.site.config['DEFAULT_LANG']:
context["lang_folder"] = ""
else:
context["lang_folder"] = "/" + lang
context["tutorials_intro"] = tutorials_intro
context["title"] = "tutorials"
context['categories'] = categories
context["permalink"] = '/tutorials/'
short_tdst = os.path.join(self.kw['translations'][lang], "tutorials", "index.html")
tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst))
template_dep = self.site.template_system.template_deps(template_name)
template_dep += files
yield utils.apply_filters({
'basename': self.name,
'name': tdst,
'file_dep': template_dep,
'targets': [tdst],
'actions': [
(self.site.render_template, (template_name, tdst, context))
],
'clean': True,
'uptodate': [utils.config_changed({
1: self.kw,
})],
}, self.kw['filters'])