def parse_index(self, gallery, input_folder, output_folder):
"""Return a Post object if there is an index.txt."""
index_path = os.path.join(gallery, "index.txt")
destination = os.path.join(
self.kw["output_folder"], output_folder,
os.path.relpath(gallery, input_folder))
if os.path.isfile(index_path):
post = Post(
index_path,
self.site.config,
destination,
False,
self.site.MESSAGES,
'page.tmpl',
self.site.get_compiler(index_path),
None,
self.site.metadata_extractors_by
)
# If this did not exist, galleries without a title in the
# index.txt file would be errorneously named `index`
# (warning: galleries titled index and filenamed differently
# may break)
if post.title == 'index':
post.title = os.path.split(gallery)[1]
# Register the post (via #2417)
self.site.post_per_input_file[index_path] = post
else:
post = None
return post
def parse_index(self, gallery):
"""Returns a Post object if there is an index.txt."""
index_path = os.path.join(gallery, "index.txt")
destination = os.path.join(
self.kw["output_folder"],
gallery)
if os.path.isfile(index_path):
post = Post(
index_path,
self.site.config,
destination,
False,
self.site.MESSAGES,
'story.tmpl',
self.site.get_compiler(index_path)
)
# If this did not exist, galleries without a title in the
# index.txt file would be errorneously named `index`
# (warning: galleries titled index and filenamed differently
# may break)
if post.title == 'index':
post.title = os.path.split(gallery)[1]
else:
post = None
return post
def parse_index(self, gallery, input_folder, output_folder):
"""Return a Post object if there is an index.txt."""
index_path = os.path.join(gallery, "index.txt")
destination = os.path.join(
self.kw["output_folder"], output_folder,
os.path.relpath(gallery, input_folder))
if os.path.isfile(index_path):
post = Post(
index_path,
self.site.config,
destination,
False,
self.site.MESSAGES,
'story.tmpl',
self.site.get_compiler(index_path)
)
# If this did not exist, galleries without a title in the
# index.txt file would be errorneously named `index`
# (warning: galleries titled index and filenamed differently
# may break)
if post.title == 'index':
post.title = os.path.split(gallery)[1]
else:
post = None
return post
def parse_index(self, gallery, input_folder, output_folder):
"""Return a Post object if there is an index.txt."""
index_path = os.path.join(gallery, "index.txt")
destination = os.path.join(output_folder,
os.path.relpath(gallery, input_folder))
if os.path.isfile(index_path):
post = Post(index_path, self.site.config, destination, False,
self.site.MESSAGES, 'page.tmpl',
self.site.get_compiler(index_path), None,
self.site.metadata_extractors_by)
# If this did not exist, galleries without a title in the
# index.txt file would be errorneously named `index`
# (warning: galleries titled index and filenamed differently
# may break)
if post.title() == 'index':
for lang in post.meta.keys():
post.meta[lang]['title'] = os.path.split(gallery)[1]
# Register the post (via #2417)
self.site.post_per_input_file[index_path] = post
# Register post for the sitemap, too (#3598)
index_output = os.path.join(gallery, self.kw['index_file'])
self.site.post_per_file[index_output] = post
else:
post = None
return post
def crawl(node, destinations_so_far, root=True):
if node.post_source is not None:
try:
post = Post(
node.post_source,
self.site.config,
'',
False,
self.site.MESSAGES,
template_name,
self.site.get_compiler(node.post_source),
destination_base=utils.TranslatableSetting('destinations', destinations_so_far, self.site.config['TRANSLATIONS']),
metadata_extractors_by=self.site.metadata_extractors_by
)
timeline.append(post)
except Exception as err:
LOGGER.error('Error reading post {}'.format(base_path))
raise err
# Compute slugs
slugs = {}
for lang in self.site.config['TRANSLATIONS']:
slug = post.meta('slug', lang=lang)
if slug:
slugs[lang] = slug
if not slugs:
slugs[self.site.config['DEFAULT_LANG']] = node.name
node.slugs = _spread(slugs, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG'])
# Update destinations_so_far
if not root:
if node.slugs is not None:
destinations_so_far = {lang: os.path.join(dest, node.slugs[lang]) for lang, dest in destinations_so_far.items()}
else:
destinations_so_far = {lang: os.path.join(dest, node.name) for lang, dest in destinations_so_far.items()}
for p, n in node.children.items():
crawl(n, destinations_so_far, root=False)
def scan(self):
"""Scan posts in a package index."""
if 'PKGINDEX_CONFIG' not in self.site.config:
return []
config = self.site.config['PKGINDEX_CONFIG']
plugin_compiler = self.site.get_compiler('sample' +
config['extension'])
if not self.site.quiet:
print("Scanning package index posts...", end='', file=sys.stderr)
timeline = []
self.site.pkgindex_entries = {}
for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items():
destination, template_name = dirsettings
self.site.pkgindex_entries[topdir] = []
for pkgdir in glob.glob(topdir + "/*"):
if not os.path.isdir(pkgdir):
# Ignore non-directories
continue
post = Post(os.path.join(pkgdir, 'README.md'),
self.site.config, destination, False,
self.site.MESSAGES, template_name, plugin_compiler)
post.is_two_file = True
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
return timeline
def check_page_slug(page: Post, is_index_page: bool) -> None:
logger = get_logger(__name__)
index_page_slug = "index"
# for consistency, page slug should be the same as the filename
file_name = page.translated_source_path(page.default_lang).split("/")[-1]
file_ext = page.source_ext()
page_slug = page.meta[page.default_lang].get("slug")
if page_slug != index_page_slug and page_slug + file_ext != file_name:
logger.warn(
f'page {page.permalink()} uses slug "{page_slug}" which is different from source file name "{file_name}"'
)
# for consistency, index pages (that is, pages with "index_path" specified) should have slug "index"
if is_index_page and page_slug != index_page_slug:
logger.warn(
f'page {page.permalink()} is an index page but has slug "{page_slug}" which is different from expected "{index_page_slug}"'
)
def make_page_breadcrumb(page: Post, site_structure: PageDir):
breadcrumb = page.meta[page.default_lang].get("breadcrumb")
# TODO should this compare to "False"? (a string?)
if breadcrumb is None or breadcrumb != "False":
return generate_breadcrumb(page.permalink(), site_structure)
else:
return None
def scan(self):
"""Scan posts in a package index."""
if 'PKGINDEX_CONFIG' not in self.site.config:
return []
config = self.site.config['PKGINDEX_CONFIG']
compiler = self.site.get_compiler('sample' + config['extension'])
if not self.site.quiet:
print("Scanning package index posts...", end='', file=sys.stderr)
timeline = []
self.site.pkgindex_entries = {}
for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items():
destination, template_name = dirsettings
self.site.pkgindex_entries[topdir] = []
for pkgdir in glob.glob(topdir + "/*"):
if not os.path.isdir(pkgdir):
# Ignore non-directories
continue
post = Post(
os.path.join(pkgdir, 'README.md'),
self.site.config,
destination,
False,
self.site.MESSAGES,
template_name,
compiler
)
post.is_two_file = True
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
if 'special_entries' in config:
for source_path, destination, template_name, topdir in config['special_entries']:
post = Post(
source_path,
self.site.config,
destination,
False,
self.site.MESSAGES,
template_name,
compiler
)
post.is_two_file = True
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
return timeline
def scan(self):
"""Create list of posts from POSTS and PAGES options."""
seen = set([])
if not self.site.quiet:
print("Scanning posts", end='', file=sys.stderr)
timeline = []
for wildcard, destination, template_name, use_in_feeds in \
self.site.config['post_pages']:
if not self.site.quiet:
print(".", end='', file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname, followlinks=True):
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname))) # output/destination/foo/
# Get all the untranslated paths
dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst
untranslated = glob.glob(dir_glob)
# And now get all the translated paths
translated = set([])
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == self.site.config['DEFAULT_LANG']:
continue
lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst
translated = translated.union(set(glob.glob(lang_glob)))
# untranslated globs like *.rst often match translated paths too, so remove them
# and ensure x.rst is not in the translated set
untranslated = set(untranslated) - translated
# also remove from translated paths that are translations of
# paths in untranslated_list, so x.es.rst is not in the untranslated set
for p in untranslated:
translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()])
full_list = list(translated) + list(untranslated)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.site.config,
dest_dir,
use_in_feeds,
self.site.MESSAGES,
template_name,
self.site.get_compiler(base_path)
)
timeline.append(post)
return timeline
def crawl(node, destinations_so_far, root=True):
if node.post_source is not None:
try:
post = Post(node.post_source,
self.site.config,
'',
False,
self.site.MESSAGES,
template_name,
self.site.get_compiler(node.post_source),
destination_base=utils.TranslatableSetting(
'destinations', destinations_so_far,
self.site.config['TRANSLATIONS']),
metadata_extractors_by=self.site.
metadata_extractors_by)
timeline.append(post)
except Exception as err:
LOGGER.error('Error reading post {}'.format(base_path))
raise err
# Compute slugs
slugs = {}
for lang in self.site.config['TRANSLATIONS']:
slug = post.meta('slug', lang=lang)
if slug:
slugs[lang] = slug
if not slugs:
slugs[self.site.config['DEFAULT_LANG']] = node.name
node.slugs = _spread(slugs,
self.site.config['TRANSLATIONS'],
self.site.config['DEFAULT_LANG'])
# Update destinations_so_far
if not root:
if node.slugs is not None:
destinations_so_far = {
lang: os.path.join(dest, node.slugs[lang])
for lang, dest in destinations_so_far.items()
}
else:
destinations_so_far = {
lang: os.path.join(dest, node.name)
for lang, dest in destinations_so_far.items()
}
for p, n in node.children.items():
crawl(n, destinations_so_far, root=False)
def parse_index(self, post_path):
"""Returns a Post object from a foo.txt."""
destination = os.path.join(self.kw["output_folder"], 'series')
if os.path.isfile(post_path):
post = Post(post_path, self.site.config, destination, False,
self.site.MESSAGES, 'story.tmpl',
self.site.get_compiler(post_path))
else:
post = None
return post
def parse_index(self, gallery):
"""Returns a Post object if there is an index.txt."""
index_path = os.path.join(gallery, "index.txt")
destination = os.path.join(self.kw["output_folder"], gallery)
if os.path.isfile(index_path):
post = Post(index_path, self.site.config, destination, False,
self.site.MESSAGES, 'story.tmpl',
self.site.get_compiler(index_path))
else:
post = None
return post
def setHtmlFromRst(self, rst):
""" Create html output from rst string """
tmpdir = tempfile.mkdtemp()
inf = os.path.join(tmpdir, 'inf')
outf = os.path.join(tmpdir, 'outf')
depf = os.path.join(tmpdir, 'outf.dep')
with io.open(inf, 'w+', encoding='utf8') as f:
f.write(rst)
p = Post(inf, self.site.config, outf, False, None, '', self.compiler)
self.site.post_per_input_file[inf] = p
p.compile_html(inf, outf, post=p)
with io.open(outf, 'r', encoding='utf8') as f:
self.html = f.read()
os.unlink(inf)
os.unlink(outf)
p.write_depfile(outf, p._depfile[outf])
if os.path.isfile(depf):
with io.open(depf, 'r', encoding='utf8') as f:
self.assertEqual(self.deps.strip(), f.read().strip())
os.unlink(depf)
else:
self.assertEqual(self.deps, None)
os.rmdir(tmpdir)
self.html_doc = html.parse(StringIO(self.html))
def generate_index(self,
site: Nikola,
post: Post,
data: str = None,
lang: str = None,
depth: int = 1) -> Tuple[str, List[str]]:
"""
Generate a hierarchical list of pages when shortcode is invoked.
Ouput: a HTML string (nested <ul> elements containing <a> elements)
and a list of file dependencies. Depth determines how far the index
recurses. Set depth to 0 to recurse as much as possible.
"""
if not post.metadata.is_index_page:
raise RuntimeError(
f'Attempt to generate index on "{post.permalink()}" which is not an index page!'
)
# index pages can list contents of arbitrary directory
# find first which directory the page refers to
index_root: PageDir = site.GLOBAL_CONTEXT["metadata"].structure()
index_path: str = post.meta[post.default_lang]["index_path"]
if index_path == ".":
for directory_name in split_path(post.permalink()):
index_root = index_root.enter(directory_name)
elif index_path != "/":
for directory_name in split_path(index_path):
index_root = index_root.enter(directory_name)
self.logger.info(
f"generating index structure for {post.permalink()} that starts in {index_path}"
)
html_result: str = generate_hierarchical_html(index_root, depth)
# We need to regenerate indexes every time a page is added or removed.
# We can not return wildcard paths or generally - paths which do not exist
# but there is a hidden (undocumented) feature that does exactly this.
# https://github.com/getnikola/nikola/issues/3293#issuecomment-523210046
file_dependencies = ["####MAGIC####TIMELINE"]
return html_result, file_dependencies
def reload_site(self):
"""Reload the site from the database."""
rev = int(self.db.get('site:rev'))
if rev != self.revision and self.db.exists('site:rev'):
timeline = self.db.lrange('site:timeline', 0, -1)
self._timeline = []
for data in timeline:
data = json.loads(data)
self._timeline.append(
Post(data[0], self.config, data[1], data[2], data[3],
self.messages, self._site.compilers[data[4]]))
self._read_indexlist('posts')
self._read_indexlist('all_posts')
self._read_indexlist('pages')
self.revision = rev
self.logger.info("Site updated to revision {0}.".format(rev))
elif rev == self.revision and self.db.exists('site:rev'):
pass
else:
self.logger.warn("Site needs rescanning.")
def setHtmlFromRst(self, rst):
""" Create html output from rst string """
tmpdir = tempfile.mkdtemp()
inf = os.path.join(tmpdir, 'inf')
outf = os.path.join(tmpdir, 'outf')
depf = os.path.join(tmpdir, 'outf.dep')
with io.open(inf, 'w+', encoding='utf8') as f:
f.write(rst)
p = Post(inf, self.site.config, outf, False, None, '', self.compiler)
self.site.post_per_input_file[inf] = p
p.compile_html(inf, outf, post=p)
with io.open(outf, 'r', encoding='utf8') as f:
self.html = f.read()
os.unlink(inf)
os.unlink(outf)
p.write_depfile(outf, p._depfile[outf])
if os.path.isfile(depf):
with io.open(depf, 'r', encoding='utf8') as f:
self.assertEqual(self.deps.strip(), f.read().strip())
os.unlink(depf)
else:
self.assertEqual(self.deps, None)
os.rmdir(tmpdir)
self.html_doc = html.parse(StringIO(self.html))
def scan(self):
"""Scan posts in a package index."""
if 'PKGINDEX_CONFIG' not in self.site.config:
return []
config = self.site.config['PKGINDEX_CONFIG']
compiler = self.site.get_compiler('sample' + config['extension'])
if not self.site.quiet:
print("Scanning package index posts...", end='', file=sys.stderr)
timeline = []
self.site.pkgindex_entries = {}
self.site.pkgindex_by_name = {}
self.site.pkgindex_multiver = {}
for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items():
destination, template_name = dirsettings
self.site.pkgindex_entries[topdir] = []
for pkgdir in glob.glob(topdir + "/*"):
if not os.path.isdir(pkgdir):
# Ignore non-directories
continue
post = Post(os.path.join(pkgdir, 'README.md'),
self.site.config, destination, False,
self.site.MESSAGES, template_name, compiler)
post.is_two_file = True
for d in post.meta.values():
d['is_special_entry'] = False
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
self._update_name_multiver(post)
if 'special_entries' in config:
for source_path, destination, template_name, topdir in config[
'special_entries']:
post = Post(source_path, self.site.config, destination, False,
self.site.MESSAGES, template_name, compiler)
post.is_two_file = True
for d in post.meta.values():
d['is_special_entry'] = True
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
self._update_name_multiver(post)
# But wait, we need to change tags on multiver stuff!
# This is kinda... hacky...
maxver = config['versions_supported'][-1]
for versions in self.site.pkgindex_multiver.values():
versions = sorted(versions, key=lambda post: post.meta('dirver'))
v2p = {}
for post in versions:
dirver = post.meta('dirver')
for v in range(dirver, maxver + 1):
v2p[v] = post
p2v = {}
for v, p in v2p.items():
if p in p2v:
p2v[p].append(v)
else:
p2v[p] = [v]
for post, versions in p2v.items():
# And finally, update tags.
tags = post._tags[self.site.default_lang]
tags = [
i for i in tags
if not (i.startswith('v') and i[1:].isdigit())
]
tags += ['v{0}'.format(i) for i in versions]
tags.append('multiver')
post._tags[self.site.default_lang] = tags
post.meta['en']['tags'] = tags
post.meta['en']['multiver'] = True
post.meta['en']['allver'] = versions
if not post.meta['en']['maxver'] and versions[-1] != maxver:
post.meta['en']['maxver'] = versions[-1]
# And generate self.site.pkgindex_by_version
self.site.pkgindex_by_version = {
i: []
for i in config['versions_supported']
}
for l in self.site.pkgindex_entries.values():
for post in l:
for version in post.meta['en']['allver']:
self.site.pkgindex_by_version[version] = post
return timeline
def scan(self):
"""Scan posts in a package index."""
if 'PKGINDEX_CONFIG' not in self.site.config:
return []
config = self.site.config['PKGINDEX_CONFIG']
compiler = self.site.get_compiler('sample' + config['extension'])
if not self.site.quiet:
print("Scanning package index posts...", end='', file=sys.stderr)
timeline = []
self.site.pkgindex_entries = {}
self.site.pkgindex_by_name = {}
self.site.pkgindex_multiver = {}
for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items():
destination, template_name = dirsettings
self.site.pkgindex_entries[topdir] = []
for pkgdir in glob.glob(topdir + "/*"):
if not os.path.isdir(pkgdir):
# Ignore non-directories
continue
post = Post(
os.path.join(pkgdir, 'README.md'),
self.site.config,
destination,
False,
self.site.MESSAGES,
template_name,
compiler
)
post.is_two_file = True
for d in post.meta.values():
d['is_special_entry'] = False
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
self._update_name_multiver(post)
if 'special_entries' in config:
for source_path, destination, template_name, topdir in config['special_entries']:
post = Post(
source_path,
self.site.config,
destination,
False,
self.site.MESSAGES,
template_name,
compiler
)
post.is_two_file = True
for d in post.meta.values():
d['is_special_entry'] = True
timeline.append(post)
self.site.pkgindex_entries[topdir].append(post)
self._update_name_multiver(post)
# But wait, we need to change tags on multiver stuff!
# This is kinda... hacky...
maxver = config['versions_supported'][-1]
for versions in self.site.pkgindex_multiver.values():
versions = sorted(versions, key=lambda post: post.meta('dirver'))
v2p = {}
for post in versions:
dirver = post.meta('dirver')
for v in range(dirver, maxver + 1):
v2p[v] = post
p2v = {}
for v, p in v2p.items():
if p in p2v:
p2v[p].append(v)
else:
p2v[p] = [v]
for post, versions in p2v.items():
# And finally, update tags.
tags = post._tags[self.site.default_lang]
tags = [i for i in tags if not (i.startswith('v') and i[1:].isdigit())]
tags += ['v{0}'.format(i) for i in versions]
tags.append('multiver')
post._tags[self.site.default_lang] = tags
post.meta['en']['tags'] = tags
post.meta['en']['multiver'] = True
post.meta['en']['allver'] = versions
if not post.meta['en']['maxver'] and versions[-1] != maxver:
post.meta['en']['maxver'] = versions[-1]
# And generate self.site.pkgindex_by_version
self.site.pkgindex_by_version = {i: [] for i in config['versions_supported']}
for l in self.site.pkgindex_entries.values():
for post in l:
for version in post.meta['en']['allver']:
self.site.pkgindex_by_version[version] = post
return timeline
def scan(self):
"""Create list of posts from POSTS and PAGES options."""
seen = set([])
if not self.site.quiet:
print("Scanning posts", end='', file=sys.stderr)
timeline = []
for wildcard, destination, template_name, use_in_feeds in \
self.site.config['post_pages']:
if not self.site.quiet:
print(".", end='', file=sys.stderr)
destination_translatable = utils.TranslatableSetting('destination', destination, self.site.config['TRANSLATIONS'])
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname, followlinks=True):
rel_dest_dir = os.path.relpath(dirpath, dirname)
# Get all the untranslated paths
dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst
untranslated = glob.glob(dir_glob)
# And now get all the translated paths
translated = set([])
for lang in self.site.config['TRANSLATIONS'].keys():
if lang == self.site.config['DEFAULT_LANG']:
continue
lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst
translated = translated.union(set(glob.glob(lang_glob)))
# untranslated globs like *.rst often match translated paths too, so remove them
# and ensure x.rst is not in the translated set
untranslated = set(untranslated) - translated
# also remove from translated paths that are translations of
# paths in untranslated_list, so x.es.rst is not in the untranslated set
for p in untranslated:
translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()])
full_list = list(translated) + list(untranslated)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in sorted(full_list):
if base_path in seen:
continue
try:
post = Post(
base_path,
self.site.config,
rel_dest_dir,
use_in_feeds,
self.site.MESSAGES,
template_name,
self.site.get_compiler(base_path),
destination_base=destination_translatable,
metadata_extractors_by=self.site.metadata_extractors_by
)
for lang in post.translated_to:
seen.add(post.translated_source_path(lang))
timeline.append(post)
except Exception:
LOGGER.error('Error reading post {}'.format(base_path))
raise
return timeline
def _is_subpage(self, subpage: Post, page: Post):
# permalink of subpage starts with page's permalink
return subpage.permalink().startswith(page.permalink())
