def parse_index(self, gallery, input_folder, output_folder): """Return a Post object if there is an index.txt.""" index_path = os.path.join(gallery, "index.txt") destination = os.path.join( self.kw["output_folder"], output_folder, os.path.relpath(gallery, input_folder)) if os.path.isfile(index_path): post = Post( index_path, self.site.config, destination, False, self.site.MESSAGES, 'page.tmpl', self.site.get_compiler(index_path), None, self.site.metadata_extractors_by ) # If this did not exist, galleries without a title in the # index.txt file would be errorneously named `index` # (warning: galleries titled index and filenamed differently # may break) if post.title == 'index': post.title = os.path.split(gallery)[1] # Register the post (via #2417) self.site.post_per_input_file[index_path] = post else: post = None return post
def parse_index(self, gallery): """Returns a Post object if there is an index.txt.""" index_path = os.path.join(gallery, "index.txt") destination = os.path.join( self.kw["output_folder"], gallery) if os.path.isfile(index_path): post = Post( index_path, self.site.config, destination, False, self.site.MESSAGES, 'story.tmpl', self.site.get_compiler(index_path) ) # If this did not exist, galleries without a title in the # index.txt file would be errorneously named `index` # (warning: galleries titled index and filenamed differently # may break) if post.title == 'index': post.title = os.path.split(gallery)[1] else: post = None return post
def parse_index(self, gallery, input_folder, output_folder): """Return a Post object if there is an index.txt.""" index_path = os.path.join(gallery, "index.txt") destination = os.path.join( self.kw["output_folder"], output_folder, os.path.relpath(gallery, input_folder)) if os.path.isfile(index_path): post = Post( index_path, self.site.config, destination, False, self.site.MESSAGES, 'story.tmpl', self.site.get_compiler(index_path) ) # If this did not exist, galleries without a title in the # index.txt file would be errorneously named `index` # (warning: galleries titled index and filenamed differently # may break) if post.title == 'index': post.title = os.path.split(gallery)[1] else: post = None return post
def parse_index(self, gallery, input_folder, output_folder): """Return a Post object if there is an index.txt.""" index_path = os.path.join(gallery, "index.txt") destination = os.path.join(output_folder, os.path.relpath(gallery, input_folder)) if os.path.isfile(index_path): post = Post(index_path, self.site.config, destination, False, self.site.MESSAGES, 'page.tmpl', self.site.get_compiler(index_path), None, self.site.metadata_extractors_by) # If this did not exist, galleries without a title in the # index.txt file would be errorneously named `index` # (warning: galleries titled index and filenamed differently # may break) if post.title() == 'index': for lang in post.meta.keys(): post.meta[lang]['title'] = os.path.split(gallery)[1] # Register the post (via #2417) self.site.post_per_input_file[index_path] = post # Register post for the sitemap, too (#3598) index_output = os.path.join(gallery, self.kw['index_file']) self.site.post_per_file[index_output] = post else: post = None return post
def crawl(node, destinations_so_far, root=True): if node.post_source is not None: try: post = Post( node.post_source, self.site.config, '', False, self.site.MESSAGES, template_name, self.site.get_compiler(node.post_source), destination_base=utils.TranslatableSetting('destinations', destinations_so_far, self.site.config['TRANSLATIONS']), metadata_extractors_by=self.site.metadata_extractors_by ) timeline.append(post) except Exception as err: LOGGER.error('Error reading post {}'.format(base_path)) raise err # Compute slugs slugs = {} for lang in self.site.config['TRANSLATIONS']: slug = post.meta('slug', lang=lang) if slug: slugs[lang] = slug if not slugs: slugs[self.site.config['DEFAULT_LANG']] = node.name node.slugs = _spread(slugs, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG']) # Update destinations_so_far if not root: if node.slugs is not None: destinations_so_far = {lang: os.path.join(dest, node.slugs[lang]) for lang, dest in destinations_so_far.items()} else: destinations_so_far = {lang: os.path.join(dest, node.name) for lang, dest in destinations_so_far.items()} for p, n in node.children.items(): crawl(n, destinations_so_far, root=False)
def scan(self): """Scan posts in a package index.""" if 'PKGINDEX_CONFIG' not in self.site.config: return [] config = self.site.config['PKGINDEX_CONFIG'] plugin_compiler = self.site.get_compiler('sample' + config['extension']) if not self.site.quiet: print("Scanning package index posts...", end='', file=sys.stderr) timeline = [] self.site.pkgindex_entries = {} for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items(): destination, template_name = dirsettings self.site.pkgindex_entries[topdir] = [] for pkgdir in glob.glob(topdir + "/*"): if not os.path.isdir(pkgdir): # Ignore non-directories continue post = Post(os.path.join(pkgdir, 'README.md'), self.site.config, destination, False, self.site.MESSAGES, template_name, plugin_compiler) post.is_two_file = True timeline.append(post) self.site.pkgindex_entries[topdir].append(post) return timeline
def check_page_slug(page: Post, is_index_page: bool) -> None: logger = get_logger(__name__) index_page_slug = "index" # for consistency, page slug should be the same as the filename file_name = page.translated_source_path(page.default_lang).split("/")[-1] file_ext = page.source_ext() page_slug = page.meta[page.default_lang].get("slug") if page_slug != index_page_slug and page_slug + file_ext != file_name: logger.warn( f'page {page.permalink()} uses slug "{page_slug}" which is different from source file name "{file_name}"' ) # for consistency, index pages (that is, pages with "index_path" specified) should have slug "index" if is_index_page and page_slug != index_page_slug: logger.warn( f'page {page.permalink()} is an index page but has slug "{page_slug}" which is different from expected "{index_page_slug}"' )
def make_page_breadcrumb(page: Post, site_structure: PageDir): breadcrumb = page.meta[page.default_lang].get("breadcrumb") # TODO should this compare to "False"? (a string?) if breadcrumb is None or breadcrumb != "False": return generate_breadcrumb(page.permalink(), site_structure) else: return None
def scan(self): """Scan posts in a package index.""" if 'PKGINDEX_CONFIG' not in self.site.config: return [] config = self.site.config['PKGINDEX_CONFIG'] compiler = self.site.get_compiler('sample' + config['extension']) if not self.site.quiet: print("Scanning package index posts...", end='', file=sys.stderr) timeline = [] self.site.pkgindex_entries = {} for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items(): destination, template_name = dirsettings self.site.pkgindex_entries[topdir] = [] for pkgdir in glob.glob(topdir + "/*"): if not os.path.isdir(pkgdir): # Ignore non-directories continue post = Post( os.path.join(pkgdir, 'README.md'), self.site.config, destination, False, self.site.MESSAGES, template_name, compiler ) post.is_two_file = True timeline.append(post) self.site.pkgindex_entries[topdir].append(post) if 'special_entries' in config: for source_path, destination, template_name, topdir in config['special_entries']: post = Post( source_path, self.site.config, destination, False, self.site.MESSAGES, template_name, compiler ) post.is_two_file = True timeline.append(post) self.site.pkgindex_entries[topdir].append(post) return timeline
def scan(self): """Create list of posts from POSTS and PAGES options.""" seen = set([]) if not self.site.quiet: print("Scanning posts", end='', file=sys.stderr) timeline = [] for wildcard, destination, template_name, use_in_feeds in \ self.site.config['post_pages']: if not self.site.quiet: print(".", end='', file=sys.stderr) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname, followlinks=True): dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname))) # output/destination/foo/ # Get all the untranslated paths dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst untranslated = glob.glob(dir_glob) # And now get all the translated paths translated = set([]) for lang in self.site.config['TRANSLATIONS'].keys(): if lang == self.site.config['DEFAULT_LANG']: continue lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst translated = translated.union(set(glob.glob(lang_glob))) # untranslated globs like *.rst often match translated paths too, so remove them # and ensure x.rst is not in the translated set untranslated = set(untranslated) - translated # also remove from translated paths that are translations of # paths in untranslated_list, so x.es.rst is not in the untranslated set for p in untranslated: translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()]) full_list = list(translated) + list(untranslated) # We eliminate from the list the files inside any .ipynb folder full_list = [p for p in full_list if not any([x.startswith('.') for x in p.split(os.sep)])] for base_path in full_list: if base_path in seen: continue else: seen.add(base_path) post = Post( base_path, self.site.config, dest_dir, use_in_feeds, self.site.MESSAGES, template_name, self.site.get_compiler(base_path) ) timeline.append(post) return timeline
def crawl(node, destinations_so_far, root=True): if node.post_source is not None: try: post = Post(node.post_source, self.site.config, '', False, self.site.MESSAGES, template_name, self.site.get_compiler(node.post_source), destination_base=utils.TranslatableSetting( 'destinations', destinations_so_far, self.site.config['TRANSLATIONS']), metadata_extractors_by=self.site. metadata_extractors_by) timeline.append(post) except Exception as err: LOGGER.error('Error reading post {}'.format(base_path)) raise err # Compute slugs slugs = {} for lang in self.site.config['TRANSLATIONS']: slug = post.meta('slug', lang=lang) if slug: slugs[lang] = slug if not slugs: slugs[self.site.config['DEFAULT_LANG']] = node.name node.slugs = _spread(slugs, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG']) # Update destinations_so_far if not root: if node.slugs is not None: destinations_so_far = { lang: os.path.join(dest, node.slugs[lang]) for lang, dest in destinations_so_far.items() } else: destinations_so_far = { lang: os.path.join(dest, node.name) for lang, dest in destinations_so_far.items() } for p, n in node.children.items(): crawl(n, destinations_so_far, root=False)
def parse_index(self, post_path): """Returns a Post object from a foo.txt.""" destination = os.path.join(self.kw["output_folder"], 'series') if os.path.isfile(post_path): post = Post(post_path, self.site.config, destination, False, self.site.MESSAGES, 'story.tmpl', self.site.get_compiler(post_path)) else: post = None return post
def parse_index(self, gallery): """Returns a Post object if there is an index.txt.""" index_path = os.path.join(gallery, "index.txt") destination = os.path.join(self.kw["output_folder"], gallery) if os.path.isfile(index_path): post = Post(index_path, self.site.config, destination, False, self.site.MESSAGES, 'story.tmpl', self.site.get_compiler(index_path)) else: post = None return post
def setHtmlFromRst(self, rst): """ Create html output from rst string """ tmpdir = tempfile.mkdtemp() inf = os.path.join(tmpdir, 'inf') outf = os.path.join(tmpdir, 'outf') depf = os.path.join(tmpdir, 'outf.dep') with io.open(inf, 'w+', encoding='utf8') as f: f.write(rst) p = Post(inf, self.site.config, outf, False, None, '', self.compiler) self.site.post_per_input_file[inf] = p p.compile_html(inf, outf, post=p) with io.open(outf, 'r', encoding='utf8') as f: self.html = f.read() os.unlink(inf) os.unlink(outf) p.write_depfile(outf, p._depfile[outf]) if os.path.isfile(depf): with io.open(depf, 'r', encoding='utf8') as f: self.assertEqual(self.deps.strip(), f.read().strip()) os.unlink(depf) else: self.assertEqual(self.deps, None) os.rmdir(tmpdir) self.html_doc = html.parse(StringIO(self.html))
def generate_index(self, site: Nikola, post: Post, data: str = None, lang: str = None, depth: int = 1) -> Tuple[str, List[str]]: """ Generate a hierarchical list of pages when shortcode is invoked. Ouput: a HTML string (nested <ul> elements containing <a> elements) and a list of file dependencies. Depth determines how far the index recurses. Set depth to 0 to recurse as much as possible. """ if not post.metadata.is_index_page: raise RuntimeError( f'Attempt to generate index on "{post.permalink()}" which is not an index page!' ) # index pages can list contents of arbitrary directory # find first which directory the page refers to index_root: PageDir = site.GLOBAL_CONTEXT["metadata"].structure() index_path: str = post.meta[post.default_lang]["index_path"] if index_path == ".": for directory_name in split_path(post.permalink()): index_root = index_root.enter(directory_name) elif index_path != "/": for directory_name in split_path(index_path): index_root = index_root.enter(directory_name) self.logger.info( f"generating index structure for {post.permalink()} that starts in {index_path}" ) html_result: str = generate_hierarchical_html(index_root, depth) # We need to regenerate indexes every time a page is added or removed. # We can not return wildcard paths or generally - paths which do not exist # but there is a hidden (undocumented) feature that does exactly this. # https://github.com/getnikola/nikola/issues/3293#issuecomment-523210046 file_dependencies = ["####MAGIC####TIMELINE"] return html_result, file_dependencies
def reload_site(self): """Reload the site from the database.""" rev = int(self.db.get('site:rev')) if rev != self.revision and self.db.exists('site:rev'): timeline = self.db.lrange('site:timeline', 0, -1) self._timeline = [] for data in timeline: data = json.loads(data) self._timeline.append( Post(data[0], self.config, data[1], data[2], data[3], self.messages, self._site.compilers[data[4]])) self._read_indexlist('posts') self._read_indexlist('all_posts') self._read_indexlist('pages') self.revision = rev self.logger.info("Site updated to revision {0}.".format(rev)) elif rev == self.revision and self.db.exists('site:rev'): pass else: self.logger.warn("Site needs rescanning.")
def scan(self): """Scan posts in a package index.""" if 'PKGINDEX_CONFIG' not in self.site.config: return [] config = self.site.config['PKGINDEX_CONFIG'] compiler = self.site.get_compiler('sample' + config['extension']) if not self.site.quiet: print("Scanning package index posts...", end='', file=sys.stderr) timeline = [] self.site.pkgindex_entries = {} self.site.pkgindex_by_name = {} self.site.pkgindex_multiver = {} for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items(): destination, template_name = dirsettings self.site.pkgindex_entries[topdir] = [] for pkgdir in glob.glob(topdir + "/*"): if not os.path.isdir(pkgdir): # Ignore non-directories continue post = Post(os.path.join(pkgdir, 'README.md'), self.site.config, destination, False, self.site.MESSAGES, template_name, compiler) post.is_two_file = True for d in post.meta.values(): d['is_special_entry'] = False timeline.append(post) self.site.pkgindex_entries[topdir].append(post) self._update_name_multiver(post) if 'special_entries' in config: for source_path, destination, template_name, topdir in config[ 'special_entries']: post = Post(source_path, self.site.config, destination, False, self.site.MESSAGES, template_name, compiler) post.is_two_file = True for d in post.meta.values(): d['is_special_entry'] = True timeline.append(post) self.site.pkgindex_entries[topdir].append(post) self._update_name_multiver(post) # But wait, we need to change tags on multiver stuff! # This is kinda... hacky... maxver = config['versions_supported'][-1] for versions in self.site.pkgindex_multiver.values(): versions = sorted(versions, key=lambda post: post.meta('dirver')) v2p = {} for post in versions: dirver = post.meta('dirver') for v in range(dirver, maxver + 1): v2p[v] = post p2v = {} for v, p in v2p.items(): if p in p2v: p2v[p].append(v) else: p2v[p] = [v] for post, versions in p2v.items(): # And finally, update tags. tags = post._tags[self.site.default_lang] tags = [ i for i in tags if not (i.startswith('v') and i[1:].isdigit()) ] tags += ['v{0}'.format(i) for i in versions] tags.append('multiver') post._tags[self.site.default_lang] = tags post.meta['en']['tags'] = tags post.meta['en']['multiver'] = True post.meta['en']['allver'] = versions if not post.meta['en']['maxver'] and versions[-1] != maxver: post.meta['en']['maxver'] = versions[-1] # And generate self.site.pkgindex_by_version self.site.pkgindex_by_version = { i: [] for i in config['versions_supported'] } for l in self.site.pkgindex_entries.values(): for post in l: for version in post.meta['en']['allver']: self.site.pkgindex_by_version[version] = post return timeline
def scan(self): """Scan posts in a package index.""" if 'PKGINDEX_CONFIG' not in self.site.config: return [] config = self.site.config['PKGINDEX_CONFIG'] compiler = self.site.get_compiler('sample' + config['extension']) if not self.site.quiet: print("Scanning package index posts...", end='', file=sys.stderr) timeline = [] self.site.pkgindex_entries = {} self.site.pkgindex_by_name = {} self.site.pkgindex_multiver = {} for topdir, dirsettings in self.site.config['PKGINDEX_DIRS'].items(): destination, template_name = dirsettings self.site.pkgindex_entries[topdir] = [] for pkgdir in glob.glob(topdir + "/*"): if not os.path.isdir(pkgdir): # Ignore non-directories continue post = Post( os.path.join(pkgdir, 'README.md'), self.site.config, destination, False, self.site.MESSAGES, template_name, compiler ) post.is_two_file = True for d in post.meta.values(): d['is_special_entry'] = False timeline.append(post) self.site.pkgindex_entries[topdir].append(post) self._update_name_multiver(post) if 'special_entries' in config: for source_path, destination, template_name, topdir in config['special_entries']: post = Post( source_path, self.site.config, destination, False, self.site.MESSAGES, template_name, compiler ) post.is_two_file = True for d in post.meta.values(): d['is_special_entry'] = True timeline.append(post) self.site.pkgindex_entries[topdir].append(post) self._update_name_multiver(post) # But wait, we need to change tags on multiver stuff! # This is kinda... hacky... maxver = config['versions_supported'][-1] for versions in self.site.pkgindex_multiver.values(): versions = sorted(versions, key=lambda post: post.meta('dirver')) v2p = {} for post in versions: dirver = post.meta('dirver') for v in range(dirver, maxver + 1): v2p[v] = post p2v = {} for v, p in v2p.items(): if p in p2v: p2v[p].append(v) else: p2v[p] = [v] for post, versions in p2v.items(): # And finally, update tags. tags = post._tags[self.site.default_lang] tags = [i for i in tags if not (i.startswith('v') and i[1:].isdigit())] tags += ['v{0}'.format(i) for i in versions] tags.append('multiver') post._tags[self.site.default_lang] = tags post.meta['en']['tags'] = tags post.meta['en']['multiver'] = True post.meta['en']['allver'] = versions if not post.meta['en']['maxver'] and versions[-1] != maxver: post.meta['en']['maxver'] = versions[-1] # And generate self.site.pkgindex_by_version self.site.pkgindex_by_version = {i: [] for i in config['versions_supported']} for l in self.site.pkgindex_entries.values(): for post in l: for version in post.meta['en']['allver']: self.site.pkgindex_by_version[version] = post return timeline
def scan(self): """Create list of posts from POSTS and PAGES options.""" seen = set([]) if not self.site.quiet: print("Scanning posts", end='', file=sys.stderr) timeline = [] for wildcard, destination, template_name, use_in_feeds in \ self.site.config['post_pages']: if not self.site.quiet: print(".", end='', file=sys.stderr) destination_translatable = utils.TranslatableSetting('destination', destination, self.site.config['TRANSLATIONS']) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname, followlinks=True): rel_dest_dir = os.path.relpath(dirpath, dirname) # Get all the untranslated paths dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst untranslated = glob.glob(dir_glob) # And now get all the translated paths translated = set([]) for lang in self.site.config['TRANSLATIONS'].keys(): if lang == self.site.config['DEFAULT_LANG']: continue lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst translated = translated.union(set(glob.glob(lang_glob))) # untranslated globs like *.rst often match translated paths too, so remove them # and ensure x.rst is not in the translated set untranslated = set(untranslated) - translated # also remove from translated paths that are translations of # paths in untranslated_list, so x.es.rst is not in the untranslated set for p in untranslated: translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()]) full_list = list(translated) + list(untranslated) # We eliminate from the list the files inside any .ipynb folder full_list = [p for p in full_list if not any([x.startswith('.') for x in p.split(os.sep)])] for base_path in sorted(full_list): if base_path in seen: continue try: post = Post( base_path, self.site.config, rel_dest_dir, use_in_feeds, self.site.MESSAGES, template_name, self.site.get_compiler(base_path), destination_base=destination_translatable, metadata_extractors_by=self.site.metadata_extractors_by ) for lang in post.translated_to: seen.add(post.translated_source_path(lang)) timeline.append(post) except Exception: LOGGER.error('Error reading post {}'.format(base_path)) raise return timeline
def _is_subpage(self, subpage: Post, page: Post): # permalink of subpage starts with page's permalink return subpage.permalink().startswith(page.permalink())