def scan(self): """Create list of posts from POSTS and PAGES options.""" seen = set([]) if not self.site.quiet: print("Scanning posts", end='', file=sys.stderr) timeline = [] for wildcard, destination, template_name, use_in_feeds in \ self.site.config['post_pages']: if not self.site.quiet: print(".", end='', file=sys.stderr) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname, followlinks=True): dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname))) # output/destination/foo/ # Get all the untranslated paths dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst untranslated = glob.glob(dir_glob) # And now get all the translated paths translated = set([]) for lang in self.site.config['TRANSLATIONS'].keys(): if lang == self.site.config['DEFAULT_LANG']: continue lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst translated = translated.union(set(glob.glob(lang_glob))) # untranslated globs like *.rst often match translated paths too, so remove them # and ensure x.rst is not in the translated set untranslated = set(untranslated) - translated # also remove from translated paths that are translations of # paths in untranslated_list, so x.es.rst is not in the untranslated set for p in untranslated: translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()]) full_list = list(translated) + list(untranslated) # We eliminate from the list the files inside any .ipynb folder full_list = [p for p in full_list if not any([x.startswith('.') for x in p.split(os.sep)])] for base_path in full_list: if base_path in seen: continue else: seen.add(base_path) post = Post( base_path, self.site.config, dest_dir, use_in_feeds, self.site.MESSAGES, template_name, self.site.get_compiler(base_path) ) timeline.append(post) return timeline
def find_metadata(self, gallery, lang): """Search for a gallery metadata file. If there is an metadata file for the gallery, use that to determine captions and the order in which images shall be displayed in the gallery. You only need to list the images if a specific ordering or caption is required. The metadata file is YAML-formatted, with field names of # name: caption: order: # If a numeric order value is specified, we use that directly, otherwise we depend on how PyYAML returns the information - which may or may not be in the same order as in the file itself. Non-numeric ordering is not supported. If no caption is specified, then we return an empty string. Returns a string (l18n'd filename), list (ordering), dict (captions), dict (image metadata). """ base_meta_path = os.path.join(gallery, "metadata.yml") localized_meta_path = utils.get_translation_candidate(self.site.config, base_meta_path, lang) order = [] captions = {} custom_metadata = {} used_path = "" if os.path.isfile(localized_meta_path): used_path = localized_meta_path elif os.path.isfile(base_meta_path): used_path = base_meta_path else: return "", [], {}, {} self.logger.debug("Using {0} for gallery {1}".format( used_path, gallery)) with open(used_path, "r", encoding='utf-8-sig') as meta_file: if yaml is None: utils.req_missing(['PyYAML'], 'use metadata.yml files for galleries') meta = yaml.safe_load_all(meta_file) for img in meta: # load_all and safe_load_all both return None as their # final element, so skip it if not img: continue if 'name' in img: img_name = img.pop('name') if 'caption' in img and img['caption']: captions[img_name] = img.pop('caption') if 'order' in img and img['order'] is not None: order.insert(img.pop('order'), img_name) else: order.append(img_name) custom_metadata[img_name] = img else: self.logger.error("no 'name:' for ({0}) in {1}".format( img, used_path)) return used_path, order, captions, custom_metadata
def find_metadata(self, gallery, lang): """Search for a gallery metadata file. If there is an metadata file for the gallery, use that to determine captions and the order in which images shall be displayed in the gallery. You only need to list the images if a specific ordering or caption is required. The metadata file is YAML-formatted, with field names of # name: caption: order: # If a numeric order value is specified, we use that directly, otherwise we depend on how PyYAML returns the information - which may or may not be in the same order as in the file itself. Non-numeric ordering is not supported. If no caption is specified, then we return an empty string. Returns a string (l18n'd filename), list (ordering), dict (captions), dict (image metadata). """ base_meta_path = os.path.join(gallery, "metadata.yml") localized_meta_path = utils.get_translation_candidate(self.site.config, base_meta_path, lang) order = [] captions = {} custom_metadata = {} used_path = "" if os.path.isfile(localized_meta_path): used_path = localized_meta_path elif os.path.isfile(base_meta_path): used_path = base_meta_path else: return "", [], {}, {} self.logger.debug("Using {0} for gallery {1}".format( used_path, gallery)) with open(used_path, "r") as meta_file: if yaml is None: utils.req_missing(['PyYAML'], 'use metadata.yml files for galleries') meta = yaml.safe_load_all(meta_file) for img in meta: # load_all and safe_load_all both return None as their # final element, so skip it if not img: continue if 'name' in img: img_name = img.pop('name') if 'caption' in img and img['caption']: captions[img_name] = img.pop('caption') if 'order' in img and img['order'] is not None: order.insert(img.pop('order'), img_name) else: order.append(img_name) custom_metadata[img_name] = img else: self.logger.error("no 'name:' for ({0}) in {1}".format( img, used_path)) return used_path, order, captions, custom_metadata
def gen_tasks(self): """Publish the page sources into the output. Required keyword arguments: translations default_lang post_pages output_folder """ kw = { "translations": self.site.config["TRANSLATIONS"], "output_folder": self.site.config["OUTPUT_FOLDER"], "default_lang": self.site.config["DEFAULT_LANG"], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], } self.site.scan_posts() yield self.group_task() if self.site.config['COPY_SOURCES']: for lang in kw["translations"]: for post in self.site.timeline: if not kw[ "show_untranslated_posts"] and lang not in post.translated_to: continue if post.meta('password'): continue output_name = os.path.join( kw['output_folder'], post.destination_path(lang, post.source_ext(True))) # do not publish PHP sources if post.source_ext(True) == post.compiler.extension(): continue source = post.source_path if lang != kw["default_lang"]: source_lang = utils.get_translation_candidate( self.site.config, source, lang) if os.path.exists(source_lang): source = source_lang if os.path.isfile(source): yield { 'basename': 'render_sources', 'name': os.path.normpath(output_name), 'file_dep': [source], 'targets': [output_name], 'actions': [(utils.copy_file, (source, output_name))], 'clean': True, 'uptodate': [ utils.config_changed( kw, 'nikola.plugins.task.sources') ], }
def test_get_translation_candidate(pattern, path, lang, expected_path): config = { "TRANSLATIONS_PATTERN": pattern, "DEFAULT_LANG": "en", "TRANSLATIONS": { "es": "1", "en": 1 }, } assert get_translation_candidate(config, path, lang) == expected_path
def _execute(self, options, args): L = utils.get_logger('upgrade_metadata', utils.STDERR_HANDLER) nikola.post._UPGRADE_METADATA_ADVERTISED = True # scan posts self.site.scan_posts() flagged = [] for post in self.site.timeline: if not post.newstylemeta: flagged.append(post) if flagged: if len(flagged) == 1: L.info( '1 post (and/or its translations) contains old-style metadata:' ) else: L.info( '{0} posts (and/or their translations) contain old-style metadata:' .format(len(flagged))) for post in flagged: L.info(' ' + post.metadata_path) if not options['yes']: yesno = utils.ask_yesno("Proceed with metadata upgrade?") if options['yes'] or yesno: for post in flagged: for lang in self.site.config['TRANSLATIONS'].keys(): if lang == post.default_lang: fname = post.metadata_path else: meta_path = os.path.splitext( post.source_path)[0] + '.meta' fname = utils.get_translation_candidate( post.config, meta_path, lang) if os.path.exists(fname): with io.open(fname, 'r', encoding='utf-8') as fh: meta = fh.readlines() if not meta[min(1, len(meta) - 1)].startswith('.. '): # check if we’re dealing with old style metadata with io.open(fname, 'w', encoding='utf-8') as fh: for k, v in zip(self.fields, meta): fh.write('.. {0}: {1}'.format(k, v)) L.debug(fname) L.info('{0} posts upgraded.'.format(len(flagged))) else: L.info('Metadata not upgraded.') else: L.info( 'No old-style metadata posts found. No action is required.')
def gen_tasks(self): """Publish the page sources into the output. Required keyword arguments: translations default_lang post_pages output_folder """ kw = { "translations": self.site.config["TRANSLATIONS"], "output_folder": self.site.config["OUTPUT_FOLDER"], "default_lang": self.site.config["DEFAULT_LANG"], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], } self.site.scan_posts() yield self.group_task() if self.site.config['COPY_SOURCES']: for lang in kw["translations"]: for post in self.site.timeline: if not kw["show_untranslated_posts"] and lang not in post.translated_to: continue if post.meta('password'): continue output_name = os.path.join( kw['output_folder'], post.destination_path( lang, post.source_ext(True))) # do not publish PHP sources if post.source_ext(True) == post.compiler.extension(): continue source = post.source_path if lang != kw["default_lang"]: source_lang = utils.get_translation_candidate(self.site.config, source, lang) if os.path.exists(source_lang): source = source_lang if os.path.isfile(source): yield { 'basename': 'render_sources', 'name': os.path.normpath(output_name), 'file_dep': [source], 'targets': [output_name], 'actions': [(utils.copy_file, (source, output_name))], 'clean': True, 'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.sources')], }
def _execute(self, options, args): L = utils.get_logger('upgrade_metadata', utils.STDERR_HANDLER) nikola.post._UPGRADE_METADATA_ADVERTISED = True # scan posts self.site.scan_posts() flagged = [] for post in self.site.timeline: if not post.newstylemeta: flagged.append(post) if flagged: if len(flagged) == 1: L.info('1 post (and/or its translations) contains old-style metadata:') else: L.info('{0} posts (and/or their translations) contain old-style metadata:'.format(len(flagged))) for post in flagged: L.info(' ' + post.metadata_path) if not options['yes']: yesno = utils.ask_yesno("Proceed with metadata upgrade?") if options['yes'] or yesno: for post in flagged: for lang in self.site.config['TRANSLATIONS'].keys(): if lang == post.default_lang: fname = post.metadata_path else: meta_path = os.path.splitext(post.source_path)[0] + '.meta' fname = utils.get_translation_candidate(post.config, meta_path, lang) if os.path.exists(fname): with io.open(fname, 'r', encoding='utf-8') as fh: meta = fh.readlines() if not meta[min(1, len(meta) - 1)].startswith('.. '): # check if we’re dealing with old style metadata with io.open(fname, 'w', encoding='utf-8') as fh: for k, v in zip(self.fields, meta): fh.write('.. {0}: {1}'.format(k, v)) L.debug(fname) L.info('{0} posts upgraded.'.format(len(flagged))) else: L.info('Metadata not upgraded.') else: L.info('No old-style metadata posts found. No action is required.')
def import_item(self, item, wordpress_namespace, out_folder=None): """Takes an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) path = unquote(urlparse(link).path.strip('/')) # In python 2, path is a str. slug requires a unicode # object. According to wikipedia, unquoted strings will # usually be UTF8 if isinstance(path, utils.bytes_str): path = path.decode('utf8') pathlist = path.split('/') if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) slug = utils.slugify(pathlist[-1]) if not slug: # it happens if the post has no "nice" URL slug = get_text_tag( item, '{{{0}}}post_name'.format(wordpress_namespace), None) if not slug: # it *may* happen slug = get_text_tag( item, '{{{0}}}post_id'.format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) return description = get_text_tag(item, 'description', '') post_date = get_text_tag( item, '{{{0}}}post_date'.format(wordpress_namespace), None) dt = utils.to_datetime(post_date) if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag( item, '{{{0}}}status'.format(wordpress_namespace), 'publish') content = get_text_tag( item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') tags = [] if status == 'trash': LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) return elif status != 'publish': tags.append('draft') is_draft = True else: is_draft = False for tag in item.findall('category'): text = tag.text if text == 'Uncategorized': continue tags.append(text) if '$latex' in content: tags.append('mathjax') if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) elif content.strip(): # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + out_folder + '/' + slug + '.html') if hasattr(self, "separate_qtranslate_content") \ and self.separate_qtranslate_content: content_translations = separate_qtranslate_content(content) else: content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): if lang: out_meta_filename = slug + '.meta' if lang == default_language: out_content_filename = slug + '.wp' else: out_content_filename \ = utils.get_translation_candidate(self.context, slug + ".wp", lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + '.meta' out_content_filename = slug + '.wp' meta_slug = slug content = self.transform_content(content) self.write_metadata(os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), content) else: LOGGER.warn('Not going to import "{0}" because it seems to contain' ' no content.'.format(title))
def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None): """Take an item from the feed and creates a post file.""" if out_folder is None: out_folder = "posts" title = get_text_tag(item, "title", "NO TITLE") # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, "link", None) parsed = urlparse(link) path = unquote(parsed.path.strip("/")) try: if isinstance(path, utils.bytes_str): path = path.decode("utf8", "replace") else: path = path except AttributeError: pass # Cut out the base directory. if path.startswith(self.base_dir.strip("/")): path = path.replace(self.base_dir.strip("/"), "", 1) pathlist = path.split("/") if parsed.query: # if there are no nice URLs and query strings are used out_folder = os.path.join(*([out_folder] + pathlist)) slug = get_text_tag(item, "{{{0}}}post_name".format(wordpress_namespace), None) if not slug: # it *may* happen slug = get_text_tag(item, "{{{0}}}post_id".format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) return False else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) slug = utils.slugify(pathlist[-1]) description = get_text_tag(item, "description", "") post_date = get_text_tag(item, "{{{0}}}post_date".format(wordpress_namespace), None) try: dt = utils.to_datetime(post_date) except ValueError: dt = datetime.datetime(1970, 1, 1, 0, 0, 0) LOGGER.error( 'Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format( post_date, title, slug ) ) post_date = dt.strftime("%Y-%m-%d %H:%M:%S") if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag(item, "{{{0}}}status".format(wordpress_namespace), "publish") content = get_text_tag(item, "{http://purl.org/rss/1.0/modules/content/}encoded", "") excerpt = get_text_tag(item, "{http://wordpress.org/export/1.2/excerpt/}encoded", None) if excerpt is not None: if len(excerpt) == 0: excerpt = None tags = [] categories = [] if status == "trash": LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) return False elif status == "private": tags.append("private") is_draft = False is_private = True elif status != "publish": tags.append("draft") is_draft = True is_private = False else: is_draft = False is_private = False for tag in item.findall("category"): text = tag.text type = "category" if "domain" in tag.attrib: type = tag.attrib["domain"] if text == "Uncategorized" and type == "category": continue if type == "category": categories.append(text) else: tags.append(text) if "$latex" in content: tags.append("mathjax") for i, cat in enumerate(categories[:]): cat = self._sanitize(cat, True) categories[i] = cat self.all_tags.add(cat) for i, tag in enumerate(tags[:]): tag = self._sanitize(tag, False) tags[i] = tag self.all_tags.add(tag) # Find post format if it's there post_format = "wp" format_tag = [x for x in item.findall("*//{%s}meta_key" % wordpress_namespace) if x.text == "_tc_post_format"] if format_tag: post_format = format_tag[0].getparent().find("{%s}meta_value" % wordpress_namespace).text if post_format == "wpautop": post_format = "wp" if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) return False elif is_private and self.exclude_privates: LOGGER.notice('Private post "{0}" will not be imported.'.format(title)) return False elif content.strip() or self.import_empty_items: # If no content is found, no files are written. self.url_map[link] = (self.context["SITE_URL"] + out_folder.rstrip("/") + "/" + slug + ".html").replace( os.sep, "/" ) if hasattr(self, "separate_qtranslate_content") and self.separate_qtranslate_content: content_translations = separate_qtranslate_content(content) else: content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): try: content, extension, rewrite_html = self.transform_content(content, post_format, attachments) except: LOGGER.error( ('Cannot interpret post "{0}" (language {1}) with post ' + "format {2}!").format( os.path.join(out_folder, slug), lang, post_format ) ) return False if lang: out_meta_filename = slug + ".meta" if lang == default_language: out_content_filename = slug + "." + extension else: out_content_filename = utils.get_translation_candidate( self.context, slug + "." + extension, lang ) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + ".meta" out_content_filename = slug + "." + extension meta_slug = slug tags, other_meta = self._create_metadata( status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug) ) self.write_metadata( os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags, **other_meta ) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), content, rewrite_html ) if self.export_comments: comments = [] for tag in item.findall("{{{0}}}comment".format(wordpress_namespace)): comment = self._extract_comment(tag, wordpress_namespace) if comment is not None: comments.append(comment) for comment in comments: comment_filename = "{0}.{1}.wpcomment".format(slug, comment["id"]) self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment) return (out_folder, slug) else: LOGGER.warn(('Not going to import "{0}" because it seems to contain' " no content.").format(title)) return False
def scan(self): """Create list of posts from HIERARCHICAL_PAGES options.""" seen = set([]) if not self.site.quiet: print("Scanning hierarchical pages", end='', file=sys.stderr) timeline = [] for wildcard, destination, template_name in self.site.config.get('HIERARCHICAL_PAGES', []): if not self.site.quiet: print(".", end='', file=sys.stderr) root = Node(slugs=_spread(destination, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG'])) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname, followlinks=True): # Get all the untranslated paths dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) # posts/foo/*.rst untranslated = glob.glob(dir_glob) # And now get all the translated paths translated = set([]) for lang in self.site.config['TRANSLATIONS'].keys(): if lang == self.site.config['DEFAULT_LANG']: continue lang_glob = utils.get_translation_candidate(self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst translated = translated.union(set(glob.glob(lang_glob))) # untranslated globs like *.rst often match translated paths too, so remove them # and ensure x.rst is not in the translated set untranslated = set(untranslated) - translated # also remove from translated paths that are translations of # paths in untranslated_list, so x.es.rst is not in the untranslated set for p in untranslated: translated = translated - set([utils.get_translation_candidate(self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys()]) full_list = list(translated) + list(untranslated) # We eliminate from the list the files inside any .ipynb folder full_list = [p for p in full_list if not any([x.startswith('.') for x in p.split(os.sep)])] for base_path in full_list: if base_path in seen: continue else: seen.add(base_path) # Extract path path = utils.os_path_split(os.path.relpath(base_path, dirname)) path[-1] = os.path.splitext(path[-1])[0] if path[-1] == 'index': path = path[:-1] # Find node node = root for path_elt in path: if path_elt not in node.children: node.children[path_elt] = Node(path_elt) node = node.children[path_elt] node.post_source = base_path # Add posts def crawl(node, destinations_so_far, root=True): if node.post_source is not None: try: post = Post( node.post_source, self.site.config, '', False, self.site.MESSAGES, template_name, self.site.get_compiler(node.post_source), destination_base=utils.TranslatableSetting('destinations', destinations_so_far, self.site.config['TRANSLATIONS']), metadata_extractors_by=self.site.metadata_extractors_by ) timeline.append(post) except Exception as err: LOGGER.error('Error reading post {}'.format(base_path)) raise err # Compute slugs slugs = {} for lang in self.site.config['TRANSLATIONS']: slug = post.meta('slug', lang=lang) if slug: slugs[lang] = slug if not slugs: slugs[self.site.config['DEFAULT_LANG']] = node.name node.slugs = _spread(slugs, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG']) # Update destinations_so_far if not root: if node.slugs is not None: destinations_so_far = {lang: os.path.join(dest, node.slugs[lang]) for lang, dest in destinations_so_far.items()} else: destinations_so_far = {lang: os.path.join(dest, node.name) for lang, dest in destinations_so_far.items()} for p, n in node.children.items(): crawl(n, destinations_so_far, root=False) crawl(root, root.slugs) return timeline
def scan(self): """Create list of posts from HIERARCHICAL_PAGES options.""" seen = set([]) if not self.site.quiet: print("Scanning hierarchical pages", end='', file=sys.stderr) timeline = [] for wildcard, destination, template_name in self.site.config.get( 'HIERARCHICAL_PAGES', []): if not self.site.quiet: print(".", end='', file=sys.stderr) root = Node( slugs=_spread(destination, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG'])) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname, followlinks=True): # Get all the untranslated paths dir_glob = os.path.join( dirpath, os.path.basename(wildcard)) # posts/foo/*.rst untranslated = glob.glob(dir_glob) # And now get all the translated paths translated = set([]) for lang in self.site.config['TRANSLATIONS'].keys(): if lang == self.site.config['DEFAULT_LANG']: continue lang_glob = utils.get_translation_candidate( self.site.config, dir_glob, lang) # posts/foo/*.LANG.rst translated = translated.union(set(glob.glob(lang_glob))) # untranslated globs like *.rst often match translated paths too, so remove them # and ensure x.rst is not in the translated set untranslated = set(untranslated) - translated # also remove from translated paths that are translations of # paths in untranslated_list, so x.es.rst is not in the untranslated set for p in untranslated: translated = translated - set([ utils.get_translation_candidate( self.site.config, p, l) for l in self.site.config['TRANSLATIONS'].keys() ]) full_list = list(translated) + list(untranslated) # We eliminate from the list the files inside any .ipynb folder full_list = [ p for p in full_list if not any([x.startswith('.') for x in p.split(os.sep)]) ] for base_path in sorted(full_list): if base_path in seen: continue for lang in self.site.config['TRANSLATIONS'].keys(): seen.add( utils.get_translation_candidate( self.site.config, base_path, lang)) # Extract path path = utils.os_path_split( os.path.relpath(base_path, dirname)) path[-1] = os.path.splitext(path[-1])[0] if path[-1] == 'index': path = path[:-1] # Find node node = root for path_elt in path: if path_elt not in node.children: node.children[path_elt] = Node(path_elt) node = node.children[path_elt] node.post_source = base_path # Add posts def crawl(node, destinations_so_far, root=True): if node.post_source is not None: try: post = Post(node.post_source, self.site.config, '', False, self.site.MESSAGES, template_name, self.site.get_compiler(node.post_source), destination_base=utils.TranslatableSetting( 'destinations', destinations_so_far, self.site.config['TRANSLATIONS'])) timeline.append(post) except Exception as err: LOGGER.error('Error reading post {}'.format(base_path)) raise err # Compute slugs slugs = {} for lang in self.site.config['TRANSLATIONS']: slug = post.meta('slug', lang=lang) if slug: slugs[lang] = slug if not slugs: slugs[self.site.config['DEFAULT_LANG']] = node.name node.slugs = _spread(slugs, self.site.config['TRANSLATIONS'], self.site.config['DEFAULT_LANG']) # Update destinations_so_far if not root: if node.slugs is not None: destinations_so_far = { lang: os.path.join(dest, node.slugs[lang]) for lang, dest in destinations_so_far.items() } else: destinations_so_far = { lang: os.path.join(dest, node.name) for lang, dest in destinations_so_far.items() } for p, n in node.children.items(): crawl(n, destinations_so_far, root=False) crawl(root, root.slugs) return timeline
def gen_tasks(self): self.kw = { 'strip_indexes': self.site.config['STRIP_INDEXES'], 'output_folder': self.site.config['OUTPUT_FOLDER'], 'cache_folder': self.site.config['CACHE_FOLDER'], 'default_lang': self.site.config['DEFAULT_LANG'], 'filters': self.site.config['FILTERS'], 'translations': self.site.config['TRANSLATIONS'], 'global_context': self.site.GLOBAL_CONTEXT, 'tzinfo': self.site.tzinfo, } tasks = {} classes = [] folder_name = "learning" directory = os.path.join(self.site.original_cwd, folder_name) template_name = "tutorials.mako" categories = [] dirs = os.listdir(directory) dirs.sort() files = [] for catfolder in dirs: if not os.path.isdir(os.path.join(directory,catfolder)): continue articles = [] translations = [] category = catfolder[catfolder.find("_")+1:] articlesfiles = os.listdir(os.path.join(directory,catfolder)); articlesfiles.sort() for article in articlesfiles: file_split = os.path.splitext(article) extension = file_split[1] lang_split = article.split(".") lang = self.site.config['DEFAULT_LANG'] if len(lang_split)<3 else lang_split[1] is_translation = lang != self.site.config['DEFAULT_LANG'] folder = os.path.join(directory,catfolder,article) if extension=='.markdown': path = os.path.join(directory,catfolder,article) files += [path] articleobj = MarkdownArticle(path, directory, lang, is_translation) if not is_translation: articles.append(articleobj) else: translations.append(articleobj) elif extension=='.asciidoc': path = os.path.join(directory,catfolder,article) files += [path] articleobj = AsciidocArticle(path, directory, lang, is_translation) if not is_translation: articles.append(articleobj) else: translations.append(articleobj) elif os.path.isdir(folder): for lang in self.kw['translations']: if lang == self.site.config['DEFAULT_LANG']: out_folder = os.path.join(self.site.original_cwd, 'output',folder_name,catfolder,article.lower()) else: out_folder = os.path.join(self.site.original_cwd, 'output',lang,folder_name,catfolder,article.lower()) for root, dirs, file_ins in os.walk(folder): for f in file_ins: in_path = os.path.join(root,f) out_path = os.path.join(out_folder, f) yield utils.apply_filters({ 'basename': self.name, 'name': in_path + "." + lang, 'file_dep': [in_path, __file__], 'targets': [out_path], 'actions': [ (create_file, (in_path, out_path)) ], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters']) def find_translations(article): article_file_name = os.path.splitext(article.file)[0] it = filter((lambda possible_translation: os.path.splitext(os.path.splitext(possible_translation.file)[0])[0] == article_file_name), translations) article_translations = {} for translation in it: if article.modification_date > translation.modification_date: translation.original_newer = True article_translations[translation.lang] = translation return article_translations def collect_translations(article): article.translations = find_translations(article) return article articles = list(map(collect_translations, articles)) categories.append({'category': category, 'articles': articles}); for lang in self.kw['translations']: ### ----------------------------------- ### 3) BOTTOM SECTION: GUIDES FROM OF-BOOK ### of_book_path = os.path.join(directory, "of_book.md") if lang != self.site.config['DEFAULT_LANG']: of_book_lang_path = utils.get_translation_candidate(self.site.config, of_book_path, lang) p = pathlib.Path(of_book_lang_path) if p.exists(): of_book_path = of_book_lang_path of_book = open(of_book_path).read() ### ----------------------------------- context = {} context["lang"] = lang if lang == self.site.config['DEFAULT_LANG']: context["permalink"] = '/' + folder_name + '/' else: context["permalink"] = '/' + lang + '/' + folder_name + '/' context["of_book"] = of_book context["title"] = "learning" context['categories'] = categories short_tdst = os.path.join(self.kw['translations'][lang], folder_name, "index.html") tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst)) template_dep = self.site.template_system.template_deps(template_name) template_dep += files template_dep += [__file__] template_dep += [os.path.join(self.site.original_cwd, "messages", "of_messages_" + lang + ".py")] yield utils.apply_filters({ 'basename': self.name, 'name': tdst, 'file_dep': template_dep, 'targets': [tdst], 'actions': [ (self.site.render_template, (template_name, tdst, context)) ], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters'])
def _execute(self, options, args): L = utils.get_logger('upgrade_metadata_v8', utils.STDERR_HANDLER) if not self.site.config['USE_TAG_METADATA']: L.error('This plugin can only be used if USE_TAG_METADATA is set to True.') sys.exit(-1) self.site.config['WARN_ABOUT_TAG_METADATA'] = False # scan posts self.site.scan_posts() flagged = [] for post in self.site.timeline: flag = False if post.has_oldstyle_metadata_tags: flag = True for lang in self.site.config['TRANSLATIONS'].keys(): if 'section' in post.meta[lang]: flag = True if flag: flagged.append(post) if flagged: if len(flagged) == 1: L.info('1 post (and/or its translations) contains old-style metadata or has section metadata:') else: L.info('{0} posts (and/or their translations) contain old-style metadata or have section metadata:'.format(len(flagged))) for post in flagged: L.info(' ' + (post.metadata_path if post.is_two_file else post.source_path)) L.warn('Please make a backup before running this plugin. It might eat your data.') if not options['yes']: yesno = utils.ask_yesno("Proceed with metadata upgrade?") if options['yes'] or yesno: number_converted = 0 number_converted_partial = 0 for post in flagged: converted = False fully_converted = True for lang in self.site.config['TRANSLATIONS'].keys(): # Get file names and extractor extractor = post.used_extractor[lang] is_two_file = post.is_two_file if lang == post.default_lang: fname = post.metadata_path if is_two_file else post.source_path else: meta_path = os.path.splitext(post.source_path)[0] + '.meta' if is_two_file else post.source_path fname = utils.get_translation_candidate(post.config, meta_path, lang) # We don't handle compilers which extract metadata for now if post.compiler is extractor: L.warn('Cannot convert {0} (language {1}), as metadata was extracted by compiler.'.format(fname, lang)) fully_converted = False continue # Read metadata and text from post file if not os.path.exists(fname): L.debug("File {0} does not exist, skipping.".format(fname)) continue with io.open(fname, "r", encoding="utf-8-sig") as meta_file: source_text = meta_file.read() if not is_two_file: _, content_str = extractor.split_metadata_from_text(source_text) meta = extractor.extract_text(source_text) # Consider metadata mappings sources = {} for m in ('tags', 'status', 'has_math', 'section', 'category'): sources[m] = m for foreign, ours in self.site.config.get('METADATA_MAPPING', {}).get(extractor.map_from, {}).items(): if ours in sources: sources[ours] = foreign for meta_key, hook in self.site.config.get('METADATA_VALUE_MAPPING', {}).get(extractor.map_from, {}).items(): if meta_key in sources.values(): L.warn('Cannot convert {0} (language {1}): a metadata value mapping is defined for "{2}"!'.format(fname, lang, meta_key)) # Update metadata updated = False tags = meta.get(sources['tags'], []) tags_are_string = False if not isinstance(tags, list): tags_are_string = True tags = [tag.strip() for tag in tags.split(',') if tag.strip()] if 'draft' in [_.lower() for _ in tags]: tags.remove('draft') meta[sources['status']] = 'draft' updated = True if 'private' in tags: tags.remove('private') meta[sources['status']] = 'private' updated = True if 'mathjax' in tags: tags.remove('mathjax') meta[sources['has_math']] = 'yes' updated = True if meta.get(sources['section']): if meta.get(sources['category']): L.warn('Cannot completely {0} (language {1}): both section and category are specified. Please determine the correct category to use yourself!'.format(fname, lang)) fully_converted = False else: meta[sources['category']] = meta[sources['section']] del meta[sources['section']] updated = True if tags_are_string: meta[sources['tags']] = ', '.join(tags) if not updated: # Nothing to do (but successful)! converted = True continue # Recombine metadata with post text if necessary, and write back to file meta_str = utils.write_metadata(meta, metadata_format=extractor.name, compiler=post.compiler, comment_wrap=(post.compiler.name != 'rest'), site=self.site) final_str = meta_str if is_two_file else (meta_str + content_str) with io.open(fname, "w", encoding="utf-8") as meta_file: meta_file.write(final_str) converted = True if converted: if fully_converted: number_converted += 1 else: number_converted_partial += 1 L.info('{0} out of {2} posts upgraded; {1} only converted partially ' '(see above output).'.format(number_converted + number_converted_partial, number_converted_partial, len(flagged))) else: L.info('Metadata not upgraded.') else: L.info('No posts found with special tags or section metadata. No action is required.') L.info('You can safely set the USE_TAG_METADATA and the WARN_ABOUT_TAG_METADATA settings to False.')
def import_item(self, item, wordpress_namespace, out_folder=None): """Takes an item from the feed and creates a post file.""" if out_folder is None: out_folder = "posts" title = get_text_tag(item, "title", "NO TITLE") # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, "link", None) parsed = urlparse(link) path = unquote(parsed.path.strip("/")) # In python 2, path is a str. slug requires a unicode # object. According to wikipedia, unquoted strings will # usually be UTF8 if isinstance(path, utils.bytes_str): path = path.decode("utf8") # Cut out the base directory. if path.startswith(self.base_dir.strip("/")): path = path.replace(self.base_dir.strip("/"), "", 1) pathlist = path.split("/") if parsed.query: # if there are no nice URLs and query strings are used out_folder = os.path.join(*([out_folder] + pathlist)) slug = get_text_tag(item, "{{{0}}}post_name".format(wordpress_namespace), None) if not slug: # it *may* happen slug = get_text_tag(item, "{{{0}}}post_id".format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) return else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) slug = utils.slugify(pathlist[-1]) description = get_text_tag(item, "description", "") post_date = get_text_tag(item, "{{{0}}}post_date".format(wordpress_namespace), None) try: dt = utils.to_datetime(post_date) except ValueError: dt = datetime.datetime(1970, 1, 1, 0, 0, 0) LOGGER.error( 'Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format( post_date, title, slug ) ) post_date = dt.strftime("%Y-%m-%d %H:%M:%S") if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag(item, "{{{0}}}status".format(wordpress_namespace), "publish") content = get_text_tag(item, "{http://purl.org/rss/1.0/modules/content/}encoded", "") tags = [] if status == "trash": LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) return elif status != "publish": tags.append("draft") is_draft = True else: is_draft = False for tag in item.findall("category"): text = tag.text if text == "Uncategorized": continue tags.append(text) if "$latex" in content: tags.append("mathjax") if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) elif content.strip(): # If no content is found, no files are written. self.url_map[link] = (self.context["SITE_URL"] + out_folder.rstrip("/") + "/" + slug + ".html").replace( os.sep, "/" ) if hasattr(self, "separate_qtranslate_content") and self.separate_qtranslate_content: content_translations = separate_qtranslate_content(content) else: content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): if lang: out_meta_filename = slug + ".meta" if lang == default_language: out_content_filename = slug + ".wp" else: out_content_filename = utils.get_translation_candidate(self.context, slug + ".wp", lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + ".meta" out_content_filename = slug + ".wp" meta_slug = slug content = self.transform_content(content) self.write_metadata( os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags, ) self.write_content(os.path.join(self.output_folder, out_folder, out_content_filename), content) else: LOGGER.warn('Not going to import "{0}" because it seems to contain' " no content.".format(title))
def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None): """Take an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) parsed = urlparse(link) path = unquote(parsed.path.strip('/')) try: path = path.decode('utf8') except AttributeError: pass # Cut out the base directory. if path.startswith(self.base_dir.strip('/')): path = path.replace(self.base_dir.strip('/'), '', 1) pathlist = path.split('/') if parsed.query: # if there are no nice URLs and query strings are used out_folder = os.path.join(*([out_folder] + pathlist)) slug = get_text_tag( item, '{{{0}}}post_name'.format(wordpress_namespace), None) if not slug: # it *may* happen slug = get_text_tag( item, '{{{0}}}post_id'.format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) return False else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) slug = utils.slugify(pathlist[-1]) description = get_text_tag(item, 'description', '') post_date = get_text_tag( item, '{{{0}}}post_date'.format(wordpress_namespace), None) try: dt = utils.to_datetime(post_date) except ValueError: dt = datetime.datetime(1970, 1, 1, 0, 0, 0) LOGGER.error('Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(post_date, title, slug)) post_date = dt.strftime('%Y-%m-%d %H:%M:%S') if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag( item, '{{{0}}}status'.format(wordpress_namespace), 'publish') content = get_text_tag( item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') excerpt = get_text_tag( item, '{http://wordpress.org/export/1.2/excerpt/}encoded', None) if excerpt is not None: if len(excerpt) == 0: excerpt = None tags = [] categories = [] if status == 'trash': LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) return False elif status == 'private': tags.append('private') is_draft = False is_private = True elif status != 'publish': tags.append('draft') is_draft = True is_private = False else: is_draft = False is_private = False for tag in item.findall('category'): text = tag.text type = 'category' if 'domain' in tag.attrib: type = tag.attrib['domain'] if text == 'Uncategorized' and type == 'category': continue self.all_tags.add(text) if type == 'category': categories.append(type) else: tags.append(text) if '$latex' in content: tags.append('mathjax') # Find post format if it's there post_format = 'wp' format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] if format_tag: post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text if post_format == 'wpautop': post_format = 'wp' if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) return False elif is_private and self.exclude_privates: LOGGER.notice('Private post "{0}" will not be imported.'.format(title)) return False elif content.strip() or self.import_empty_items: # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + out_folder.rstrip('/') + '/' + slug + '.html').replace(os.sep, '/') if hasattr(self, "separate_qtranslate_content") \ and self.separate_qtranslate_content: content_translations = separate_qtranslate_content(content) else: content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): try: content, extension, rewrite_html = self.transform_content(content, post_format, attachments) except: LOGGER.error(('Cannot interpret post "{0}" (language {1}) with post ' + 'format {2}!').format(os.path.join(out_folder, slug), lang, post_format)) return False if lang: out_meta_filename = slug + '.meta' if lang == default_language: out_content_filename = slug + '.' + extension else: out_content_filename \ = utils.get_translation_candidate(self.context, slug + "." + extension, lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + '.meta' out_content_filename = slug + '.' + extension meta_slug = slug tags, other_meta = self._create_metadata(status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug)) self.write_metadata(os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags, **other_meta) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), content, rewrite_html) if self.export_comments: comments = [] for tag in item.findall('{{{0}}}comment'.format(wordpress_namespace)): comment = self._extract_comment(tag, wordpress_namespace) if comment is not None: comments.append(comment) for comment in comments: comment_filename = slug + "." + str(comment['id']) + ".wpcomment" self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment) return (out_folder, slug) else: LOGGER.warn(('Not going to import "{0}" because it seems to contain' ' no content.').format(title)) return False
def create_docs(self): tasks = {} classes = [] directory = "documentation" classes = markdown_file.getclass_list() classes_simple_name = markdown_file.getclass_list(False) addon_classes = markdown_file.list_all_addons() module_lookup = dict() core_index = dict() addons_index = dict() module_subtitles = dict() docs_dir = os.path.join(self.site.original_cwd, "documentation") md_extensions = self.site.config.get("MARKDOWN_EXTENSIONS") # Create an index of which module each class is in for generated links to other classes for class_name in classes: clazz = markdown_file.getclass(class_name) if clazz.istemplated: module_lookup[class_name[:-1]] = clazz.module else: module_lookup[class_name] = clazz.module for clazz_name in classes: clazz = markdown_file.getclass(clazz_name) if clazz.istemplated: clazz.name = clazz.name[:-1] clazz.detailed_inline_description = markdown(clazz.detailed_inline_description, md_extensions) # clazz.description = str(markdown(clazz.description, md_extensions).encode('ascii', 'ignore')) for class_name in classes_simple_name: rep = class_name + "[\s]" clazz.detailed_inline_description = re.sub( rep, '<a href="/documentation/' + module_lookup[class_name] + "/" + class_name + '" class="docs_class" >' + class_name + "</a> ", clazz.detailed_inline_description, ) rep = class_name + "[(]" clazz.detailed_inline_description = re.sub( rep, '<a href="/documentation/' + module_lookup[class_name] + "/" + class_name + '" class="docs_class" >' + class_name + "</a>(", clazz.detailed_inline_description, ) clazz.reference = markdown(clazz.reference, md_extensions) for class_name in classes_simple_name: rep = class_name + "[\s]" clazz.reference = re.sub( rep, '<a href="/documentation/' + module_lookup[class_name] + "/" + class_name + '" class="docs_class" >' + class_name + "</a> ", clazz.reference, ) rep = class_name + "[(]" clazz.reference = re.sub( rep, '<a href="/documentation/' + module_lookup[class_name] + "/" + class_name + '" class="docs_class" >' + class_name + "</a>(", clazz.reference, ) for function in clazz.function_list: function.description = markdown(function.description, md_extensions) function.inlined_description = markdown(function.inlined_description, md_extensions) def gen_link(class_name): return ( '<a href="/documentation/' + module_lookup[class_name] + "/" + class_name + '" class="docs_class" >' + class_name + "</a> " if class_name in module_lookup else "" ) def filter_out_empty(class_name): return class_name != "" clazz.extends = list(filter(filter_out_empty, map(gen_link, clazz.extends))) functions_file = markdown_file.getfunctionsfile(clazz.name) for function in functions_file.function_list: function.description = markdown(function.description, md_extensions) function.inlined_description = markdown(function.inlined_description, md_extensions) # print clazz.name # print clazz.function_list env = { "modulename": clazz.name, "clazz": clazz, "functions": functions_file, "classes_list": classes, "is_addon": (clazz.name in addon_classes), } # print("class " + clazz_name) template_name = "documentation_class.mako" for lang in self.kw["translations"]: env["lang"] = lang env["title"] = clazz.name env["permalink"] = ( self.kw["translations"][lang] + "/documentation/" + clazz.module + "/" + clazz.name + "/" ) short_tdst = os.path.join( self.kw["translations"][lang], "documentation", clazz.module, clazz.name, "index.html" ) tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst)) self.site.render_template(template_name, tdst, env) if not clazz.module in addon_classes: if not clazz.module in core_index.keys(): core_index[clazz.module] = [] if functions_file != None: for function in functions_file.function_list: clazz.function_list.append(function) core_index[clazz.module].append(clazz) else: if not clazz.module in addons_index.keys(): addons_index[clazz.module] = [] if functions_file != None: for function in functions_file.function_list: clazz.function_list.append(function) addons_index[clazz.module].append(clazz) function_files = markdown_file.getfunctionsfiles_list() for functionfile_name in function_files: if functionfile_name in classes_simple_name: continue functions_file = markdown_file.getfunctionsfile(functionfile_name) # might be needed at some point? # functions_file.reference = str(functions_file.reference) # for func in function_files: # functions_file.reference = str.replace(functions_file.reference, class_name, "<a href=\"../"+clazz.module+"/"+class_name+".html\">"+class_name+"</a>") for function in functions_file.function_list: function.description = markdown(function.description, md_extensions) function.inlined_description = markdown(function.inlined_description, md_extensions) env = { "modulename": functions_file.name, "clazz": None, "functions": functions_file, "is_addon": (functions_file.name in addon_classes), } template_name = "documentation_class.mako" for lang in self.kw["translations"]: env["lang"] = lang env["title"] = clazz.name env["permalink"] = ( self.kw["translations"][lang] + "/documentation/" + functions_file.module + "/" + functions_file.name + "/" ) short_tdst = os.path.join( self.kw["translations"][lang], "documentation", functions_file.module, functions_file.name, "index.html", ) tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst)) self.site.render_template(template_name, tdst, env) if not functions_file.module in addon_classes: if not functions_file.module in core_index: core_index[functions_file.module] = [] core_index[functions_file.module].append(functions_file) else: if not functions_file.module in addons_index: addons_index[functions_file.module] = [] addons_index[functions_file.module].append(functions_file) for root, dirs, files in os.walk(directory): """ copy images to their folders """ for name in files: file_split = os.path.splitext(name) if ( file_split[1] == ".jpeg" or file_split[1] == ".jpg" or file_split[1] == ".gif" or file_split[1] == ".png" ): try: os.mkdir(os.path.join("_site", "documentation", os.path.basename(root))) except: pass shutil.copyfile( os.path.join(root, name), os.path.join("output", "documentation", os.path.basename(root), name) ) """ create module introductions """ for module in dirs: if module != "addons": module_intro = os.path.join(root, module, "introduction.markdown") if os.path.isfile(module_intro): module_intro_file = open(module_intro) module_intro_content = module_intro_file.read() module_subtitles[module] = module_intro_content.splitlines()[0].strip("##").strip(" ") module_intro_content = markdown(module_intro_content, md_extensions) template_name = "documentation_module_intro.mako" for lang in self.kw["translations"]: context = {} context["lang"] = lang context["title"] = clazz.name context["module"] = module context["intro_content"] = module_intro_content context["permalink"] = ( self.kw["translations"][lang] + "/documentation/" + module + "/introduction.html" ) short_tdst = os.path.join( self.kw["translations"][lang], "documentation", module, "introduction.html" ) tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst)) if module.find("ofx") == 0: context["classes"] = addons_index[module] self.site.render_template(template_name, tdst, context) else: context["classes"] = core_index[module] self.site.render_template(template_name, tdst, context) else: module_subtitles[module] = None print("couldn't find " + module_intro) # process index file template_name = "documentation.mako" for lang in self.kw["translations"]: # lang_suffix = self.kw['translations'][lang] docs_intro_path = os.path.join(docs_dir, "index.md") if lang != self.site.config["DEFAULT_LANG"]: docs_intro_lang_path = utils.get_translation_candidate(self.site.config, docs_intro_path, lang) p = pathlib.Path(docs_intro_lang_path) if p.exists(): docs_intro_path = docs_intro_lang_path docs_intro = open(docs_intro_path).read() for key in self.site.GLOBAL_CONTEXT.keys(): if isinstance(self.site.GLOBAL_CONTEXT[key], str): docs_intro = docs_intro.replace("${" + key + "}", self.site.GLOBAL_CONTEXT[key]) docs_intro = markdown(docs_intro, md_extensions) context = {} context["lang"] = lang context["title"] = "documentation" context["docs_intro"] = docs_intro context["core"] = core_index context["addons"] = addons_index context["module_subtitles"] = module_subtitles context["permalink"] = self.kw["translations"][lang] + "/documentation/" short_tdst = os.path.join(self.kw["translations"][lang], "documentation", "index.html") tdst = os.path.normpath(os.path.join(self.kw["output_folder"], short_tdst)) self.site.render_template(template_name, tdst, context)
def test_get_translation_candidate(): config = {'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}', 'DEFAULT_LANG': 'en', 'TRANSLATIONS': {'es': '1', 'en': 1}} assert get_translation_candidate(config, '*.rst', 'es') == '*.es.rst' assert get_translation_candidate( config, 'fancy.post.rst', 'es') == 'fancy.post.es.rst' assert get_translation_candidate(config, '*.es.rst', 'es') == '*.es.rst' assert get_translation_candidate(config, '*.es.rst', 'en') == '*.rst' assert get_translation_candidate( config, 'cache/posts/fancy.post.es.html', 'en') == 'cache/posts/fancy.post.html' assert get_translation_candidate( config, 'cache/posts/fancy.post.html', 'es') == 'cache/posts/fancy.post.es.html' assert get_translation_candidate( config, 'cache/pages/charts.html', 'es') == 'cache/pages/charts.es.html' assert get_translation_candidate( config, 'cache/pages/charts.html', 'en') == 'cache/pages/charts.html' config = {'TRANSLATIONS_PATTERN': '{path}.{ext}.{lang}', 'DEFAULT_LANG': 'en', 'TRANSLATIONS': {'es': '1', 'en': 1}} assert get_translation_candidate(config, '*.rst', 'es') == '*.rst.es' assert get_translation_candidate(config, '*.rst.es', 'es') == '*.rst.es' assert get_translation_candidate(config, '*.rst.es', 'en') == '*.rst' assert get_translation_candidate( config, 'cache/posts/fancy.post.html.es', 'en') == 'cache/posts/fancy.post.html' assert get_translation_candidate( config, 'cache/posts/fancy.post.html', 'es') == 'cache/posts/fancy.post.html.es'
def create_docs(self): tasks = {} classes = [] directory = "documentation" classes = markdown_file.getclass_list() classes_simple_name = markdown_file.getclass_list(False) addon_classes = markdown_file.list_all_addons() module_lookup = dict() core_index = dict() addons_index = dict() module_subtitles = dict() docs_dir = os.path.join(self.site.original_cwd, "documentation") md_extensions = self.site.config.get("MARKDOWN_EXTENSIONS") content_js = {} class_template = "documentation_class.mako" class_template_dep = self.site.template_system.template_deps( class_template) module_template = "documentation_module_intro.mako" # start js string for docs search for lang in self.kw['translations']: content_js[lang] = 'var tipuesearch = {"pages": [' # Create an index of which module each class is in for generated links to other classes for class_name in classes: clazz = markdown_file.getclass(class_name) if clazz.istemplated: module_lookup[class_name[:-1]] = clazz.module else: module_lookup[class_name] = clazz.module # classes docs for clazz_name in classes: clazz = markdown_file.getclass(clazz_name) if clazz.istemplated: clazz.name = clazz.name[:-1] clazz.detailed_inline_description = relative_urls( clazz.detailed_inline_description) clazz.detailed_inline_description = markdown( clazz.detailed_inline_description, md_extensions) clazz.detailed_inline_description = of_classes_to_links( clazz.detailed_inline_description, classes_simple_name, module_lookup) clazz.reference = relative_urls(clazz.reference) clazz.reference = markdown(clazz.reference, md_extensions) clazz.reference = of_classes_to_links(clazz.reference, classes_simple_name, module_lookup) # methods in class for function in clazz.function_list: function.description = relative_urls(function.description) function.description = markdown(function.description, md_extensions) function.description = of_classes_to_links( function.description, classes_simple_name, module_lookup) function.inlined_description = relative_urls( function.inlined_description) function.inlined_description = markdown( function.inlined_description, md_extensions) function.inlined_description = of_classes_to_links( function.inlined_description, classes_simple_name, module_lookup) for lang in self.kw['translations']: content_js[lang] += method_to_js(function, clazz, self.site, lang) # inheritance def gen_link(class_name): return "<a href=\"/documentation/" + module_lookup[ class_name] + "/" + class_name + "\" class=\"docs_class\" >" + class_name + "</a> " if class_name in module_lookup else "" def filter_out_empty(class_name): return class_name != "" clazz.extends = list( filter(filter_out_empty, map(gen_link, clazz.extends))) # c functions in the class file functions_file = markdown_file.getfunctionsfile(clazz.name) for function in functions_file.function_list: function.description = relative_urls(function.description) function.description = markdown(function.description, md_extensions) function.description = of_classes_to_links( function.description, classes_simple_name, module_lookup) function.inlined_description = relative_urls( function.inlined_description) function.inlined_description = markdown( function.inlined_description, md_extensions) function.inlined_description = of_classes_to_links( function.inlined_description, classes_simple_name, module_lookup) for lang in self.kw['translations']: content_js[lang] += function_to_js(function, functions_file, self.site, lang) # render template + js for search env = { "modulename": clazz.name, "clazz": clazz, "functions": functions_file, "classes_list": classes, "is_addon": (clazz.name in addon_classes) } md_file = "documentation/" + module_lookup[ class_name] + "/" + class_name + ".markdown" for lang in self.kw['translations']: env["lang"] = lang env["title"] = clazz.name env["permalink"] = self.kw['translations'][ lang] + '/documentation/' + clazz.module + "/" + clazz.name + "/" short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', clazz.module, clazz.name, "index.html") tdst = os.path.normpath( os.path.join(self.kw['output_folder'], short_tdst)) """yield utils.apply_filters({ 'basename': self.name, 'name': clazz.name, 'file_dep': class_template_dep + md_file, 'targets': tdst, 'actions': [ (self.site.render_template, (template_name, tdst, env)) ], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters'])""" self.site.render_template(class_template, tdst, env) content_js[lang] += class_to_js(clazz, self.site, lang) # add to index core or addons if not clazz.module in addon_classes: if not clazz.module in core_index.keys(): core_index[clazz.module] = [] if functions_file != None: for function in functions_file.function_list: clazz.function_list.append(function) core_index[clazz.module].append(clazz) else: if not clazz.module in addons_index.keys(): addons_index[clazz.module] = [] if functions_file != None: for function in functions_file.function_list: clazz.function_list.append(function) addons_index[clazz.module].append(clazz) # generate c functions docs function_files = markdown_file.getfunctionsfiles_list() for functionfile_name in function_files: if functionfile_name in classes_simple_name: continue functions_file = markdown_file.getfunctionsfile(functionfile_name) # might be needed at some point? # functions_file.reference = str(functions_file.reference) # for func in function_files: # functions_file.reference = str.replace(functions_file.reference, class_name, "<a href=\"../"+clazz.module+"/"+class_name+".html\">"+class_name+"</a>") for function in functions_file.function_list: function.description = relative_urls(function.description) function.description = markdown(function.description, md_extensions) function.description = of_classes_to_links( function.description, classes_simple_name, module_lookup) function.inlined_description = relative_urls( function.inlined_description) function.inlined_description = markdown( function.inlined_description, md_extensions) function.inlined_description = of_classes_to_links( function.inlined_description, classes_simple_name, module_lookup) for lang in self.kw['translations']: content_js[lang] += function_to_js(function, functions_file, self.site, lang) # render template + js for search env = { "modulename": functions_file.name, "clazz": None, "functions": functions_file, "is_addon": (functions_file.name in addon_classes) } for lang in self.kw['translations']: env["lang"] = lang env["title"] = clazz.name env["permalink"] = self.kw['translations'][ lang] + '/documentation/' + functions_file.module + "/" + functions_file.name + "/" short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', functions_file.module, functions_file.name, "index.html") tdst = os.path.normpath( os.path.join(self.kw['output_folder'], short_tdst)) self.site.render_template(class_template, tdst, env) content_js[lang] += functions_file_to_js( functions_file, self.site, lang) # add to index core or addons if not functions_file.module in addon_classes: if not functions_file.module in core_index: core_index[functions_file.module] = [] core_index[functions_file.module].append(functions_file) else: if not functions_file.module in addons_index: addons_index[functions_file.module] = [] addons_index[functions_file.module].append(functions_file) # copy images and render intros for root, dirs, files in os.walk(directory): """ copy images to their folders """ for name in files: file_split = os.path.splitext(name) if file_split[1] == ".jpeg" or file_split[ 1] == ".jpg" or file_split[1] == ".gif" or file_split[ 1] == ".png": try: os.mkdir( os.path.join('_site', 'documentation', os.path.basename(root))) except: pass shutil.copyfile( os.path.join(root, name), os.path.join('output', 'documentation', os.path.basename(root), name)) """ create module introductions """ for module in dirs: if module != "addons": module_intro = os.path.join(root, module, "introduction.markdown") if os.path.isfile(module_intro): module_intro_file = open(module_intro) module_intro_content = module_intro_file.read() module_subtitles[ module] = module_intro_content.splitlines( )[0].strip('##').strip(' ') module_intro_content = markdown( module_intro_content, md_extensions) for lang in self.kw['translations']: context = {} context["lang"] = lang context["title"] = module context["module"] = module context["intro_content"] = module_intro_content context["permalink"] = self.kw['translations'][ lang] + '/documentation/' + module + "/" if lang == self.site.config['DEFAULT_LANG']: short_tdst = os.path.join( 'documentation', module, "index.html") else: short_tdst = os.path.join( self.kw['translations'][lang], 'documentation', module, "index.html") tdst = os.path.normpath( os.path.join(self.kw['output_folder'], short_tdst)) if module.find("ofx") == 0: context["classes"] = addons_index[module] self.site.render_template( module_template, tdst, context) else: context["classes"] = core_index[module] self.site.render_template( module_template, tdst, context) content_js[lang] += module_to_js( module, module_intro_content, self.site, lang) else: module_subtitles[module] = None # close js for docs search and save per language for lang in self.kw['translations']: content_js[lang] += ']};' content_js_file = open( "output" + lang_prefix(lang, self.site) + "/tipuesearch_content.js", "w") content_js_file.write(content_js[lang]) content_js_file.close() # render index file template_name = "documentation.mako" for lang in self.kw['translations']: #lang_suffix = self.kw['translations'][lang] docs_intro_path = os.path.join(docs_dir, "index.md") if lang != self.site.config['DEFAULT_LANG']: docs_intro_lang_path = utils.get_translation_candidate( self.site.config, docs_intro_path, lang) p = pathlib.Path(docs_intro_lang_path) if p.exists(): docs_intro_path = docs_intro_lang_path docs_intro = open(docs_intro_path).read() for key in self.site.GLOBAL_CONTEXT.keys(): if isinstance(self.site.GLOBAL_CONTEXT[key], str): docs_intro = docs_intro.replace( '${' + key + '}', self.site.GLOBAL_CONTEXT[key]) docs_intro = markdown(docs_intro, md_extensions) context = {} context["lang"] = lang context["title"] = "documentation" context["docs_intro"] = docs_intro context['core'] = core_index context['addons'] = addons_index context['module_subtitles'] = module_subtitles context["permalink"] = self.kw['translations'][ lang] + '/documentation/' short_tdst = os.path.join(self.kw['translations'][lang], "documentation", "index.html") tdst = os.path.normpath( os.path.join(self.kw['output_folder'], short_tdst)) self.site.render_template(template_name, tdst, context)
def import_item(self, item, wordpress_namespace, out_folder=None): """Takes an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) parsed = urlparse(link) path = unquote(parsed.path.strip('/')) # In python 2, path is a str. slug requires a unicode # object. According to wikipedia, unquoted strings will # usually be UTF8 if isinstance(path, utils.bytes_str): path = path.decode('utf8') # Cut out the base directory. if path.startswith(self.base_dir.strip('/')): path = path.replace(self.base_dir.strip('/'), '', 1) pathlist = path.split('/') if parsed.query: # if there are no nice URLs and query strings are used out_folder = os.path.join(*([out_folder] + pathlist)) slug = get_text_tag(item, '{{{0}}}post_name'.format(wordpress_namespace), None) if not slug: # it *may* happen slug = get_text_tag( item, '{{{0}}}post_id'.format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) return else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) slug = utils.slugify(pathlist[-1]) description = get_text_tag(item, 'description', '') post_date = get_text_tag( item, '{{{0}}}post_date'.format(wordpress_namespace), None) try: dt = utils.to_datetime(post_date) except ValueError: dt = datetime.datetime(1970, 1, 1, 0, 0, 0) LOGGER.error( 'Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.' .format(post_date, title, slug)) post_date = dt.strftime('%Y-%m-%d %H:%M:%S') if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag(item, '{{{0}}}status'.format(wordpress_namespace), 'publish') content = get_text_tag( item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') tags = [] if status == 'trash': LOGGER.warn( 'Trashed post "{0}" will not be imported.'.format(title)) return elif status != 'publish': tags.append('draft') is_draft = True else: is_draft = False for tag in item.findall('category'): text = tag.text if text == 'Uncategorized': continue tags.append(text) if '$latex' in content: tags.append('mathjax') if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) elif content.strip(): # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + out_folder.rstrip('/') + '/' + slug + '.html').replace(os.sep, '/') if hasattr(self, "separate_qtranslate_content") \ and self.separate_qtranslate_content: content_translations = separate_qtranslate_content(content) else: content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): if lang: out_meta_filename = slug + '.meta' if lang == default_language: out_content_filename = slug + '.wp' else: out_content_filename \ = utils.get_translation_candidate(self.context, slug + ".wp", lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + '.meta' out_content_filename = slug + '.wp' meta_slug = slug content = self.transform_content(content) self.write_metadata( os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), content) else: LOGGER.warn('Not going to import "{0}" because it seems to contain' ' no content.'.format(title))
def import_item(self, item, wordpress_namespace, out_folder=None): """Takes an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) parsed = urlparse(link) path = unquote(parsed.path.strip('/')) # In python 2, path is a str. slug requires a unicode # object. According to wikipedia, unquoted strings will # usually be UTF8 if isinstance(path, utils.bytes_str): path = path.decode('utf8') # Cut out the base directory. if path.startswith(self.base_dir.strip('/')): path = path.replace(self.base_dir.strip('/'), '', 1) pathlist = path.split('/') if parsed.query: # if there are no nice URLs and query strings are used out_folder = os.path.join(*([out_folder] + pathlist)) slug = get_text_tag( item, '{{{0}}}post_name'.format(wordpress_namespace), None) if not slug: # it *may* happen slug = get_text_tag( item, '{{{0}}}post_id'.format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) return else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) slug = utils.slugify(pathlist[-1]) description = get_text_tag(item, 'description', '') post_date = get_text_tag( item, '{{{0}}}post_date'.format(wordpress_namespace), None) try: dt = utils.to_datetime(post_date) except ValueError: dt = datetime.datetime(1970, 1, 1, 0, 0, 0) LOGGER.error('Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(post_date, title, slug)) post_date = dt.strftime('%Y-%m-%d %H:%M:%S') if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag( item, '{{{0}}}status'.format(wordpress_namespace), 'publish') content = get_text_tag( item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') tags = [] if status == 'trash': LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) return elif status != 'publish': tags.append('draft') is_draft = True else: is_draft = False for tag in item.findall('category'): text = tag.text if text == 'Uncategorized': continue tags.append(text) if '$latex' in content: tags.append('mathjax') # Find post format if it's there post_format = 'wp' format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] if format_tag: post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) elif content.strip(): # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + out_folder.rstrip('/') + '/' + slug + '.html').replace(os.sep, '/') if hasattr(self, "separate_qtranslate_content") \ and self.separate_qtranslate_content: content_translations = separate_qtranslate_content(content) else: content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): if lang: out_meta_filename = slug + '.meta' if lang == default_language: out_content_filename = slug + '.wp' else: out_content_filename \ = utils.get_translation_candidate(self.context, slug + ".wp", lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + '.meta' out_content_filename = slug + '.wp' meta_slug = slug if post_format == 'wp': content = self.transform_content(content) self.write_metadata(os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), content) else: LOGGER.warn('Not going to import "{0}" because it seems to contain' ' no content.'.format(title))
def gen_tasks(self): self.kw = { 'strip_indexes': self.site.config['STRIP_INDEXES'], 'output_folder': self.site.config['OUTPUT_FOLDER'], 'cache_folder': self.site.config['CACHE_FOLDER'], 'default_lang': self.site.config['DEFAULT_LANG'], 'filters': self.site.config['FILTERS'], 'translations': self.site.config['TRANSLATIONS'], 'global_context': self.site.GLOBAL_CONTEXT, 'tzinfo': self.site.tzinfo, } tasks = {} classes = [] directory = os.path.join(self.site.original_cwd, "tutorials") template_name = "tutorials.mako" categories = [] dirs = os.listdir(directory) dirs.sort() files = [] for catfolder in dirs: if not os.path.isdir(os.path.join(directory, catfolder)): continue articles = [] translations = [] category = catfolder[catfolder.find("_") + 1:] articlesfiles = os.listdir(os.path.join(directory, catfolder)) articlesfiles.sort() for article in articlesfiles: file_split = os.path.splitext(article) extension = file_split[1] lang_split = article.split(".") lang = self.site.config['DEFAULT_LANG'] if len( lang_split) < 3 else lang_split[1] is_translation = lang != self.site.config['DEFAULT_LANG'] folder = os.path.join(directory, catfolder, article) if extension == '.markdown': path = os.path.join(directory, catfolder, article) files += [path] articleobj = MarkdownArticle(path, directory, lang, is_translation) if not is_translation: articles.append(articleobj) else: translations.append(articleobj) elif extension == '.asciidoc': path = os.path.join(directory, catfolder, article) files += [path] articleobj = AsciidocArticle(path, directory, lang, is_translation) if not is_translation: articles.append(articleobj) else: translations.append(articleobj) elif os.path.isdir(folder): for lang in self.kw['translations']: if lang == self.site.config['DEFAULT_LANG']: out_folder = os.path.join(self.site.original_cwd, 'output', 'tutorials', catfolder, article.lower()) else: out_folder = os.path.join(self.site.original_cwd, 'output', lang, 'tutorials', catfolder, article.lower()) for root, dirs, file_ins in os.walk(folder): for f in file_ins: in_path = os.path.join(root, f) out_path = os.path.join(out_folder, f) yield utils.apply_filters( { 'basename': self.name, 'name': in_path + "." + lang, 'file_dep': [in_path, __file__], 'targets': [out_path], 'actions': [(create_file, (in_path, out_path))], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters']) def find_translations(article): article_file_name = os.path.splitext(article.file)[0] it = filter((lambda possible_translation: os.path.splitext( os.path.splitext(possible_translation.file)[0])[0] == article_file_name), translations) article_translations = {} for translation in it: article_translations[translation.lang] = translation return article_translations def collect_translations(article): article.translations = find_translations(article) return article articles = list(map(collect_translations, articles)) categories.append({ 'category': category, 'articles': articles }) for lang in self.kw['translations']: tutorials_intro_path = os.path.join(directory, "index.md") if lang != self.site.config['DEFAULT_LANG']: tutorials_intro_lang_path = utils.get_translation_candidate( self.site.config, tutorials_intro_path, lang) p = pathlib.Path(tutorials_intro_lang_path) if p.exists(): tutorials_intro_path = tutorials_intro_lang_path tutorials_intro = open(tutorials_intro_path).read() context = {} context["lang"] = lang if lang == self.site.config['DEFAULT_LANG']: context["permalink"] = '/tutorials/' else: context["permalink"] = '/' + lang + '/tutorials/' context["tutorials_intro"] = tutorials_intro context["title"] = "tutorials" context['categories'] = categories short_tdst = os.path.join(self.kw['translations'][lang], "tutorials", "index.html") tdst = os.path.normpath( os.path.join(self.kw['output_folder'], short_tdst)) template_dep = self.site.template_system.template_deps( template_name) template_dep += files template_dep += [__file__] yield utils.apply_filters( { 'basename': self.name, 'name': tdst, 'file_dep': template_dep, 'targets': [tdst], 'actions': [(self.site.render_template, (template_name, tdst, context))], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters'])
def create_docs(self): tasks = {} classes = [] directory = "documentation" classes = markdown_file.getclass_list() classes_simple_name = markdown_file.getclass_list(False) addon_classes = markdown_file.list_all_addons() module_lookup = dict() core_index = dict() addons_index = dict() module_subtitles = dict() docs_dir = os.path.join(self.site.original_cwd, "documentation") md_extensions = self.site.config.get("MARKDOWN_EXTENSIONS") content_js = {} class_template = "documentation_class.mako" class_template_dep = self.site.template_system.template_deps(class_template) module_template = "documentation_module_intro.mako" # start js string for docs search for lang in self.kw['translations']: content_js[lang] = 'var tipuesearch = {"pages": [' # Create an index of which module each class is in for generated links to other classes for class_name in classes: clazz = markdown_file.getclass(class_name) if clazz.istemplated: module_lookup[class_name[:-1]] = clazz.module else: module_lookup[class_name] = clazz.module # classes docs for clazz_name in classes: clazz = markdown_file.getclass(clazz_name) if clazz.istemplated: clazz.name = clazz.name[:-1] clazz.detailed_inline_description = relative_urls(clazz.detailed_inline_description) clazz.detailed_inline_description = markdown(clazz.detailed_inline_description, md_extensions) clazz.detailed_inline_description = of_classes_to_links(clazz.detailed_inline_description, classes_simple_name, module_lookup) clazz.reference = relative_urls(clazz.reference) clazz.reference = markdown(clazz.reference, md_extensions) clazz.reference = of_classes_to_links(clazz.reference, classes_simple_name, module_lookup) # methods in class for function in clazz.function_list: function.description = relative_urls(function.description) function.description = of_classes_to_links(function.description, classes_simple_name, module_lookup) function.description = markdown(function.description, md_extensions) function.inlined_description = relative_urls(function.inlined_description) function.inlined_description = of_classes_to_links(function.inlined_description, classes_simple_name, module_lookup) function.inlined_description = markdown(function.inlined_description, md_extensions) for lang in self.kw['translations']: content_js[lang] += method_to_js(function, clazz, self.site, lang) # inheritance def gen_link(class_name): return "<a href=\"/documentation/" + module_lookup[class_name] + "/" + class_name + "\" class=\"docs_class\" >"+class_name+"</a> " if class_name in module_lookup else "" def filter_out_empty(class_name): return class_name!="" clazz.extends = list(filter(filter_out_empty, map(gen_link, clazz.extends))) # c functions in the class file functions_file = markdown_file.getfunctionsfile(clazz.name) for function in functions_file.function_list: function.description = relative_urls(function.description) function.description = of_classes_to_links(function.description, classes_simple_name, module_lookup) function.description = markdown(function.description, md_extensions) function.inlined_description = relative_urls(function.inlined_description) function.inlined_description = of_classes_to_links(function.inlined_description, classes_simple_name, module_lookup) function.inlined_description = markdown(function.inlined_description, md_extensions) for lang in self.kw['translations']: content_js[lang] += function_to_js(function, functions_file, self.site, lang) # render template + js for search env = { "modulename": clazz.name, "clazz": clazz, "functions": functions_file, "classes_list": classes, "is_addon": (clazz.name in addon_classes) } md_file = "documentation/" + module_lookup[class_name] + "/" + class_name + ".markdown" for lang in self.kw['translations']: env["lang"] = lang env["title"] = clazz.name env["permalink"] = self.kw['translations'][lang] + '/documentation/' + clazz.module + "/" + clazz.name + "/" short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', clazz.module, clazz.name,"index.html") tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst)) """yield utils.apply_filters({ 'basename': self.name, 'name': clazz.name, 'file_dep': class_template_dep + md_file, 'targets': tdst, 'actions': [ (self.site.render_template, (template_name, tdst, env)) ], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters'])""" self.site.render_template(class_template, tdst, env) content_js[lang] += class_to_js(clazz, self.site, lang) # add to index core or addons if not clazz.module in addon_classes: if not clazz.module in core_index.keys(): core_index[clazz.module] = [] if functions_file!=None: for function in functions_file.function_list: clazz.function_list.append(function) core_index[clazz.module].append(clazz) else: if not clazz.module in addons_index.keys(): addons_index[clazz.module] = [] if functions_file!=None: for function in functions_file.function_list: clazz.function_list.append(function) addons_index[clazz.module].append(clazz) # generate c functions docs function_files = markdown_file.getfunctionsfiles_list() for functionfile_name in function_files: if functionfile_name in classes_simple_name: continue functions_file = markdown_file.getfunctionsfile(functionfile_name) # might be needed at some point? # functions_file.reference = str(functions_file.reference) # for func in function_files: # functions_file.reference = str.replace(functions_file.reference, class_name, "<a href=\"../"+clazz.module+"/"+class_name+".html\">"+class_name+"</a>") for function in functions_file.function_list: function.description = relative_urls(function.description) function.description = of_classes_to_links(function.description, classes_simple_name, module_lookup) function.description = markdown(function.description, md_extensions) function.inlined_description = relative_urls(function.inlined_description) function.inlined_description = of_classes_to_links(function.inlined_description, classes_simple_name, module_lookup) function.inlined_description = markdown(function.inlined_description, md_extensions) for lang in self.kw['translations']: content_js[lang] += function_to_js(function, functions_file, self.site, lang) # render template + js for search env = { "modulename": functions_file.name, "clazz": None, "functions": functions_file, "is_addon": (functions_file.name in addon_classes) } for lang in self.kw['translations']: env["lang"] = lang env["title"] = clazz.name env["permalink"] = self.kw['translations'][lang] + '/documentation/' + functions_file.module + "/" + functions_file.name + "/" short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', functions_file.module, functions_file.name,"index.html") tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst)) self.site.render_template(class_template, tdst, env) content_js[lang] += functions_file_to_js(functions_file, self.site, lang) # add to index core or addons if not functions_file.module in addon_classes: if not functions_file.module in core_index: core_index[functions_file.module] = [] core_index[functions_file.module].append(functions_file) else: if not functions_file.module in addons_index: addons_index[functions_file.module] = [] addons_index[functions_file.module].append(functions_file) # copy images and render intros for root, dirs, files in os.walk(directory): """ copy images to their folders """ for name in files: file_split = os.path.splitext(name) if file_split[1]==".jpeg" or file_split[1]==".jpg" or file_split[1]==".gif" or file_split[1]==".png": try: os.mkdir(os.path.join('_site','documentation',os.path.basename(root))) except: pass shutil.copyfile(os.path.join(root,name), os.path.join('output','documentation',os.path.basename(root),name)) """ create module introductions """ for module in dirs: if module!="addons": module_intro = os.path.join(root,module,"introduction.markdown") if os.path.isfile(module_intro): module_intro_file = open(module_intro) module_intro_content = module_intro_file.read() module_subtitles[module] = module_intro_content.splitlines()[0].strip('##').strip(' ') module_intro_content = markdown(module_intro_content, md_extensions) for lang in self.kw['translations']: context = {} context["lang"] = lang context["title"] = module context["module"] = module context["intro_content"] = module_intro_content context["permalink"] = self.kw['translations'][lang] + '/documentation/' + module + "/" if lang == self.site.config['DEFAULT_LANG']: short_tdst = os.path.join('documentation', module, "index.html") else: short_tdst = os.path.join(self.kw['translations'][lang], 'documentation', module, "index.html") tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst)) if module.find("ofx") == 0: context["classes"] = addons_index[module] self.site.render_template(module_template, tdst, context) else: context["classes"] = core_index[module] self.site.render_template(module_template, tdst, context) content_js[lang] += module_to_js(module, module_intro_content, self.site, lang) else: module_subtitles[module] = None # close js for docs search and save per language for lang in self.kw['translations']: content_js[lang] += ']};' content_js_file = open("output" + lang_prefix(lang, self.site) + "/tipuesearch_content.js","w") content_js_file.write(content_js[lang]) content_js_file.close() # render index file template_name = "documentation.mako" for lang in self.kw['translations']: #lang_suffix = self.kw['translations'][lang] docs_intro_path = os.path.join(docs_dir, "index.md") if lang != self.site.config['DEFAULT_LANG']: docs_intro_lang_path = utils.get_translation_candidate(self.site.config, docs_intro_path, lang) p = pathlib.Path(docs_intro_lang_path) if p.exists(): docs_intro_path = docs_intro_lang_path docs_intro = open(docs_intro_path).read() for key in self.site.GLOBAL_CONTEXT.keys(): if isinstance(self.site.GLOBAL_CONTEXT[key], str): docs_intro = docs_intro.replace('${' + key + '}', self.site.GLOBAL_CONTEXT[key]) docs_intro = markdown(docs_intro, md_extensions) context = {} context["lang"] = lang context["title"] = "documentation" context["docs_intro"] = docs_intro context['core'] = core_index context['addons'] = addons_index context['module_subtitles'] = module_subtitles context["permalink"] = self.kw['translations'][lang] + '/documentation/' short_tdst = os.path.join(self.kw['translations'][lang], "documentation", "index.html") tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst)) self.site.render_template(template_name, tdst, context)
def test_get_translation_candidate(): config = { 'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}', 'DEFAULT_LANG': 'en', 'TRANSLATIONS': { 'es': '1', 'en': 1 } } assert get_translation_candidate(config, '*.rst', 'es') == '*.es.rst' assert get_translation_candidate(config, 'fancy.post.rst', 'es') == 'fancy.post.es.rst' assert get_translation_candidate(config, '*.es.rst', 'es') == '*.es.rst' assert get_translation_candidate(config, '*.es.rst', 'en') == '*.rst' assert get_translation_candidate(config, 'cache/posts/fancy.post.es.html', 'en') == 'cache/posts/fancy.post.html' assert get_translation_candidate(config, 'cache/posts/fancy.post.html', 'es') == 'cache/posts/fancy.post.es.html' assert get_translation_candidate(config, 'cache/pages/charts.html', 'es') == 'cache/pages/charts.es.html' assert get_translation_candidate(config, 'cache/pages/charts.html', 'en') == 'cache/pages/charts.html' config = { 'TRANSLATIONS_PATTERN': '{path}.{ext}.{lang}', 'DEFAULT_LANG': 'en', 'TRANSLATIONS': { 'es': '1', 'en': 1 } } assert get_translation_candidate(config, '*.rst', 'es') == '*.rst.es' assert get_translation_candidate(config, '*.rst.es', 'es') == '*.rst.es' assert get_translation_candidate(config, '*.rst.es', 'en') == '*.rst' assert get_translation_candidate(config, 'cache/posts/fancy.post.html.es', 'en') == 'cache/posts/fancy.post.html' assert get_translation_candidate(config, 'cache/posts/fancy.post.html', 'es') == 'cache/posts/fancy.post.html.es'
def gen_tasks(self): self.kw = { 'strip_indexes': self.site.config['STRIP_INDEXES'], 'output_folder': self.site.config['OUTPUT_FOLDER'], 'cache_folder': self.site.config['CACHE_FOLDER'], 'default_lang': self.site.config['DEFAULT_LANG'], 'filters': self.site.config['FILTERS'], 'translations': self.site.config['TRANSLATIONS'], 'global_context': self.site.GLOBAL_CONTEXT, 'tzinfo': self.site.tzinfo, } tasks = {} classes = [] directory = os.path.join(self.site.original_cwd, "tutorials") template_name = "tutorials.mako" categories = [] dirs = os.listdir(directory) dirs.sort() files = [] for catfolder in dirs: if not os.path.isdir(os.path.join(directory,catfolder)): continue articles = [] category = catfolder[catfolder.find("_")+1:] articlesfiles = os.listdir(os.path.join(directory,catfolder)); articlesfiles.sort() for article in articlesfiles: file_split = os.path.splitext(article) folder = os.path.join(directory,catfolder,article) if file_split[1]=='.markdown': path = os.path.join(directory,catfolder,article) files += [path] articleobj = MarkdownArticle(path, directory) articles.append(articleobj) elif file_split[1]=='.asciidoc': path = os.path.join(directory,catfolder,article) files += [path] articleobj = AsciidocArticle(path, directory) articles.append(articleobj) elif os.path.isdir(folder): out_folder = os.path.join(self.site.original_cwd, 'output','tutorials',catfolder,article.lower()) for root, dirs, file_ins in os.walk(folder): for f in file_ins: in_path = os.path.join(root,f) out_path = os.path.join(out_folder, f) yield utils.apply_filters({ 'basename': self.name, 'name': in_path, 'file_dep': [in_path], 'targets': [out_path], 'actions': [ (create_file, (in_path, out_path)) ], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters']) categories.append({'category': category, 'articles': articles}); for lang in self.kw['translations']: tutorials_intro_path = os.path.join(directory, "index.md") if lang != self.site.config['DEFAULT_LANG']: tutorials_intro_lang_path = utils.get_translation_candidate(self.site.config, tutorials_intro_path, lang) p = pathlib.Path(tutorials_intro_lang_path) if p.exists(): tutorials_intro_path = tutorials_intro_lang_path tutorials_intro = open(tutorials_intro_path).read() context = {} context["lang"] = lang if lang == self.site.config['DEFAULT_LANG']: context["lang_folder"] = "" else: context["lang_folder"] = "/" + lang context["tutorials_intro"] = tutorials_intro context["title"] = "tutorials" context['categories'] = categories context["permalink"] = '/tutorials/' short_tdst = os.path.join(self.kw['translations'][lang], "tutorials", "index.html") tdst = os.path.normpath(os.path.join(self.kw['output_folder'], short_tdst)) template_dep = self.site.template_system.template_deps(template_name) template_dep += files yield utils.apply_filters({ 'basename': self.name, 'name': tdst, 'file_dep': template_dep, 'targets': [tdst], 'actions': [ (self.site.render_template, (template_name, tdst, context)) ], 'clean': True, 'uptodate': [utils.config_changed({ 1: self.kw, })], }, self.kw['filters'])