def _build_template(name, template, files, config, nav):
"""
Return rendered output for given template as a string.
"""
# Run `pre_template` plugin events.
template = config['plugins'].run_event(
'pre_template', template, template_name=name, config=config
)
if utils.is_error_template(name):
# Force absolute URLs in the nav of error pages and account for the
# possability that the docs root might be different than the server root.
# See https://github.com/mkdocs/mkdocs/issues/77.
# However, if site_url is not set, assume the docs root and server root
# are the same. See https://github.com/mkdocs/mkdocs/issues/1598.
base_url = utils.urlparse(config['site_url'] or '/').path
else:
base_url = utils.get_relative_url('.', name)
context = get_context(nav, files, config, base_url=base_url)
# Run `template_context` plugin events.
context = config['plugins'].run_event(
'template_context', context, template_name=name, config=config
)
output = template.render(context)
# Run `post_template` plugin events.
output = config['plugins'].run_event(
'post_template', output, template_name=name, config=config
)
return output
def post_validation(self, config, key_name):
repo_host = utils.urlparse(config['repo_url']).netloc.lower()
edit_uri = config.get('edit_uri')
# derive repo_name from repo_url if unset
if config['repo_url'] is not None and config.get('repo_name') is None:
if repo_host == 'github.com':
config['repo_name'] = 'GitHub'
elif repo_host == 'bitbucket.org':
config['repo_name'] = 'Bitbucket'
elif repo_host == 'gitlab.com':
config['repo_name'] = 'GitLab'
else:
config['repo_name'] = repo_host.split('.')[0].title()
# derive edit_uri from repo_name if unset
if config['repo_url'] is not None and edit_uri is None:
if repo_host == 'github.com' or repo_host == 'gitlab.com':
edit_uri = 'edit/master/docs/'
elif repo_host == 'bitbucket.org':
edit_uri = 'src/default/docs/'
else:
edit_uri = ''
# ensure a well-formed edit_uri
if edit_uri:
if not edit_uri.startswith(('?', '#')) \
and not config['repo_url'].endswith('/'):
config['repo_url'] += '/'
if not edit_uri.endswith('/'):
edit_uri += '/'
config['edit_uri'] = edit_uri
def path_to_url(self, url):
scheme, netloc, path, params, query, fragment = urlparse(url)
if scheme or netloc or not path or AMP_SUBSTITUTE in url or '.' not in os.path.split(path)[-1]:
# Ignore URLs unless they are a relative link to a source file.
# AMP_SUBSTITUTE is used internally by Markdown only for email.
# No '.' in the last part of a path indicates path does not point to a file.
return url
# Determine the filepath of the target.
target_path = os.path.join(os.path.dirname(self.file.src_path), path)
target_path = os.path.normpath(target_path).lstrip(os.sep)
if not self.is_source_code_link(target_path):
return url
source_url = convert_path_to_source_url(
target_path,
repos_prefix=self.config['repos_prefix'],
repos_info=self.config['repos_info'],
default_url_template=self.config['default_url_template'],
github_url_template=self.config['github_url_template'],
)
return source_url or url
def post_validation(self, config, key_name):
repo_host = utils.urlparse(config['repo_url']).netloc.lower()
edit_uri = config.get('edit_uri')
# derive repo_name from repo_url if unset
if config['repo_url'] is not None and config.get('repo_name') is None:
if repo_host == 'github.com':
config['repo_name'] = 'GitHub'
elif repo_host == 'bitbucket.org':
config['repo_name'] = 'Bitbucket'
else:
config['repo_name'] = repo_host.split('.')[0].title()
# derive edit_uri from repo_name if unset
if config['repo_url'] is not None and edit_uri is None:
if repo_host == 'github.com':
edit_uri = 'edit/master/docs/'
elif repo_host == 'bitbucket.org':
edit_uri = 'src/default/docs/'
else:
edit_uri = ''
# ensure a well-formed edit_uri
if edit_uri:
if not edit_uri.startswith(('?', '#')) \
and not config['repo_url'].endswith('/'):
config['repo_url'] += '/'
if not edit_uri.endswith('/'):
edit_uri += '/'
config['edit_uri'] = edit_uri
def path_to_url(self, url):
scheme, netloc, path, params, query, fragment = urlparse(url)
if scheme or netloc or not path or AMP_SUBSTITUTE in url or '.' not in os.path.split(
path)[-1]:
# Ignore URLs unless they are a relative link to a source file.
# AMP_SUBSTITUTE is used internally by Markdown only for email.
# No '.' in the last part of a path indicates path does not point to a file.
return url
# Determine the filepath of the target.
target_path = os.path.join(os.path.dirname(self.file.src_path), path)
target_path = os.path.normpath(target_path).lstrip(os.sep)
# Validate that the target exists in files collection.
if target_path not in self.files:
msg = (
"Documentation file '{}' contains a link to '{}' which does not exist "
"in the documentation directory.".format(
self.file.src_path, target_path))
# In strict mode raise an error at this point.
if self.strict:
raise MarkdownNotFound(msg)
# Otherwise, when strict mode isn't enabled, log a warning
# to the user and leave the URL as it is.
log.warning(msg)
return url
target_file = self.files.get_file_from_path(target_path)
path = target_file.url_relative_to(self.file)
components = (scheme, netloc, path, params, query, fragment)
return urlunparse(components)
def _build_template(name, template, files, config, nav):
"""
Return rendered output for given template as a string.
"""
# Run `pre_template` plugin events.
template = config['plugins'].run_event(
'pre_template', template, template_name=name, config=config
)
if utils.is_error_template(name):
# Force absolute URLs in the nav of error pages and account for the
# possability that the docs root might be different than the server root.
# See https://github.com/mkdocs/mkdocs/issues/77.
# However, if site_url is not set, assume the docs root and server root
# are the same. See https://github.com/mkdocs/mkdocs/issues/1598.
base_url = utils.urlparse(config['site_url'] or '/').path
else:
base_url = utils.get_relative_url('.', name)
context = get_context(nav, files, config, base_url=base_url)
# Run `template_context` plugin events.
context = config['plugins'].run_event(
'template_context', context, template_name=name, config=config
)
output = template.render(context)
# Run `post_template` plugin events.
output = config['plugins'].run_event(
'post_template', output, template_name=name, config=config
)
return output
def path_to_url(self, url):
scheme, netloc, path, params, query, fragment = urlparse(url)
if (scheme or netloc or not path or url.startswith('/')
or AMP_SUBSTITUTE in url or '.' not in os.path.split(path)[-1]):
# Ignore URLs unless they are a relative link to a source file.
# AMP_SUBSTITUTE is used internally by Markdown only for email.
# No '.' in the last part of a path indicates path does not point to a file.
return url
# Determine the filepath of the target.
target_path = os.path.join(os.path.dirname(self.file.src_path), urlunquote(path))
target_path = os.path.normpath(target_path).lstrip(os.sep)
# Validate that the target exists in files collection.
if target_path not in self.files:
log.warning(
"Documentation file '{}' contains a link to '{}' which is not found "
"in the documentation files.".format(self.file.src_path, target_path)
)
return url
target_file = self.files.get_file_from_path(target_path)
path = target_file.url_relative_to(self.file)
components = (scheme, netloc, path, params, query, fragment)
return urlunparse(components)
def path_to_url(self, url):
scheme, netloc, path, params, query, fragment = urlparse(url)
if (scheme or netloc or not path or url.startswith('/')
or AMP_SUBSTITUTE in url or '.' not in os.path.split(path)[-1]):
# Ignore URLs unless they are a relative link to a source file.
# AMP_SUBSTITUTE is used internally by Markdown only for email.
# No '.' in the last part of a path indicates path does not point to a file.
return url
# Determine the filepath of the target.
target_path = os.path.join(os.path.dirname(self.file.src_path), urlunquote(path))
target_path = os.path.normpath(target_path).lstrip(os.sep)
# Validate that the target exists in files collection.
if target_path not in self.files:
log.warning(
"Documentation file '{}' contains a link to '{}' which is not found "
"in the documentation files.".format(self.file.src_path, target_path)
)
return url
target_file = self.files.get_file_from_path(target_path)
path = target_file.url_relative_to(self.file)
components = (scheme, netloc, path, params, query, fragment)
return urlunparse(components)
def _set_canonical_url(self, base):
if base:
if not base.endswith('/'):
base += '/'
self.canonical_url = urljoin(base, self.url)
self.abs_url = urlparse(self.canonical_url).path
else:
self.canonical_url = None
self.abs_url = None
def _set_canonical_url(self, base):
if base:
if not base.endswith('/'):
base += '/'
self.canonical_url = urljoin(base, self.url)
self.abs_url = urlparse(self.canonical_url).path
else:
self.canonical_url = None
self.abs_url = None
def post_validation(self, config, key_name):
if config['repo_url'] is not None and config.get('repo_name') is None:
repo_host = utils.urlparse(config['repo_url']).netloc.lower()
if repo_host == 'github.com':
config['repo_name'] = 'GitHub'
elif repo_host == 'bitbucket.org':
config['repo_name'] = 'Bitbucket'
else:
config['repo_name'] = repo_host.split('.')[0].title()
def post_validation(self, config, key_name):
if config['repo_url'] is not None and config.get('repo_name') is None:
repo_host = utils.urlparse(
config['repo_url']).netloc.lower()
if repo_host == 'github.com':
config['repo_name'] = 'GitHub'
elif repo_host == 'bitbucket.org':
config['repo_name'] = 'Bitbucket'
else:
config['repo_name'] = repo_host.split('.')[0].title()
def run_validation(self, value):
try:
parsed_url = utils.urlparse(value)
except (AttributeError, TypeError):
raise ValidationError("Unable to parse the URL.")
if parsed_url.scheme:
return value
raise ValidationError(
"The URL isn't valid, it should include the http:// (scheme)")
def run_validation(self, value):
try:
parsed_url = utils.urlparse(value)
except (AttributeError, TypeError):
raise ValidationError("Unable to parse the URL.")
if parsed_url.scheme:
return value
raise ValidationError(
"The URL isn't valid, it should include the http:// (scheme)")
def get_navigation(files, config):
""" Build site navigation from config and files."""
nav_config = config['nav'] or nest_paths(
f.src_path for f in files.documentation_pages())
items = _data_to_navigation(nav_config, files, config)
if not isinstance(items, list):
items = [items]
# Get only the pages from the navigation, ignoring any sections and links.
pages = _get_by_type(items, Page)
# Include next, previous and parent links.
_add_previous_and_next_links(pages)
_add_parent_links(items)
missing_from_config = [
file for file in files.documentation_pages() if file.page is None
]
if missing_from_config:
log.info(
'The following pages exist in the docs directory, but are not '
'included in the "nav" configuration:\n - {}'.format(
'\n - '.join([file.src_path
for file in missing_from_config])))
# Any documentation files not found in the nav should still have an associated page.
# However, these page objects are only accessable from File instances as `file.page`.
for file in missing_from_config:
Page(None, file, config)
links = _get_by_type(items, Link)
for link in links:
scheme, netloc, path, params, query, fragment = urlparse(link.url)
if scheme or netloc:
log.debug("An external link to '{}' is included in "
"the 'nav' configuration.".format(link.url))
elif link.url.startswith('/'):
log.debug(
"An absolute path to '{}' is included in the 'nav' configuration, "
"which presumably points to an external resource.".format(
link.url))
else:
msg = (
"A relative path to '{}' is included in the 'nav' configuration, "
"which is not found in the documentation files".format(
link.url))
log.warning(msg)
return Navigation(items, pages)
def munge(url):
scheme, netloc, path, params, query, fragment = (utils.urlparse(url))
if scheme or netloc or not path:
# Ignore URLs unless they are a relative link to a markdown file.
return url
if not path.endswith(".md"):
return url
else:
path = path.replace(".md", ".html")
# Convert the .md hyperlink to a relative hyperlink to the HTML page.
fragments = (scheme, netloc, path, params, query, fragment)
url = utils.urlunparse(fragments)
return url
def get_navigation(files, config):
""" Build site navigation from config and files."""
nav_config = config['nav'] or nest_paths(f.src_path for f in files.documentation_pages())
items = _data_to_navigation(nav_config, files, config)
if not isinstance(items, list):
items = [items]
# Get only the pages from the navigation, ignoring any sections and links.
pages = _get_by_type(items, Page)
# Include next, previous and parent links.
_add_previous_and_next_links(pages)
_add_parent_links(items)
missing_from_config = [file for file in files.documentation_pages() if file.page is None]
if missing_from_config:
log.info(
'The following pages exist in the docs directory, but are not '
'included in the "nav" configuration:\n - {}'.format(
'\n - '.join([file.src_path for file in missing_from_config]))
)
# Any documentation files not found in the nav should still have an associated page.
# However, these page objects are only accessable from File instances as `file.page`.
for file in missing_from_config:
Page(None, file, config)
links = _get_by_type(items, Link)
for link in links:
scheme, netloc, path, params, query, fragment = urlparse(link.url)
if scheme or netloc:
log.debug(
"An external link to '{}' is included in "
"the 'nav' configuration.".format(link.url)
)
elif link.url.startswith('/'):
log.debug(
"An absolute path to '{}' is included in the 'nav' configuration, "
"which presumably points to an external resource.".format(link.url)
)
else:
msg = (
"A relative path to '{}' is included in the 'nav' configuration, "
"which is not found in the documentation files".format(link.url)
)
log.warning(msg)
return Navigation(items, pages)
def munge(url):
scheme, netloc, path, params, query, fragment = (
utils.urlparse(url))
if scheme or netloc or not path:
# Ignore URLs unless they are a relative link to a markdown file.
return url
if not path.endswith(".md"):
return url
else:
path = path.replace(".md",".html");
# Convert the .md hyperlink to a relative hyperlink to the HTML page.
fragments = (scheme, netloc, path, params, query, fragment)
url = utils.urlunparse(fragments)
return url
def build_error_template(template, env, config, site_navigation):
"""
Build error template.
Force absolute URLs in the nav of error pages and account for the
possability that the docs root might be different than the server root.
See https://github.com/mkdocs/mkdocs/issues/77
"""
site_navigation.url_context.force_abs_urls = True
default_base = site_navigation.url_context.base_path
site_navigation.url_context.base_path = utils.urlparse(config['site_url']).path
build_template(template, env, config, site_navigation)
# Reset nav behavior to the default
site_navigation.url_context.force_abs_urls = False
site_navigation.url_context.base_path = default_base
def path_to_url(url, nav, strict):
scheme, netloc, path, params, query, fragment = (utils.urlparse(url))
if scheme or netloc or not path or AMP_SUBSTITUTE in url:
# Ignore URLs unless they are a relative link to a markdown file.
# AMP_SUBSTITUTE is used internally by Markdown only for email,which is
# not a relative link. As urlparse errors on them, skip explicitly
return url
if nav and not utils.is_markdown_file(path):
path = utils.create_relative_media_url(nav, path)
elif nav:
# If the site navigation has been provided, then validate
# the internal hyperlink, making sure the target actually exists.
target_file = nav.file_context.make_absolute(path)
if target_file.startswith(os.path.sep):
target_file = target_file[1:]
if target_file not in nav.source_files:
source_file = nav.file_context.current_file
msg = ('The page "%s" contained a hyperlink to "%s" which '
'is not listed in the "pages" configuration.') % (
source_file, target_file)
# In strict mode raise an error at this point.
if strict:
raise MarkdownNotFound(msg)
# Otherwise, when strict mode isn't enabled, log a warning
# to the user and leave the URL as it is.
# suppress the Warning
# log.warning(msg)
# return url
path = utils.get_url_path(target_file, nav.use_directory_urls)
path = nav.url_context.make_relative(path)
else:
path = utils.get_url_path(path).lstrip('/')
# Convert the .md hyperlink to a relative hyperlink to the HTML page.
fragments = (scheme, netloc, path, params, query, fragment)
url = utils.urlunparse(fragments)
return url
def path_to_url(url, nav, strict):
scheme, netloc, path, params, query, fragment = (
utils.urlparse(url))
if scheme or netloc or not path or AMP_SUBSTITUTE in url:
# Ignore URLs unless they are a relative link to a markdown file.
# AMP_SUBSTITUTE is used internally by Markdown only for email,which is
# not a relative link. As urlparse errors on them, skip explicitly
return url
if nav and not utils.is_markdown_file(path):
path = utils.create_relative_media_url(nav, path)
elif nav:
# If the site navigation has been provided, then validate
# the internal hyperlink, making sure the target actually exists.
target_file = nav.file_context.make_absolute(path)
if target_file.startswith(os.path.sep):
target_file = target_file[1:]
if target_file not in nav.source_files:
source_file = nav.file_context.current_file
msg = (
'The page "%s" contained a hyperlink to "%s" which '
'is not listed in the "pages" configuration.'
) % (source_file, target_file)
# In strict mode raise an error at this point.
if strict:
raise MarkdownNotFound(msg)
# Otherwise, when strict mode isn't enabled, log a warning
# to the user and leave the URL as it is.
log.warning(msg)
return url
path = utils.get_url_path(target_file, nav.use_directory_urls)
path = nav.url_context.make_relative(path)
else:
path = utils.get_url_path(path).lstrip('/')
# Convert the .md hyperlink to a relative hyperlink to the HTML page.
fragments = (scheme, netloc, path, params, query, fragment)
url = utils.urlunparse(fragments)
return url
def build_error_template(template, env, config, site_navigation):
"""
Build error template.
Force absolute URLs in the nav of error pages and account for the
possability that the docs root might be different than the server root.
See https://github.com/mkdocs/mkdocs/issues/77
"""
site_navigation.url_context.force_abs_urls = True
default_base = site_navigation.url_context.base_path
site_navigation.url_context.base_path = utils.urlparse(
config['site_url']).path
build_template(template, env, config, site_navigation)
# Reset nav behavior to the default
site_navigation.url_context.force_abs_urls = False
site_navigation.url_context.base_path = default_base
def run(self, root):
"""
Update urls on anchors and images to make them relative
Iterates through the full document tree looking for specific
tags and then makes them relative based on the site navigation
"""
for element in root.iter():
if element.tag == 'a':
key = 'href'
elif element.tag == 'img':
key = 'src'
else:
continue
url = element.get(key)
scheme, netloc, path, params, query, fragment = urlparse(url) # can optimize reusing as components in path_to_url()
new_url = self.path_to_url(url) # components
element.set(key, new_url)
if (key == 'href' and (scheme or netloc or not path or url.startswith('/'))):
element.set('rel', 'external')
return root
def build_pages(config, dump_json=False, dirty=False):
"""
Builds all the pages and writes them into the build directory.
"""
site_navigation = nav.SiteNavigation(config['pages'], config['use_directory_urls'])
loader = jinja2.FileSystemLoader(config['theme_dir'] + [config['mkdocs_templates'], ])
env = jinja2.Environment(loader=loader)
# TODO: remove DeprecationContext in v1.0 when all deprecated vars have been removed
from jinja2.runtime import Context
deprecated_vars = {
'page_title': 'page.title',
'content': 'page.content',
'toc': 'page.toc',
'meta': 'page.meta',
'canonical_url': 'page.canonical_url',
'previous_page': 'page.previous_page',
'next_page': 'page.next_page',
'current_page': 'page',
'include_nav': 'nav|length>1',
'include_next_prev': '(page.next_page or page.previous_page)',
'site_name': 'config.site_name',
'site_author': 'config.site_author',
'page_description': 'config.site_description',
'repo_url': 'config.repo_url',
'repo_name': 'config.repo_name',
'site_url': 'config.site_url',
'copyright': 'config.copyright',
'google_analytics': 'config.google_analytics',
'homepage_url': 'nav.homepage.url',
'favicon': '{{ base_url }}/img/favicon.ico',
}
class DeprecationContext(Context):
def resolve(self, key):
""" Log a warning when accessing any deprecated variable name. """
if key in deprecated_vars:
log.warn(
"Template variable warning: '{0}' is being deprecated "
"and will not be available in a future version. Use "
"'{1}' instead.".format(key, deprecated_vars[key])
)
return super(DeprecationContext, self).resolve(key)
env.context_class = DeprecationContext
# TODO: end remove DeprecationContext
env.filters['tojson'] = filters.tojson
search_index = search.SearchIndex()
# Force absolute URLs in the nav of error pages and account for the
# possability that the docs root might be different than the server root.
# See https://github.com/mkdocs/mkdocs/issues/77
site_navigation.url_context.force_abs_urls = True
default_base = site_navigation.url_context.base_path
site_navigation.url_context.base_path = utils.urlparse(config['site_url']).path
build_template('404.html', env, config, site_navigation)
# Reset nav behavior to the default
site_navigation.url_context.force_abs_urls = False
site_navigation.url_context.base_path = default_base
if not build_template('search.html', env, config, site_navigation):
log.debug("Search is enabled but the theme doesn't contain a "
"search.html file. Assuming the theme implements search "
"within a modal.")
build_template('sitemap.xml', env, config, site_navigation)
build_extra_templates(config['extra_templates'], config, site_navigation)
for page in site_navigation.walk_pages():
try:
# When --dirty is used, only build the page if the markdown has been modified since the
# previous build of the output.
input_path, output_path = get_complete_paths(config, page)
if dirty and (utils.modified_time(input_path) < utils.modified_time(output_path)):
continue
log.debug("Building page %s", page.input_path)
build_result = _build_page(page, config, site_navigation, env,
dump_json)
html_content, table_of_contents, _ = build_result
search_index.add_entry_from_context(
page, html_content, table_of_contents)
except Exception:
log.error("Error building page %s", page.input_path)
raise
search_index = search_index.generate_search_index()
json_output_path = os.path.join(config['site_dir'], 'mkdocs', 'search_index.json')
utils.write_file(search_index.encode('utf-8'), json_output_path)
def build_pages(config, dump_json=False, dirty=False):
"""
Builds all the pages and writes them into the build directory.
"""
site_navigation = nav.SiteNavigation(config['pages'], config['use_directory_urls'])
loader = jinja2.FileSystemLoader(config['theme_dir'] + [config['mkdocs_templates'], ])
env = jinja2.Environment(loader=loader)
# TODO: remove DeprecationContext in v1.0 when all deprecated vars have been removed
from jinja2.runtime import Context
deprecated_vars = [
'page_title',
'content',
'toc',
'meta',
'current_page',
'canonical_url',
'previous_page',
'next_page'
]
class DeprecationContext(Context):
def resolve(self, key):
""" Log a warning when acessing any deprecated variable name. """
if key in deprecated_vars:
replacement = "page" if key == 'current_page' else "page.{0}".format(key)
log.warn(
"Template variable warning: '{0}' is being deprecated and will not be "
"available in a future version. Use '{1}' instead.".format(key, replacement)
)
return super(DeprecationContext, self).resolve(key)
env.context_class = DeprecationContext
# TODO: end remove DeprecationContext
env.filters['tojson'] = filters.tojson
search_index = search.SearchIndex()
# Force absolute URLs in the nav of error pages and account for the
# possability that the docs root might be different than the server root.
# See https://github.com/mkdocs/mkdocs/issues/77
site_navigation.url_context.force_abs_urls = True
default_base = site_navigation.url_context.base_path
site_navigation.url_context.base_path = utils.urlparse(config['site_url']).path
build_template('404.html', env, config, site_navigation)
# Reset nav behavior to the default
site_navigation.url_context.force_abs_urls = False
site_navigation.url_context.base_path = default_base
if not build_template('search.html', env, config, site_navigation):
log.debug("Search is enabled but the theme doesn't contain a "
"search.html file. Assuming the theme implements search "
"within a modal.")
build_template('sitemap.xml', env, config, site_navigation)
build_extra_templates(config['extra_templates'], config, site_navigation)
# append extra pages not including in [email protected]
append_extra_pages(config, env, dump_json, site_navigation)
for page in site_navigation.walk_pages():
try:
# When --dirty is used, only build the page if the markdown has been modified since the
# previous build of the output.
input_path, output_path = get_complete_paths(config, page)
if dirty and (utils.modified_time(input_path) < utils.modified_time(output_path)):
continue
log.debug("Building page %s", page.input_path)
build_result = _build_page(page, config, site_navigation, env,
dump_json)
html_content, table_of_contents, _ = build_result
search_index.add_entry_from_context(
page, html_content, table_of_contents)
except Exception:
log.error("Error building page %s", page.input_path)
raise
search_index = search_index.generate_search_index()
json_output_path = os.path.join(config['site_dir'], 'mkdocs', 'search_index.json')
utils.write_file(search_index.encode('utf-8'), json_output_path)