def on_files(self, files: Files, config: Config):
json_dir = "api"
(pathlib.Path(config['docs_dir']).resolve() / json_dir).mkdir(
parents=True, exist_ok=True)
(pathlib.Path(config['site_dir']).resolve() / json_dir).mkdir(
parents=True, exist_ok=True)
all_json_file = File(f"{json_dir}/codes.json", config["docs_dir"],
config["site_dir"], config["use_directory_urls"])
with open(all_json_file.abs_src_path, "w") as file:
json.dump(self.codes, file, indent=2)
with open(all_json_file.abs_dest_path, "w") as file:
json.dump(self.codes, file, indent=2)
for category, codes in self.codes_by_category.items():
dir = parameterize(f"{category}-responses")
for name, emojicode in codes.items():
md_file = File(f"{dir}/{name}.md", config["docs_dir"],
config["site_dir"],
config["use_directory_urls"])
files.append(md_file)
json_file = File(f"{json_dir}/{name}.json", config["docs_dir"],
config["site_dir"],
config["use_directory_urls"])
with open(json_file.abs_src_path, "w") as file:
json.dump(emojicode, file, indent=2)
files.append(json_file)
def on_files(self, files: Files, config: Config):
"""
From `https://www.mkdocs.org/user-guide/plugins/#on_files`:
The files event is called after the files collection is populated from
the docs_dir. Use this event to add, remove, or alter files in the
collection. Note that Page objects have not yet been associated with
the file objects in the collection.
Use Page Events to manipulate page specific data.
Parameters:
files: global files collection
config: global configuration object
Returns:
global files collection
"""
_posts_by_date = dict()
self.config.update(
dict(docs_dir=config['docs_dir'], site_dir=config['site_dir']))
for self._blog_md_file in self.get_blog_md():
_posts_by_date.update(self.build_blog_dest())
for post in sorted(_posts_by_date):
post = _posts_by_date[post]
blogpost = File(path=post['path'],
src_dir=post['src_dir'],
dest_dir=post['dest_dir'],
use_directory_urls=config['use_directory_urls'])
blogpost.abs_src_path = post['src_dir']
files.append(blogpost)
return files
def test_build_page_dirty_not_modified(self, site_dir, mock_write_file):
cfg = load_config(site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('testing.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'page content'
page.content = '<p>page content</p>'
build._build_page(page, cfg, files, nav, cfg['theme'].get_env(), dirty=True)
mock_write_file.assert_called_once()
def test_build_page(self, site_dir):
cfg = load_config(site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'page content'
page.content = '<p>page content</p>'
build._build_page(page, cfg, files, nav, self._get_env_with_null_translations(cfg))
self.assertPathIsFile(site_dir, 'index.html')
def test_build_page_dirty_modified(self, site_dir, docs_dir, mock_write_file):
cfg = load_config(docs_dir=docs_dir, site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'new page content'
page.content = '<p>new page content</p>'
build._build_page(page, cfg, files, nav, cfg['theme'].get_env(), dirty=True)
mock_write_file.assert_not_called()
def test_build_page(self, site_dir):
cfg = load_config(site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'page content'
page.content = '<p>page content</p>'
build._build_page(page, cfg, files, nav, cfg['theme'].get_env())
self.assertPathIsFile(site_dir, 'index.html')
def test_build_page_dirty_not_modified(self, site_dir, mock_write_file):
cfg = load_config(site_dir=site_dir, nav=['testing.md'], plugins=[])
files = Files([File('testing.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'page content'
page.content = '<p>page content</p>'
build._build_page(page, cfg, files, nav, self._get_env_with_null_translations(cfg), dirty=True)
self.assert_mock_called_once(mock_write_file)
def test_build_page_custom_template(self, site_dir):
cfg = load_config(site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.meta = {'template': '404.html'}
page.markdown = 'page content'
page.content = '<p>page content</p>'
build._build_page(page, cfg, files, nav, cfg['theme'].get_env())
self.assertPathIsFile(site_dir, 'index.html')
def remove_files(files: Files) -> Files:
to_remove = []
for file in files:
if file.src_path in {'plugins.py', 'requirements.txt', 'cli_help.txt'}:
to_remove.append(file)
elif file.src_path.startswith('__pycache__/'):
to_remove.append(file)
logger.debug('removing files: %s', [f.src_path for f in to_remove])
for f in to_remove:
files.remove(f)
return files
def on_files(self, files: Files, config: Config) -> Files: # noqa: WPS210
"""
Files event handler.
It's called when MkDocs discovers all files.
Here we filter all default pages and pages for translation.
:param files: discovered files.
:param config: mkdocs global config.
:return: files for default language.
"""
default_language = self.config.get("default_language")
all_languages = set([default_language] + list(self.config["languages"]))
# Idk we we need to process main_files separate from
# translations, but it doesn't work without it.
main_files = Files([])
# Aux files, such as js or css.
for aux_file in files:
if aux_file not in files.documentation_pages():
main_files.append(aux_file)
for language in all_languages:
self.i18pages[language].append(aux_file)
for page in files.documentation_pages():
page_lang = self._get_lang(page)
if page_lang == default_language:
main_files.append(page)
self.i18pages[page_lang].append(
self.translate_page(page, Path(config.get("site_dir"))),
)
self._sort_translated_files()
return main_files
def on_files(self, files, config):
templates_dir = os.path.join(here, "templates")
css_dest_dir = os.path.join(config["site_dir"], "css")
ansi_colours_css = File(
path="ansi-colours.css",
src_dir=templates_dir,
dest_dir=css_dest_dir,
use_directory_urls=False,
)
pandas_dataframe_css = File(
path="pandas-dataframe.css",
src_dir=templates_dir,
dest_dir=css_dest_dir,
use_directory_urls=False,
)
jupyter_cells_css = File(
path="jupyter-cells.css",
src_dir=templates_dir,
dest_dir=css_dest_dir,
use_directory_urls=False,
)
files = Files([
NotebookFile(f, **config) if str(f.abs_src_path).
endswith("ipynb") else f for f in files
] + [ansi_colours_css, pandas_dataframe_css, jupyter_cells_css])
return files
def test_nav_from_nested_files(self):
expected = dedent("""
Page(title=[blank], url='/')
Section(title='About')
Page(title=[blank], url='/about/license/')
Page(title=[blank], url='/about/release-notes/')
Section(title='Api guide')
Page(title=[blank], url='/api-guide/debugging/')
Page(title=[blank], url='/api-guide/running/')
Page(title=[blank], url='/api-guide/testing/')
Section(title='Advanced')
Page(title=[blank], url='/api-guide/advanced/part-1/')
""")
cfg = load_config(site_url='http://example.com/')
files = Files([
File('test.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
File('about/license.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
File('about/release-notes.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
File('api-guide/debugging.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
File('api-guide/running.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
File('api-guide/testing.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
File('api-guide/advanced/part-1.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']),
])
site_navigation = get_navigation(files, cfg)
self.assertEqual(str(site_navigation).strip(), expected)
self.assertEqual(len(site_navigation.items), 3)
self.assertEqual(len(site_navigation.pages), 7)
self.assertEqual(repr(site_navigation.homepage), "Page(title=[blank], url='/')")
def test_get_url_status__local_page(plugin):
index_page = Mock(spec=Page, markdown='# Heading\nContent')
page1_page = Mock(spec=Page,
markdown='# Page One\n## Sub Heading\nContent')
files = Files([
Mock(spec=File,
src_path='index.md',
dest_path='index.html',
url='index.html',
page=index_page),
Mock(spec=File,
src_path='page1.md',
dest_path='page1.html',
url='page1.html',
page=page1_page),
])
assert plugin.get_url_status('index.html#heading', 'page1.md', set(),
files, False) == 0
assert plugin.get_url_status('index.html#bad-heading', 'page1.md', set(),
files, False) == 404
assert plugin.get_url_status('page1.html#sub-heading', 'page1.md', set(),
files, False) == 0
assert plugin.get_url_status('page1.html#heading', 'page1.md', set(),
files, False) == 404
assert plugin.get_url_status('page2.html#heading', 'page1.md', set(),
files, False) == 404
def test_nav_external_links(self):
nav_cfg = [{
'Home': 'index.md'
}, {
'Local': '/local.html'
}, {
'External': 'http://example.com/external.html'
}]
expected = dedent("""
Page(title='Home', url='/')
Link(title='Local', url='/local.html')
Link(title='External', url='http://example.com/external.html')
""")
cfg = load_config(nav=nav_cfg, site_url='http://example.com/')
files = Files([
File('index.md', cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls'])
])
with self.assertLogs('mkdocs', level='DEBUG') as cm:
site_navigation = get_navigation(files, cfg)
self.assertEqual(cm.output, [
"DEBUG:mkdocs.structure.nav:An absolute path to '/local.html' is included in the "
"'nav' configuration, which presumably points to an external resource.",
"DEBUG:mkdocs.structure.nav:An external link to 'http://example.com/external.html' "
"is included in the 'nav' configuration."
])
self.assertEqual(str(site_navigation).strip(), expected)
self.assertEqual(len(site_navigation.items), 3)
self.assertEqual(len(site_navigation.pages), 1)
def moderate_files(self, files: Files, octiron: Octiron):
"""
Exclude some files when building site locally.
This helps reduce the site building time for the sake of local dev.
"""
logging.warning(
'A few files have been stripped from the site because local dev env'
' has been detected. Beware of 404 errors!', )
directories = [
row['version_directory'].toPython().replace('local:', '')
for row in octiron.query(
'''
SELECT * WHERE {
?version_directory a :Flake8PluginVersion .
FILTER NOT EXISTS {
?version_directory a :LatestVersion .
}
}
''', )
]
return Files([
mkdocs_file for mkdocs_file in files if not any(
mkdocs_file.src_path.startswith(directory)
for directory in directories)
])
def files(self) -> Files:
"""Access the files as they currently are, as a MkDocs [Files][] collection.
[Files]: https://github.com/mkdocs/mkdocs/blob/master/mkdocs/structure/files.py
"""
files = sorted(self._files.values(), key=file_sort_key)
return Files(files)
def get_files(config):
""" Walk the `source_dir` and return a Files collection. """
files = []
exclude = ['.*', '/templates']
for source_dir, dirnames, filenames in os.walk(config['source_dir'],
followlinks=True):
relative_dir = os.path.relpath(source_dir, config['source_dir'])
for dirname in list(dirnames):
path = os.path.normpath(os.path.join(relative_dir, dirname))
# Skip any excluded directories
if _filter_paths(basename=dirname,
path=path,
is_dir=True,
exclude=exclude):
dirnames.remove(dirname)
dirnames.sort()
for filename in _sort_files(filenames):
path = os.path.normpath(os.path.join(relative_dir, filename))
# Skip any excluded files
if _filter_paths(basename=filename,
path=path,
is_dir=False,
exclude=exclude):
continue
# Skip README.md is an index file also exists in dir
if filename.lower() == 'readme.md' and 'index.md' in filenames:
continue
files.append(
File(path, config['source_dir'], config['site_dir'],
config['use_directory_urls']))
return Files(files)
def test_nested_ungrouped_nav(self):
nav_cfg = [
{
'Home': 'index.md'
},
{
'Contact': 'about/contact.md'
},
{
'License Title': 'about/sub/license.md'
},
]
expected = dedent("""
Page(title='Home', url='/')
Page(title='Contact', url='/about/contact/')
Page(title='License Title', url='/about/sub/license/')
""")
cfg = load_config(nav=nav_cfg, site_url='http://example.com/')
files = Files([
File(
list(item.values())[0], cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls']) for item in nav_cfg
])
site_navigation = get_navigation(files, cfg)
self.assertEqual(str(site_navigation).strip(), expected)
self.assertEqual(len(site_navigation.items), 3)
self.assertEqual(len(site_navigation.pages), 3)
def test_find_index_of_dir(tmp_path_factory, use_directory_urls):
src_dir, dest_dir = map(tmp_path_factory.mktemp, ["src", "dest"])
file_names = [
" README.md",
"README.md ",
"index.md",
"bad/index.html",
"bad/foo.md",
"bad/foo.index.md",
"both1/index.md",
"both1/README.md",
"both2/readme.md",
"both2/index.md",
"a/a/a/a/a/README.md",
]
files = {f: File(f, src_dir, dest_dir, use_directory_urls) for f in file_names}
files_o = Files(list(files.values()))
assert plugin._find_index_of_dir({}, files_o, "") == files["index.md"]
assert plugin._find_index_of_dir({}, files_o, "nonexistent") is None
assert plugin._find_index_of_dir({}, files_o, "bad") is None
assert plugin._find_index_of_dir({}, files_o, "bad/foo") is None
assert plugin._find_index_of_dir({}, files_o, "bad/foo.md") is None
assert plugin._find_index_of_dir({}, files_o, "both1") == files["both1/README.md"]
assert plugin._find_index_of_dir({}, files_o, "both2") == files["both2/index.md"]
assert plugin._find_index_of_dir({"nav_file": "foo.md"}, files_o, "bad") == files["bad/foo.md"]
assert plugin._find_index_of_dir({"nav_file": "foo.md"}, files_o, "both2") is None
assert plugin._find_index_of_dir({"nav_file": "a"}, files_o, "a/a/a/a") is None
assert plugin._find_index_of_dir({}, files_o, "a/a/a/a/a") == files["a/a/a/a/a/README.md"]
assert plugin._find_index_of_dir({}, files_o, "a/a/a/a/a/a") is None
def test_build_page_error(self, site_dir, mock_write_file):
cfg = load_config(site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'page content'
page.content = '<p>page content</p>'
with self.assertLogs('mkdocs', level='ERROR') as cm:
self.assertRaises(IOError, build._build_page, page, cfg, files, nav, cfg['theme'].get_env())
self.assertEqual(
cm.output,
["ERROR:mkdocs.commands.build:Error building page 'index.md': Error message."]
)
mock_write_file.assert_called_once()
def test_nav_bad_links(self):
nav_cfg = [{
'Home': 'index.md'
}, {
'Missing': 'missing.html'
}, {
'Bad External': 'example.com'
}]
expected = dedent("""
Page(title='Home', url='/')
Link(title='Missing', url='missing.html')
Link(title='Bad External', url='example.com')
""")
cfg = load_config(nav=nav_cfg, site_url='http://example.com/')
files = Files([
File('index.md', cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls'])
])
with self.assertLogs('mkdocs', level='WARNING') as cm:
site_navigation = get_navigation(files, cfg)
self.assertEqual(cm.output, [
"WARNING:mkdocs.structure.nav:A relative path to 'missing.html' is included "
"in the 'nav' configuration, which is not found in the documentation files",
"WARNING:mkdocs.structure.nav:A relative path to 'example.com' is included "
"in the 'nav' configuration, which is not found in the documentation files"
])
self.assertEqual(str(site_navigation).strip(), expected)
self.assertEqual(len(site_navigation.items), 3)
self.assertEqual(len(site_navigation.pages), 1)
def test_build_page_plugin_error(self, site_dir, mock_write_file):
cfg = load_config(site_dir=site_dir, nav=['index.md'], plugins=[])
files = Files([File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])])
nav = get_navigation(files, cfg)
page = files.documentation_pages()[0].page
# Fake populate page
page.title = 'Title'
page.markdown = 'page content'
page.content = '<p>page content</p>'
with self.assertLogs('mkdocs', level='ERROR') as cm:
self.assertRaises(PluginError, build._build_page, page, cfg, files, nav, cfg['theme'].get_env())
self.assertEqual(
cm.output,
["ERROR:mkdocs.commands.build:Error building page 'index.md':"]
)
self.assert_mock_called_once(mock_write_file)
def on_files(self, files, config):
ret = Files([
NotebookFile(file, **config)
if str(file.abs_src_path).endswith("ipynb") else file
for file in files
])
return ret
def test_populate_page_dirty_modified(self, site_dir):
cfg = load_config(site_dir=site_dir)
file = File('testing.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])
page = Page('Foo', file, cfg)
build._populate_page(page, cfg, Files([file]), dirty=True)
self.assertTrue(page.markdown.startswith('# Welcome to MkDocs'))
self.assertTrue(page.content.startswith('<h1 id="welcome-to-mkdocs">Welcome to MkDocs</h1>'))
def test_populate_page(self, docs_dir):
cfg = load_config(docs_dir=docs_dir)
file = File('test.md', cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls'])
page = Page('Foo', file, cfg)
build._populate_page(page, cfg, Files([file]))
self.assertEqual(page.content, '<p>page content</p>')
def test_build_extra_template(self):
cfg = load_config()
files = Files([
File('foo.html', cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls']),
])
build._build_extra_template('foo.html', files, cfg, mock.Mock())
def on_files(self, files: Files, *args, **kwargs):
# Make sure paywall pages have the same url as the original page
doc_files = list(files.documentation_pages())
for f in doc_files:
if self._is_paywall_file(f):
url = Path(f.url)
f.url = str(url.relative_to(self._paywall_dir))
return files
def test_populate_page_dirty_not_modified(self, site_dir, docs_dir):
cfg = load_config(docs_dir=docs_dir, site_dir=site_dir)
file = File('index.md', cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls'])
page = Page('Foo', file, cfg)
build._populate_page(page, cfg, Files([file]), dirty=True)
# Content is empty as file read was skipped
self.assertEqual(page.markdown, None)
self.assertEqual(page.content, None)
def build_page(title, path, config, md_src=''):
""" Helper which returns a Page object. """
files = Files([File(path, config['docs_dir'], config['site_dir'], config['use_directory_urls'])])
page = Page(title, list(files)[0], config)
# Fake page.read_source()
page.markdown, page.meta = meta.get_data(md_src)
return page, files
def get_rendered_result(self, files):
cfg = load_config(docs_dir=self.DOCS_DIR)
fs = []
for f in files:
fs.append(File(f.replace('/', os.sep), cfg['docs_dir'], cfg['site_dir'], cfg['use_directory_urls']))
pg = Page('Foo', fs[0], cfg)
pg.read_source(cfg)
pg.render(cfg, Files(fs))
return pg.content
def test_get_by_type_nested_sections(self):
nav_cfg = [{'Section 1': [{'Section 2': [{'Page': 'page.md'}]}]}]
cfg = load_config(nav=nav_cfg, site_url='http://example.com/')
files = Files([
File('page.md', cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls'])
])
site_navigation = get_navigation(files, cfg)
self.assertEqual(len(_get_by_type(site_navigation, Section)), 2)
def on_files(self, files, config):
extensions = [".ipynb", ".py"]
ret = Files([
NotebookFile(file, **config) if os.path.splitext(
str(file.abs_src_path))[-1] in extensions else file
for file in files
])
return ret
def test_context_base_url_homepage_use_directory_urls(self):
nav_cfg = [{'Home': 'test.md'}]
cfg = load_config(nav=nav_cfg)
files = Files([
File('test.md', cfg['docs_dir'], cfg['site_dir'],
cfg['use_directory_urls']),
])
nav = get_navigation(files, cfg)
context = build.get_context(nav, files, cfg, nav.pages[0])
self.assertEqual(context['base_url'], '.')
def test_files(self):
fs = [
File('index.md', '/path/to/docs', '/path/to/site', use_directory_urls=True),
File('foo/bar.md', '/path/to/docs', '/path/to/site', use_directory_urls=True),
File('foo/bar.html', '/path/to/docs', '/path/to/site', use_directory_urls=True),
File('foo/bar.jpg', '/path/to/docs', '/path/to/site', use_directory_urls=True),
File('foo/bar.js', '/path/to/docs', '/path/to/site', use_directory_urls=True),
File('foo/bar.css', '/path/to/docs', '/path/to/site', use_directory_urls=True)
]
files = Files(fs)
self.assertEqual([f for f in files], fs)
self.assertEqual(len(files), 6)
self.assertEqual(files.documentation_pages(), [fs[0], fs[1]])
self.assertEqual(files.static_pages(), [fs[2]])
self.assertEqual(files.media_files(), [fs[3], fs[4], fs[5]])
self.assertEqual(files.javascript_files(), [fs[4]])
self.assertEqual(files.css_files(), [fs[5]])
self.assertEqual(files.get_file_from_path('foo/bar.jpg'), fs[3])
self.assertEqual(files.get_file_from_path('foo/bar.jpg'), fs[3])
self.assertEqual(files.get_file_from_path('missing.jpg'), None)
self.assertTrue(fs[2].src_path in files)
self.assertTrue(fs[2].src_path in files)
extra_file = File('extra.md', '/path/to/docs', '/path/to/site', use_directory_urls=True)
self.assertFalse(extra_file.src_path in files)
files.append(extra_file)
self.assertEqual(len(files), 7)
self.assertTrue(extra_file.src_path in files)
self.assertEqual(files.documentation_pages(), [fs[0], fs[1], extra_file])
