def __init__(self, path):
# type: (Path) -> None
self._sections = [] # type: List[Text]
self._content = ""
self._title = ""
self._subtitle = ""
self._toc_section = ""
self._path = path
self._path_finder = PathFinder(self._path.parent)
def create_external_configs(namespace):
# type: (argparse.Namespace) -> None
"""
Create `GitHub Pages` and `Read the Docs` configuration files.
"""
logger = get_logger()
configs = (
("gh_pages_config.yml", namespace.output_path / "_config.yml"),
("mkdocs.yml", namespace.input_path / "mkdocs.yml"),
("readthedocs.yml", namespace.input_path / ".readthedocs.yml"),
)
for asset_name, target_path in configs:
if target_path.exists():
continue
logger.info("Creating {} file".format(target_path))
render_asset(
asset_name,
target_path,
dict(
source_code_url=namespace.source_code_url.replace(
"blob/master/", ""),
project_name=make_title(namespace.input_path.name),
docs_path=PathFinder(namespace.input_path).relative(
namespace.output_path).as_posix(),
),
)
def test_init(self, PathMock):
path = PathMock()
self.assertIsInstance(PathFinder(path), PathFinder)
path.is_absolute.return_value = False
with self.assertRaises(PathFinderError):
self.assertIsInstance(PathFinder(path), PathFinder)
path.is_absolute.return_value = True
path.exists.return_value = True
path.is_dir.return_value = False
with self.assertRaises(PathFinderError):
self.assertIsInstance(PathFinder(path), PathFinder)
path.is_dir.return_value = True
self.assertIsInstance(PathFinder(path), PathFinder)
def __init__(
self,
input_path, # type: Path
output_path, # type: Path
source_paths, # type: Iterable[Path]
project_name=None, # type: Optional[Text]
docstring_processor=None, # type: Optional[BaseDocstringProcessor]
loader=None, # type: Optional[Loader]
raise_errors=False, # type: bool
source_code_url=None, # type: Optional[Text]
toc_depth=1, # type: int
):
# type: (...) -> None
self._logger = get_logger()
self._root_path = input_path
self._output_path = output_path
self._project_name = project_name or make_title(input_path.name)
self._root_path_finder = PathFinder(self._root_path)
self._source_code_url = source_code_url
self._toc_depth = toc_depth
self._raise_errors = raise_errors
# create output folder if it does not exist
if not self._output_path.exists():
self._logger.info("Creating folder {}".format(self._output_path))
PathFinder(self._output_path).mkdir()
self._loader = loader or Loader(
root_path=self._root_path, output_path=self._output_path
)
self._docstring_processor = docstring_processor or SmartDocstringProcessor()
self._source_paths = sorted(source_paths)
self._error_output_paths = set() # type: Set[Path]
self._logger.debug(
"Generating source map for {} source files".format(len(self._source_paths))
)
self._module_records = self._build_module_record_list()
self._logger.debug("Source map generated")
package_names = self._module_records.get_package_names()
package_names_re_expr = "|".join(package_names)
self._docstring_links_re = re.compile(
r"`+(?:{})\.\S+`+".format(package_names_re_expr)
)
self._prepare_index()
def test_relative(self):
path_finder = PathFinder(Path("/root/parent/"))
self.assertEqual(path_finder.relative(Path("/root/target.py")),
Path("../target.py"))
self.assertEqual(path_finder.relative(Path("/root/parent/target.py")),
Path("target.py"))
self.assertEqual(
path_finder.relative(Path("/root2/other/target.py")),
Path("../../root2/other/target.py"),
)
self.assertEqual(path_finder.relative(Path("/root/parent/source.py")),
Path("source.py"))
with self.assertRaises(PathFinderError):
path_finder.relative(Path("second/source.py"))
def main():
# type: () -> None
"""
Main entrypoint for CLI.
"""
args = parse_args(sys.argv[1:])
if args.version:
print(version)
return
log_level = logging.INFO
if args.debug:
log_level = logging.DEBUG
if args.quiet:
log_level = logging.CRITICAL
logger = get_logger(level=log_level)
path_finder = (
PathFinder(args.input_path).exclude(*(EXCLUDE_EXPRS + args.exclude)).include(*args.include)
)
try:
generator = Generator(
project_name=args.project_name,
input_path=args.input_path,
output_path=args.output_path,
source_paths=path_finder.glob(SOURCES_GLOB),
raise_errors=args.panic,
source_code_url=args.source_code_url,
toc_depth=args.toc_depth,
)
if args.files:
for path in args.files:
generator.generate_doc(path)
else:
generator.generate_docs()
generator.generate_index()
generator.generate_modules()
if args.cleanup:
generator.cleanup_old_docs()
if args.source_code_url:
create_external_configs(args)
except GeneratorError as e:
logger.error(e)
sys.exit(1)
def cleanup_old_docs(self):
# type: () -> None
"""
Remove old docs generated for this module.
"""
self._logger.debug("Removing orphaned docs")
preserve_paths = {
self._loader.get_output_path(i.source_path) for i in self._module_records
}
orphaned_dirs = []
preserve_paths.add(self.md_index.path)
preserve_paths.add(self.md_modules.path)
# skip error output paths
# preserve_paths.update(self._error_output_paths)
for doc_path in PathFinder(self._output_path).glob("**/*.md"):
if doc_path in preserve_paths:
continue
file_content = doc_path.read_text()
is_autogenerated = "> Auto-generated documentation" in file_content
if not is_autogenerated:
continue
self._logger.info(
"Deleting orphaned doc file {}".format(
self._root_path_finder.relative(doc_path)
)
)
doc_path.unlink()
# remove parent directory if it is empty
children = list(doc_path.parent.iterdir())
if not children:
orphaned_dirs.append(doc_path.parent)
for orphaned_dir in orphaned_dirs:
self._logger.info(
"Deleting orphaned directory {}".format(
self._root_path_finder.relative(orphaned_dir)
)
)
orphaned_dir.rmdir()
def test_glob(self, PathMock):
path = PathMock()
include_file_mock = MagicMock()
include_file_mock.relative_to(
).as_posix.return_value = "include/file.py"
exclude_file_mock = MagicMock()
exclude_file_mock.relative_to(
).as_posix.return_value = "exclude/file.py"
path.glob.return_value = [include_file_mock, exclude_file_mock]
path_finder = PathFinder(path)
self.assertEqual(list(path_finder.glob("glob_expr")),
[include_file_mock, exclude_file_mock])
path_finder.exclude_exprs = ["exclude/*"]
self.assertEqual(list(path_finder.glob("glob_expr")),
[include_file_mock])
path_finder.include_exprs = ["include/*"]
self.assertEqual(list(path_finder.glob("glob_expr")),
[include_file_mock])
def test_mkdir(self, _PathMock):
path = MagicMock()
path.exists.return_value = False
parent_path = MagicMock()
parent_path.exists.return_value = True
parent_path.is_dir.return_value = False
top_parent_path = MagicMock()
top_parent_path.exists.return_value = True
top_parent_path.is_dir.return_value = True
path.parents = [parent_path, top_parent_path]
path_finder = PathFinder(path)
path_finder.mkdir(force=True)
parent_path.unlink.assert_called_with()
parent_path.mkdir.assert_called_with()
path.mkdir.assert_called_with()
with self.assertRaises(PathFinderError):
path_finder.mkdir(force=False)
class MDDocument(object):
"""
Markdown file builder.
Can be used as a context manager, on exit context is written to `path`.
Examples::
md_doc = MDDocument(path=Path('output.md'))
md_doc.append('## New section')
md_doc.append('some content')
md_doc.title = 'My doc'
md_doc.add_toc_if_not_exists()
md_doc.write()
# output is indented for readability
Path('output.md').read_text()
'''# My doc
- [My doc](#my-doc)
- [New section](#new-section)
## New section
some content
'''
with MDDocument(path=Path('output.md')) as md_document:
md_document.title = 'My doc'
md_doc.append_title('New section', level=2)
md_doc.append('New line')
Arguments:
path -- Path to store document.
"""
# Indent in spaces for nested ToC lines
TOC_INDENT = 4
_anchor_re = re.compile(r"[^a-z0-9_-]+")
_escape_title_re = re.compile(r"(_+\S+_+)$")
_section_separator = "\n\n"
def __init__(self, path):
# type: (Path) -> None
self._sections = [] # type: List[Text]
self._content = ""
self._title = ""
self._subtitle = ""
self._toc_section = ""
self._path = path
self._path_finder = PathFinder(self._path.parent)
def __enter__(self):
# type: () -> MDDocument
return self
def __exit__(
self,
exc_type, # type: Optional[Type[BaseException]]
exc_value, # type: Optional[BaseException]
tb, # type: Optional[TracebackType]
):
# type: (...) -> None
if exc_value:
traceback.print_tb(tb)
raise exc_value
return self.write()
def read(self, source_path=None):
# type: (Optional[Path]) -> None
"""
Read and parse content from `source_path`.
Arguments:
source_path -- Input file path. If not provided - `path` is used.
"""
path = source_path or self._path
self._content = path.read_text()
self._title = ""
self._toc_section = ""
title, content = extract_md_title(self._content)
if title:
self._title = title
sections = content.split(self._section_separator)
self._sections = []
for section in sections:
section = IndentTrimmer.trim_empty_lines(section)
if not section:
continue
if self.is_toc(section) and not self._toc_section:
self._toc_section = section
if self._sections:
self._subtitle = self._section_separator.join(
self._sections)
self._sections = []
continue
self._sections.append(section)
# extract subtitle from the first section if it is not a title
if not self._subtitle and self._sections and not self._sections[
0].startswith("#"):
self._subtitle = self._sections.pop(0)
def add_toc_if_not_exists(self):
# type: () -> None
"""
Check if ToC exists in the document or create one.
"""
if not self._toc_section:
self._toc_section = self.generate_toc_section()
@classmethod
def get_anchor(cls, title):
# type: (Text) -> Text
"""
Convert title to a GitHub-friendly anchor link.
Returns:
A test of anchor link.
"""
title = title.lower().replace(" ", "-")
result = cls._anchor_re.sub("", title)
return result
@staticmethod
def is_toc(section):
# type: (Text) -> bool
"""
Check if the section is Tree of Contents.
Returns:
True the section is ToC.
"""
lines = section.split("\n")
if len(lines) < 2:
return False
for line in lines:
if "- [" not in line:
return False
return True
@classmethod
def render_link(cls, title, link):
# type: (Text, Text) -> Text
"""
Render Markdown link wih escaped title.
Examples::
MDDocument.render_link('my title', 'doc.md#test')
'[my title](doc.md#test)'
MDDocument.render_link('MyClass.__init__', 'my.md')
'[MyClass.__init__](doc.md#my.md)'
Arguments:
title -- Link text.
link -- Link target.
Returns:
A string with Markdown link.
"""
return "[{}]({})".format(title, link)
def render_md_doc_link(self, target_md_document, title=None):
# type: (MDDocument, Optional[Text]) -> Text
"""
Render Markdown link to `target_md_document` header path with a correct title.
Arguments:
target_md_document -- Target `MDDocument`.
title -- Link text. If not provided `target_md_document.title` is used.
Returns:
A string with Markdown link.
"""
return self.render_doc_link(
title=title or target_md_document.title,
anchor=self.get_anchor(target_md_document.title),
target_path=target_md_document.path,
)
def render_doc_link(self, title, anchor="", target_path=None):
# type: (Text, Text, Optional[Path]) -> Text
"""
Render Markdown link to a local MD document, use relative path as a link.
Examples::
md_doc = MDDocument(path='/root/parent/doc.md')
MDDocument.render_doc_link(
'my title',
anchor='my-anchor',
target_path=Path('/root/parent/doc.md'
)
'[my title](#my-anchor)'
MDDocument.render_doc_link('my title', target_path=Path('/root/parent/other.md'))
'[my title](other.md)'
MDDocument.render_doc_link('my title', anchor='my-anchor', target_path=Path('doc.md'))
'[my title](doc.md#my-anchor)'
MDDocument.render_doc_link('my title', anchor='my-anchor')
'[my title](#my-anchor)'
Arguments:
title -- Link text.
anchor -- Unescaped or escaped anchor tag.
target_path -- Target MDDocument path.
Returns:
A string with Markdown link.
"""
link = ""
if anchor:
link = "#{}".format(anchor)
if target_path and target_path != self._path:
link_path = self._path_finder.relative(target_path)
link = "{}{}".format(link_path, link)
return self.render_link(title, link)
def _build_content(self):
# type: () -> Text
sections = []
if self._title:
sections.append("# {}".format(self._title))
if self._subtitle:
sections.append(self._subtitle)
if self._toc_section:
sections.append(self._toc_section)
sections.extend(self._sections)
return self._section_separator.join(sections) + "\n"
def write(self):
# type: () -> None
"""
Write MD content to `path`.
"""
content = self._build_content()
self._path_finder.mkdir()
self._path.write_text(content)
@property
def title(self):
# type: () -> Text
"""
`MDDocument` title or an empty string.
"""
return self._title
@title.setter
def title(self, title):
# type: (Text) -> None
self._title = title
self._content = self._build_content()
@property
def subtitle(self):
# type: () -> Text
"""
`MDDocument` subtitle or an empty string.
"""
return self._subtitle
@subtitle.setter
def subtitle(self, subtitle):
# type: (Text) -> None
self._subtitle = subtitle
self._content = self._build_content()
@property
def toc_section(self):
# type: () -> Text
"""
Document Tree of Contents section or an empty line.
"""
return self._toc_section
@toc_section.setter
def toc_section(self, toc_section):
# type: (Text) -> None
self._toc_section = toc_section
self._content = self._build_content()
@property
def sections(self):
# type: () -> List[Text]
"""
All non-special `sections` of the document.
"""
return self._sections
@property
def path(self):
# type: () -> Path
"""
Output path of the document.
"""
return self._path
def append(self, content):
# type: (Text) -> None
"""
Append `content` to the document.
Handle trimming and sectioning the content and update
`title` and `toc_section` fields.
Arguments:
content -- Text to add.
"""
content = IndentTrimmer.trim_empty_lines(content)
if not content:
return
if not self.subtitle and not self.sections and not content.startswith(
"#"):
self.subtitle = content
else:
self._sections.append(content)
self._content = self._build_content()
def append_title(self, title, level):
# type: (Text, int) -> None
"""
Append `title` of a given `level` to the document.
Handle trimming and sectioning the content and update
`title` and `toc_section` fields.
Arguments:
title -- Title to add.
level -- Title level, number of `#` symbols.
"""
section = "{} {}".format("#" * level, self._escape_title(title))
self._sections.append(section)
self._content = self._build_content()
def generate_toc_section(self, max_depth=3):
# type: (int) -> Text
"""
Generate Table of Contents MD content.
Arguments:
max_depth -- Add headers to ToC only up to this level.
Returns:
A string with ToC.
"""
toc_lines = []
if self.title:
link = self.render_doc_link(self.title,
anchor=self.get_anchor(self.title))
toc_line = self.get_toc_line(link, level=0)
toc_lines.append(toc_line)
sections = [self.title, self.subtitle] + self.sections
for section in sections:
if not section.startswith("#"):
continue
if "\n" in section:
continue
if "# " not in section:
continue
section = section.rstrip()
header_symbols, title = section.split(" ", 1)
title = title.strip()
if not title:
continue
if header_symbols.replace("#", ""):
continue
header_level = len(header_symbols)
if header_level > max_depth:
continue
link = self.render_doc_link(title, anchor=self.get_anchor(title))
toc_line = self.get_toc_line(link, level=header_level - 1)
toc_lines.append(toc_line)
return "\n".join(toc_lines)
@classmethod
def get_toc_line(cls, line, level=0):
# type: (Text, int) -> Text
"""
Get ToC `line` of given `level`.
Arguments:
line -- Line to prepare.
level -- Line level, starts with `0`.
Returns:
Ready to insert ToC line.
"""
indent = cls.TOC_INDENT * level
return IndentTrimmer.indent_line("- {}".format(line), indent)
@classmethod
def _escape_title(cls, title):
# type: (Text) -> Text
for match in cls._escape_title_re.findall(title):
title = title.replace(match, match.replace("_", "\\_"))
return title
class Generator:
"""
Main documentation generator.
Arguments:
project_name -- Name of the project.
input_path -- Path to repo to generate docs.
output_path -- Path to folder with auto-generated docs to output.
source_paths -- List of paths to source files for generation.
docstring_processor -- Docstring converter to Markdown.
loader -- Loader for python modules.
raise_errors -- Raise `LoaderError` instead of silencing in.
source_code_url -- URL to source files to use instead of relative paths,
useful for [GitHub Pages](https://pages.github.com/).
toc_depth -- Maximum depth of child modules ToC
"""
# Name of logger
LOGGER_NAME = "handsdown"
# Docs index filename
INDEX_NAME = "README.md"
# Docs index title
INDEX_TITLE = "Index"
# Docs modules filename
MODULES_NAME = "MODULES.md"
# Docs modules title
MODULES_TITLE = "Modules"
_short_link_re = re.compile(r"`+[A-Za-z]\S+`+")
def __init__(
self,
input_path: Path,
output_path: Path,
source_paths: Iterable[Path],
project_name: Optional[str] = None,
docstring_processor: Optional[BaseDocstringProcessor] = None,
loader: Optional[Loader] = None,
raise_errors: bool = False,
source_code_url: Optional[str] = None,
toc_depth: int = 1,
) -> None:
self._logger = get_logger()
self._root_path = input_path
self._output_path = output_path
self._project_name = project_name or make_title(input_path.name)
self._root_path_finder = PathFinder(self._root_path)
self._source_code_url = source_code_url
self._toc_depth = toc_depth
self._raise_errors = raise_errors
# create output folder if it does not exist
if not self._output_path.exists():
self._logger.info("Creating folder {}".format(self._output_path))
PathFinder(self._output_path).mkdir()
self._loader = loader or Loader(root_path=self._root_path,
output_path=self._output_path)
self._docstring_processor = docstring_processor or SmartDocstringProcessor(
)
self._source_paths = sorted(source_paths)
self._error_output_paths: Set[Path] = set()
self._logger.debug("Generating source map for {} source files".format(
len(self._source_paths)))
self._module_records = self._build_module_record_list()
self._logger.debug("Source map generated")
package_names = self._module_records.get_package_names()
package_names_re_expr = "|".join(package_names)
self._docstring_links_re = re.compile(
r"`+(?:{})\.\S+`+".format(package_names_re_expr))
self._prepare_index()
def _prepare_index(self) -> None:
self.md_index = MDDocument(self._output_path / self.INDEX_NAME)
self.md_modules = MDDocument(self._output_path / self.MODULES_NAME)
# copy `README.md` content from root dir if it exists
readme_path = self._root_path / "README.md"
if readme_path.exists():
self.md_index.read(readme_path)
if not self.md_index.title:
self.md_index.title = "{} {}".format(self._project_name,
self.INDEX_TITLE)
# copy `MODULES.md` content from root dir if it exists
modules_path = self._root_path / "MODULES.md"
if modules_path.exists():
self.md_modules.read(modules_path)
if not self.md_modules.title:
self.md_modules.title = "{} {}".format(self._project_name,
self.MODULES_TITLE)
def _build_module_record_list(self) -> ModuleRecordList:
module_record_list = ModuleRecordList()
for source_path in self._source_paths:
module_record = None
try:
module_record = self._loader.get_module_record(source_path)
except LoaderError as e:
if self._raise_errors:
raise
self._logger.warning("Skipping: {}".format(e))
continue
if module_record:
if not module_record.title:
module_record.title = make_title(module_record.name)
module_record_list.add(module_record)
return module_record_list
def cleanup_old_docs(self) -> None:
"""
Remove old docs generated for this module.
"""
self._logger.debug("Removing orphaned docs")
preserve_paths = {
self._loader.get_output_path(i.source_path)
for i in self._module_records
}
orphaned_dirs = []
preserve_paths.add(self.md_index.path)
preserve_paths.add(self.md_modules.path)
# skip error output paths
# preserve_paths.update(self._error_output_paths)
for doc_path in PathFinder(self._output_path).glob("**/*.md"):
if doc_path in preserve_paths:
continue
file_content = doc_path.read_text()
is_autogenerated = "> Auto-generated documentation" in file_content
if not is_autogenerated:
continue
self._logger.info("Deleting orphaned doc file {}".format(
self._root_path_finder.relative(doc_path)))
doc_path.unlink()
# remove parent directory if it is empty
children = list(doc_path.parent.iterdir())
if not children:
orphaned_dirs.append(doc_path.parent)
for orphaned_dir in orphaned_dirs:
self._logger.info("Deleting orphaned directory {}".format(
self._root_path_finder.relative(orphaned_dir)))
orphaned_dir.rmdir()
def generate_doc(self, source_path: Path) -> None:
"""
Generate one module doc at once.
Arguments:
source_path -- Path to source file.
Raises:
GeneratorError -- If `source_path` not found in current repo.
"""
for module_record in self._module_records:
if module_record.source_path != source_path:
continue
output_path = self._loader.get_output_path(
module_record.source_path)
md_document = MDDocument(output_path)
self._generate_doc(module_record, md_document)
md_document.write()
return
raise GeneratorError("Record not found for {}".format(
source_path.name))
def _generate_doc(self, module_record: ModuleRecord,
md_document: MDDocument) -> None:
self._logger.debug("Generating doc {} for {}".format(
self._root_path_finder.relative(md_document.path),
self._root_path_finder.relative(module_record.source_path),
))
try:
self._loader.parse_module_record(module_record)
except LoaderError as e:
if self._raise_errors:
raise
self._logger.warning("Skipping: {}".format(e))
return
source_link = md_document.render_doc_link(
title=module_record.import_string.value,
target_path=module_record.source_path,
)
if self._source_code_url:
relative_path_str = self._root_path_finder.relative(
module_record.source_path).as_posix()
source_link = md_document.render_link(
title=module_record.import_string.value,
link="{}{}".format(self._source_code_url, relative_path_str),
)
md_document.title = module_record.title
self._render_docstring(module_record=module_record,
record=module_record,
md_document=md_document)
autogenerated_marker = "> Auto-generated documentation for {} module.".format(
source_link)
if md_document.subtitle:
md_document.subtitle = "{}\n\n{}".format(autogenerated_marker,
md_document.subtitle)
else:
md_document.subtitle = autogenerated_marker
self._generate_module_doc_lines(module_record, md_document)
md_document.add_toc_if_not_exists()
modules_toc_lines = self._build_modules_toc_lines(
module_record.import_string,
max_depth=self._toc_depth,
md_document=md_document,
start_level=2,
)
toc_lines = md_document.toc_section.split("\n")
breadscrumbs = self._build_breadcrumbs_string(
module_record=module_record, md_document=md_document)
toc_lines[0] = md_document.get_toc_line(breadscrumbs, level=0)
if modules_toc_lines:
toc_line = md_document.get_toc_line(self.MODULES_TITLE, level=1)
toc_lines.append(toc_line)
for line in modules_toc_lines:
toc_lines.append(line)
md_document.toc_section = "\n".join(toc_lines)
def _build_breadcrumbs_string(
self,
module_record: ModuleRecord,
md_document: MDDocument,
) -> str:
import_string_breadcrumbs: List[str] = []
parent_import_strings = []
import_string = module_record.import_string
while not import_string.is_top_level():
import_string = import_string.parent
parent_import_strings.append(import_string)
parent_import_strings.reverse()
for parent_import_string in parent_import_strings:
parent_module_record = self._module_records.find_module_record(
parent_import_string)
if not parent_module_record:
import_string_breadcrumbs.append("`{}`".format(
make_title(parent_import_string.parts[-1])))
continue
output_path = self._loader.get_output_path(
parent_module_record.source_path)
import_string_breadcrumbs.append(
md_document.render_doc_link(
parent_module_record.title,
target_path=output_path,
anchor=md_document.get_anchor(parent_module_record.title),
))
breadcrumbs = ([
md_document.render_md_doc_link(self.md_index,
title=self._project_name),
md_document.render_md_doc_link(self.md_modules,
title=self.MODULES_TITLE),
] + import_string_breadcrumbs + [module_record.title])
return " / ".join(breadcrumbs)
def generate_docs(self) -> None:
"""
Generate all doc files at once.
"""
self._logger.debug("Generating docs for {} to {}".format(
self._project_name,
self._root_path_finder.relative(self._output_path)))
for module_record in self._module_records:
output_path = self._loader.get_output_path(
module_record.source_path)
md_document = MDDocument(output_path)
self._generate_doc(module_record, md_document)
md_document.write()
def generate_index(self) -> None:
"""
Generate `<output>/README.md` file with title from `<root>/README.md` and `Modules`
section that contains a Tree of all modules in the project.
"""
self._logger.debug("Generating {}".format(
self._root_path_finder.relative(self.md_index.path)))
with self.md_index as md_index:
if not md_index.title:
md_index.title = "{} {}".format(self._project_name,
self.INDEX_TITLE)
autogenerated_marker = "> Auto-generated documentation index."
modules_section = "Full {} project documentation can be found in {}".format(
self._project_name,
md_index.render_md_doc_link(self.md_modules,
title=self.MODULES_TITLE),
)
subtitle_parts = [autogenerated_marker]
if md_index.subtitle:
subtitle_parts.append(md_index.subtitle)
subtitle_parts.append(modules_section)
md_index.subtitle = "\n\n".join(subtitle_parts)
md_index.add_toc_if_not_exists()
md_index.toc_section = "{}\n - {}".format(
md_index.toc_section,
md_index.render_md_doc_link(self.md_modules))
def generate_modules(self) -> None:
"""
Generate `<output>/README.md` file with title from `<root>/README.md` and `Modules`
section that contains a Tree of all modules in the project.
"""
self._logger.debug("Generating {}".format(
self._root_path_finder.relative(self.md_modules.path)))
with self.md_modules as md_modules:
if not md_modules.title:
md_modules.title = "{} {}".format(self._project_name,
self.MODULES_TITLE)
autogenerated_marker = "> Auto-generated documentation modules index."
subtitle_parts = [autogenerated_marker]
if md_modules.subtitle:
subtitle_parts.append(md_modules.subtitle)
subtitle_parts.append("Full list of {} project modules.".format(
md_modules.render_md_doc_link(self.md_index,
title=self._project_name)))
md_modules.subtitle = "\n\n".join(subtitle_parts)
modules_toc_lines = self._build_modules_toc_lines(
import_string=ImportString(""),
max_depth=10,
md_document=md_modules,
start_level=1,
)
md_doc_link = md_modules.render_md_doc_link(self.md_index)
modules_toc_lines.insert(
0, md_modules.get_toc_line(md_doc_link, level=0))
md_modules.toc_section = "\n".join(modules_toc_lines)
def _generate_module_doc_lines(
self,
module_record: ModuleRecord,
md_document: MDDocument,
) -> None:
for record in module_record.iter_records():
if isinstance(record, AttributeRecord):
continue
header_level = 2
if record.is_method:
header_level = 3
md_document.append_title(record.title, level=header_level)
source_path = module_record.source_path
source_line_number = record.line_number
source_link = md_document.render_doc_link(
title=FIND_IN_SOURCE_LABEL,
target_path=source_path,
anchor="L{}".format(source_line_number),
)
if self._source_code_url:
relative_path_str = self._root_path_finder.relative(
source_path).as_posix()
source_link = md_document.render_link(
title=FIND_IN_SOURCE_LABEL,
link="{}{}#L{}".format(self._source_code_url,
relative_path_str,
source_line_number),
)
md_document.append(source_link)
signature = record.render(allow_multiline=True)
md_document.append("```python\n{}\n```".format(signature))
self._render_docstring(module_record=module_record,
record=record,
md_document=md_document)
def _replace_links(
self,
module_record: ModuleRecord,
record: NodeRecord,
md_document: MDDocument,
docstring: str,
) -> str:
parent_import_string = None
if not record.import_string.is_top_level():
parent_import_string = record.import_string.parent
for match in self._short_link_re.findall(docstring):
related_record_name = match.replace("`", "")
related_import_string = None
related_record = None
target_path = md_document.path
# find record in parent
if parent_import_string:
related_import_string = parent_import_string + related_record_name
if related_import_string != record.import_string:
related_record = module_record.find_record(
related_import_string)
# find record in module
if not related_record:
related_import_string = module_record.import_string + related_record_name
related_record = module_record.find_record(
related_import_string)
# find record globally
if not related_record:
related_import_string = ImportString(related_record_name)
related_module_record = self._module_records.find_module_record(
related_import_string)
if related_module_record:
related_record = related_module_record.find_record(
related_import_string)
target_path = self._loader.get_output_path(
related_module_record.source_path)
if not related_record:
continue
if related_record.import_string.startswith(record.import_string):
continue
title = related_record.title
anchor = md_document.get_anchor(related_record.title)
if isinstance(related_record, AttributeRecord):
parent_related_record = module_record.find_record(
related_record.import_string.parent)
if parent_related_record:
anchor = md_document.get_anchor(
parent_related_record.title)
link = md_document.render_doc_link(title,
anchor=anchor,
target_path=target_path)
docstring = docstring.replace(match, link)
self._logger.debug("Adding local link '{}' to '{}'".format(
title, record.title))
return docstring
def _render_docstring(
self,
module_record: ModuleRecord,
record: NodeRecord,
md_document: MDDocument,
) -> None:
"""
Get object docstring and convert it to a valid markdown using
`handsdown.processors.base.BaseDocstringProcessor`.
Arguments:
module_record -- Parent ModuleRecord
record -- Target NodeRecord
md_document -- Output document.
Returns:
A module docstring with valid markdown.
"""
docstring = record.docstring
docstring = self._replace_links(module_record, record, md_document,
docstring)
section_map = self._docstring_processor.build_sections(docstring)
for attrubute in record.get_documented_attribute_strings():
section_map.add_line_indent("Attributes", "- {}".format(attrubute))
related_import_strings = record.get_related_import_strings(
module_record)
links = []
title = ""
for import_string in related_import_strings:
related_module_record = self._module_records.find_module_record(
import_string)
if not related_module_record:
continue
related_record = related_module_record.find_record(import_string)
if not related_record:
continue
if related_record is record:
continue
title = related_record.title
target_path = self._loader.get_output_path(
related_module_record.source_path)
link = md_document.render_doc_link(
title,
target_path=target_path,
anchor=md_document.get_anchor(title))
links.append(link)
links.sort()
for link in links:
section_map.add_line("See also", "- {}".format(link))
self._logger.debug(
"Adding link `{}` to `{}` `See also` section".format(
title, record.title))
for section in section_map.sections:
if section.title:
md_document.append_title(section.title, level=4)
for block in section.blocks:
md_document.append(block.render())
def _build_modules_toc_lines(self, import_string: ImportString,
max_depth: int, md_document: MDDocument,
start_level: int) -> List[str]:
lines: List[str] = []
parts = import_string.parts
last_import_string_parts: List[str] = []
for module_record in self._module_records:
output_path = self._loader.get_output_path(
module_record.source_path)
if module_record.import_string == import_string:
continue
if import_string and not module_record.import_string.startswith(
import_string):
continue
import_string_parts = module_record.import_string.parts
if len(import_string_parts) > len(parts) + max_depth:
continue
for index, import_string_part in enumerate(
import_string_parts[:-1]):
if index < len(parts):
continue
if (len(last_import_string_parts) > index
and last_import_string_parts[index]
== import_string_parts[index]):
continue
title = make_title(import_string_part)
toc_line = md_document.get_toc_line(title,
level=index - len(parts) +
start_level)
lines.append(toc_line)
last_import_string_parts = import_string_parts
link = md_document.render_doc_link(
title=module_record.title,
target_path=output_path,
anchor=md_document.get_anchor(module_record.title),
)
toc_line = md_document.get_toc_line(
link,
level=len(import_string_parts) - len(parts) - 1 + start_level)
lines.append(toc_line)
return lines
def test_exclude(self, PathMock):
path = PathMock()
path_finder = PathFinder(path)
self.assertEqual(path_finder.exclude_exprs, [])
path_finder = path_finder.exclude("my_dir", "expr/**/*")
self.assertEqual(path_finder.exclude_exprs, ["my_dir/*", "expr/**/*"])
def __init__(self, root_path: Path, output_path: Path) -> None:
self._logger = get_logger()
self._root_path = root_path
self._root_path_finder = PathFinder(self._root_path)
self._output_path = output_path
class Loader:
"""
Loader for python source code.
Examples::
loader = Loader(Path('path/to/my_module/'))
my_module_utils = loader.import_module('my_module.utils')
Arguments:
root_path -- Root path of the project.
output_path -- Docs output path.
"""
def __init__(self, root_path: Path, output_path: Path) -> None:
self._logger = get_logger()
self._root_path = root_path
self._root_path_finder = PathFinder(self._root_path)
self._output_path = output_path
def get_output_path(self, source_path: Path) -> Path:
"""
Get output MD document path based on `source_path`.
Arguments:
source_path -- Path to source code file.
Returns:
A path to the output `.md` file even if it does not exist yet.
"""
relative_source_path = self._root_path_finder.relative(source_path)
if relative_source_path.stem == "__init__":
relative_source_path = relative_source_path.parent / "index"
if relative_source_path.stem == "__main__":
relative_source_path = relative_source_path.parent / "module"
file_name = "{}.md".format(relative_source_path.stem)
relative_output_path = relative_source_path.parent / file_name
return self._output_path / relative_output_path
def get_module_record(self, source_path: Path) -> Optional[ModuleRecord]:
"""
Build `ModuleRecord` for given `source_path`.
Arguments:
source_path -- Absolute path to source file.
Returns:
A new `ModuleRecord` instance or None if there is ntohing to import.
Raises:
LoaderError -- If python source cannot be loaded.
"""
if not (source_path.parent / "__init__.py").exists():
return None
if source_path.name == "__init__.py" and source_path.parent == self._root_path:
return None
import_string = self.get_import_string(source_path)
docstring_parts = []
try:
module_record = ModuleRecord.create_from_source(
source_path, ImportString(import_string)
)
module_record.build_children()
except Exception as e:
raise LoaderError(
"{} while loading {}: {}".format(e.__class__.__name__, source_path, e)
) from e
if module_record.docstring:
docstring_parts.append(module_record.docstring)
if source_path.name == "__init__.py":
readme_md_path = source_path.parent / "README.md"
if readme_md_path.exists():
docstring_parts.append(readme_md_path.read_text())
docstring = "\n\n".join(docstring_parts)
title, docstring = extract_md_title(docstring)
if title:
module_record.title = title
module_record.docstring = docstring
return module_record
@staticmethod
def parse_module_record(module_record: ModuleRecord) -> None:
"""
Parse `ModuleRecord` children and fully load a tree for it.
Raises:
LoaderError -- If python source cannot be parsed.
"""
try:
module_record.parse()
except Exception as e:
raise LoaderError(
"{} while parsing {}: {}".format(e.__class__.__name__, module_record.source_path, e)
) from e
def get_import_string(self, source_path: Path) -> str:
"""
Get Python import string for a source `source_path` relative to `root_path`.
Examples::
loader = Loader(root_path=Path("/root"), ...)
loader.get_import_string('/root/my_module/test.py')
'my_module.test'
loader.get_import_string('/root/my_module/__init__.py')
'my_module'
Arguments:
source_path -- Path to a source file.
Returns:
A Python import string.
"""
relative_path = source_path.relative_to(self._root_path)
name_parts = []
for part in relative_path.parts:
stem = part.split(".")[0]
if stem == "__init__":
continue
name_parts.append(stem)
return ".".join(name_parts)