def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(
os.path.join(here, 'tmp-markdown-files'))
self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.__priv_dir)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.test_ext = TestExtension(self)
self.core_ext = CoreExtension(self)
def __setup_database(self):
self.doc_database = DocDatabase()
self.doc_database.comment_added_signal.connect(self.__add_default_tags)
self.doc_database.comment_updated_signal.connect(
self.__add_default_tags)
self.doc_database.setup(self.get_private_folder())
self.link_resolver = LinkResolver(self.doc_database)
def setUp(self):
# Cruft, should be unnecessary soon
self.tag_validators = {}
self.parser = GtkDocParser(self)
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.formatter = GtkDocStringFormatter()
def __setup_database(self):
self.doc_database = DocDatabase()
self.doc_database.comment_added_signal.connect(self.__add_default_tags)
self.doc_database.comment_updated_signal.connect(
self.__add_default_tags)
self.doc_database.setup(self.get_private_folder())
self.link_resolver = LinkResolver(self.doc_database)
def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(os.path.join(
here, 'tmp-markdown-files'))
self.__priv_dir = os.path.abspath(os.path.join(
here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(
here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(
here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.__priv_dir)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.test_ext = TestExtension(self)
self.core_ext = CoreExtension(self)
def test_incremental(self):
param = ParameterSymbol(type_tokens=[Link(None, "test-struct", "test-struct")])
func = self.doc_database.get_or_create_symbol(
FunctionSymbol, unique_name="test-symbol", filename="text_b.x", parameters=[param]
)
func.resolve_links(self.link_resolver)
self.assertEqual(param.get_type_link().get_link(), None)
struct = self.doc_database.get_or_create_symbol(StructSymbol, unique_name="test-struct", filename="test_a.x")
struct.resolve_links(self.link_resolver)
func.resolve_links(self.link_resolver)
# Not in a page but still
self.assertEqual(param.get_type_link().get_link(), "test-struct")
self.doc_database.persist()
self.doc_database.close()
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
param = ParameterSymbol(type_tokens=[Link(None, "test-struct", "test-struct")])
func = self.doc_database.get_or_create_symbol(
FunctionSymbol, unique_name="test-symbol", filename="text_b.x", parameters=[param]
)
func.resolve_links(self.link_resolver)
self.assertEqual(param.get_type_link().get_link(), "test-struct")
def setUp(self):
here = os.path.dirname(__file__)
self.__priv_dir = os.path.abspath(os.path.join(here, "tmp-private"))
self.__remove_tmp_dirs()
os.mkdir(self.__priv_dir)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
def setUp(self):
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.link_resolver.add_link(Link("here.com", "foo", "foo"))
self.link_resolver.add_link(Link("there.org", "there", "Test::test"))
self.link_resolver.add_link(Link("wherever.biz", "wherever", "bar"))
self.link_resolver.add_link(Link("whenever.net", "whenever", "Test"))
self.link_resolver.add_link(
Link("somewhere.me", "somewhere", "Test.baz"))
self.link_resolver.add_link(
Link("elsewhere.co", "elsewhere", "org.dbus.func"))
def setUp(self):
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.include_resolver = MockIncludeResolver()
def setUp(self):
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
self.link_resolver.add_link(Link("here.com", "foo", "foo"))
def setUp(self):
self.doc_database = DocDatabase()
self.link_resolver = LinkResolver(self.doc_database)
class DocRepo(object):
"""
Banana banana
"""
formatted_signal = Signal()
def __init__(self):
self.output = None
self.doc_tree = None
self.change_tracker = None
self.output_format = None
self.include_paths = None
self.extensions = {}
self.tag_validators = {}
self.link_resolver = None
self.incremental = False
self.doc_database = None
self.config = None
self.project_name = None
self.project_version = None
self.sitemap_path = None
if os.name == 'nt':
self.datadir = os.path.join(
os.path.dirname(__file__), '..', 'share')
else:
self.datadir = "/usr/share"
self.__conf_file = None
self.__extension_classes = {
CoreExtension.extension_name: CoreExtension}
self.__index_file = None
self.__root_page = None
self.__base_doc_folder = None
self.__private_folder = None
self.__dry = False
self.__load_extensions()
self.__create_arg_parser()
def register_tag_validator(self, validator):
"""
Banana banana
"""
self.tag_validators[validator.name] = validator
def format_symbol(self, symbol_name):
"""
Banana banana
"""
sym = self.doc_database.get_symbol(symbol_name)
if not sym:
return None
sym.update_children_comments()
return sym.detailed_description
def patch_page(self, symbol, raw_comment):
"""
Banana banana
"""
pages = self.doc_tree.get_pages_for_symbol(symbol.unique_name)
if not pages:
return False
old_comment = symbol.comment
# pylint: disable=no-member
new_comment = self.raw_comment_parser.parse_comment(
raw_comment,
old_comment.filename,
old_comment.lineno,
old_comment.lineno + raw_comment.count('\n'),
self.include_paths)
if new_comment is None:
return False
if new_comment.name != symbol.unique_name:
return False
pages = pages.values()
symbol.comment = new_comment
for page in pages:
formatter = self.__get_formatter(page.extension_name)
formatter.patch_page(page, symbol, self.output)
return True
def persist(self):
"""
Banana banana
"""
if self.__dry:
return
info('Persisting database and private files', 'persisting')
self.doc_tree.persist()
self.doc_database.persist()
pickle.dump(self.change_tracker,
open(os.path.join(self.get_private_folder(),
'change_tracker.p'), 'wb'))
self.__dump_deps_file()
def finalize(self):
"""
Banana banana
"""
self.formatted_signal.clear()
if self.doc_database is not None:
info('Closing database')
self.doc_database.close()
# pylint: disable=no-self-use
def get_private_folder(self):
"""
Banana banana
"""
return self.__private_folder
def setup(self):
"""
Banana banana
"""
configurable_classes = all_subclasses(Configurable)
configured = set()
for subclass in configurable_classes:
if subclass.parse_config not in configured:
subclass.parse_config(self, self.config)
configured.add(subclass.parse_config)
self.__parse_config()
self.doc_tree = DocTree(self.get_private_folder(), self.include_paths)
for extension in self.extensions.values():
info('Setting up %s' % extension.extension_name)
extension.setup()
self.doc_database.flush()
sitemap = SitemapParser().parse(self.sitemap_path)
self.doc_tree.parse_sitemap(self.change_tracker, sitemap)
info("Resolving symbols", 'resolution')
self.doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
self.doc_database.flush()
def format(self):
"""
Banana banana
"""
if not self.output:
return
self.doc_tree.format(self.link_resolver, self.output, self.extensions)
self.config.dump(conf_file=os.path.join(self.output, 'hotdoc.json'))
self.formatted_signal(self)
def __dump_deps_file(self):
dest = self.config.get('deps_file_dest')
target = self.config.get('deps_file_target')
if dest is None:
info("Not dumping deps file")
return
info("Dumping deps file to %s with target %s" % (dest, target))
destdir = os.path.dirname(dest)
if not os.path.exists(destdir):
os.makedirs(destdir)
empty_targets = []
with io.open(dest, 'w', encoding='utf-8') as _:
_.write(u'%s: ' % target)
if self.config:
for dep in self.config.get_dependencies():
empty_targets.append(dep)
_.write(u'%s ' % dep)
if self.doc_tree:
for page in self.doc_tree.get_pages().values():
if not page.generated:
empty_targets.append(page.source_file)
_.write(u'%s ' % page.source_file)
for empty_target in empty_targets:
_.write(u'\n\n%s:' % empty_target)
def __add_default_tags(self, _, comment):
for validator in self.tag_validators.values():
if validator.default and validator.name not in comment.tags:
comment.tags[validator.name] = \
Tag(name=validator.name,
description=validator.default)
def __setup_database(self):
self.doc_database = DocDatabase()
self.doc_database.comment_added_signal.connect(self.__add_default_tags)
self.doc_database.comment_updated_signal.connect(
self.__add_default_tags)
self.doc_database.setup(self.get_private_folder())
self.link_resolver = LinkResolver(self.doc_database)
def __create_change_tracker(self):
try:
self.change_tracker = \
pickle.load(open(os.path.join(self.get_private_folder(),
'change_tracker.p'), 'rb'))
if self.change_tracker.hard_dependencies_are_stale():
raise IOError
self.incremental = True
info("Building incrementally")
# pylint: disable=broad-except
except Exception:
info("Building from scratch")
shutil.rmtree(self.get_private_folder(), ignore_errors=True)
if self.output:
shutil.rmtree(self.output, ignore_errors=True)
self.change_tracker = ChangeTracker()
def __get_formatter(self, extension_name):
"""
Banana banana
"""
ext = self.extensions.get(extension_name)
if ext:
return ext.get_formatter(self.output_format)
return None
def __check_initial_args(self, args):
if args.version:
print VERSION
elif args.makefile_path:
here = os.path.dirname(__file__)
path = os.path.join(here, '..', 'utils', 'hotdoc.mk')
print os.path.abspath(path)
elif args.get_conf_path:
key = args.get_conf_path
path = self.config.get_path(key, rel_to_cwd=True)
if path is not None:
print path
elif args.get_conf_key:
key = args.get_conf_key
value = self.config.get(args.get_conf_key, None)
if value is not None:
print value
elif args.get_private_folder:
print os.path.relpath(self.__private_folder,
self.config.get_invoke_dir())
elif args.has_extension:
ext_name = args.has_extension
print ext_name in self.__extension_classes
elif args.list_extensions:
for ext_name in self.__extension_classes:
print ext_name
else:
self.parser.print_usage()
def __create_arg_parser(self):
self.parser = \
argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,)
configurable_classes = all_subclasses(Configurable)
seen = set()
for subclass in configurable_classes:
if subclass.add_arguments not in seen:
subclass.add_arguments(self.parser)
seen.add(subclass.add_arguments)
self.parser.add_argument('command', action="store",
choices=('run', 'conf', 'help'),
nargs="?")
self.parser.add_argument('--dry',
help='Dry run, nothing will be output',
dest='dry', action='store_true')
self.parser.add_argument('--conf-file', help='Path to the config file',
dest='conf_file')
self.parser.add_argument('--output-conf-file',
help='Path where to save the updated conf'
' file',
dest='output_conf_file')
self.parser.add_argument('--version', help="Print version and exit",
action="store_true")
self.parser.add_argument('--makefile-path',
help="Print path to includable "
"Makefile and exit",
action="store_true")
self.parser.add_argument('--deps-file-dest',
help='Where to output the dependencies file')
self.parser.add_argument('--deps-file-target',
help='Name of the dependencies target',
default='doc.stamp.d')
self.parser.add_argument("-i", "--index", action="store",
dest="index", help="location of the "
"index file")
self.parser.add_argument("--sitemap", action="store",
dest="sitemap",
help="Location of the sitemap file")
self.parser.add_argument("--project-name", action="store",
dest="project_name",
help="Name of the documented project")
self.parser.add_argument("--project-version", action="store",
dest="project_version",
help="Version of the documented project")
self.parser.add_argument("-o", "--output", action="store",
dest="output",
help="where to output the rendered "
"documentation")
self.parser.add_argument("--extra-extensions-paths", action="append",
dest="extra_extension_paths", default=[],
help="Extra paths to lookup extensions in")
self.parser.add_argument("--get-conf-key", action="store",
help="print the value for a configuration "
"key")
self.parser.add_argument("--get-conf-path", action="store",
help="print the value for a configuration "
"path")
self.parser.add_argument("--get-private-folder", action="store_true",
help="get the path to hotdoc's private "
"folder")
self.parser.add_argument("--output-format", action="store",
dest="output_format", help="format for the "
"output")
self.parser.add_argument("--has-extension", action="store",
dest="has_extension", help="Check if a given "
"extension is available")
self.parser.add_argument("--list-extensions", action="store_true",
dest="list_extensions", help="Print "
"available extensions")
self.parser.add_argument('--include-paths',
help='paths to look up included files in',
dest='include_paths', action='append',
default=[])
self.parser.add_argument("-", action="store_true",
help="Separator to allow finishing a list"
" of arguments before a command",
dest="whatever")
def load_command_line(self, args):
"""
Loads the repo from command line arguments
"""
args = self.parser.parse_args(args)
self.__load_config(args)
cmd = args.command
if cmd == 'help':
self.parser.print_help()
sys.exit(0)
elif cmd is None:
self.__check_initial_args(args)
sys.exit(0)
exit_now = False
save_config = False
if cmd == 'run':
self.__dry = bool(args.dry)
elif cmd == 'conf':
save_config = True
exit_now = True
if save_config:
self.config.dump(args.output_conf_file)
if exit_now:
sys.exit(0)
def load_conf_file(self, conf_file, overrides):
"""
Load the project from a configuration file and key-value
overides.
"""
if conf_file is None and os.path.exists('hotdoc.json'):
conf_file = 'hotdoc.json'
self.__conf_file = conf_file
if conf_file and not os.path.exists(conf_file):
error('invalid-config',
"No configuration file was found at %s" % conf_file)
actual_args = {}
defaults = {'output_format': 'html'}
for key, value in overrides.items():
if key in ('cmd', 'conf_file', 'dry'):
continue
if value != self.parser.get_default(key):
actual_args[key] = value
if self.parser.get_default(key) is not None:
defaults[key] = value
self.config = ConfigParser(command_line_args=actual_args,
conf_file=conf_file,
defaults=defaults)
index = self.config.get_index()
if index:
hash_obj = hashlib.md5(self.config.get_index())
priv_name = 'hotdoc-private-' + hash_obj.hexdigest()
else:
priv_name = 'hotdoc-private'
self.__private_folder = os.path.abspath(priv_name)
def __load_extensions(self):
extension_classes = get_installed_extension_classes(True)
for subclass in extension_classes:
self.__extension_classes[subclass.extension_name] = subclass
# pylint: disable=no-self-use
def __load_config(self, args):
"""
Banana banana
"""
cli = dict(vars(args))
self.load_conf_file(args.conf_file, cli)
def __setup_private_folder(self):
folder = self.get_private_folder()
if os.path.exists(folder):
if not os.path.isdir(folder):
error('setup-issue',
'%s exists but is not a directory' % folder)
else:
os.mkdir(folder)
def __create_extensions(self, extra_paths):
for ext_class in self.__extension_classes.values():
ext = ext_class(self)
self.extensions[ext.extension_name] = ext
extra_classes = get_extra_extension_classes(extra_paths)
self.__extension_classes.update(extra_classes)
for ext_class in extra_classes.values():
ext = ext_class(self)
self.extensions[ext.extension_name] = ext
def get_base_doc_folder(self):
"""Get the folder in which the main index was located
"""
return self.__base_doc_folder
def __parse_config(self):
"""
Banana banana
"""
output = self.config.get_path('output') or None
self.sitemap_path = self.config.get_path('sitemap')
if self.sitemap_path is None:
error('invalid-config',
'No sitemap was provided')
if output is not None:
self.output = os.path.abspath(output)
else:
self.output = None
self.project_name = self.config.get('project_name', None)
self.project_version = self.config.get('project_version', None)
self.output_format = self.config.get('output_format')
if self.output_format not in ["html"]:
error('invalid-config',
'Unsupported output format : %s' % self.output_format)
self.__index_file = self.config.get_index()
if self.__index_file is None:
error('invalid-config', 'index is required')
if not os.path.exists(self.__index_file):
error('invalid-config',
'The provided index "%s" does not exist' %
self.__index_file)
cmd_line_includes = self.config.get_paths('include_paths')
self.__base_doc_folder = os.path.dirname(self.__index_file)
self.include_paths = OrderedSet([self.__base_doc_folder])
self.include_paths |= OrderedSet(cmd_line_includes)
self.__create_change_tracker()
self.__setup_private_folder()
self.__setup_database()
self.__create_extensions(
self.config.get_paths('extra_extension_paths'))
if self.__conf_file:
self.change_tracker.add_hard_dependency(self.__conf_file)
class TestDocTree(unittest.TestCase):
def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(os.path.join(
here, 'tmp-markdown-files'))
self.__priv_dir = os.path.abspath(os.path.join(
here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(
here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(
here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.__priv_dir)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.test_ext = TestExtension(self)
self.core_ext = CoreExtension(self)
def tearDown(self):
self.__remove_tmp_dirs()
del self.test_ext
del self.core_ext
def get_generated_doc_folder(self):
return os.path.join(self.__priv_dir, 'generated')
def get_base_doc_folder(self):
return self.__md_dir
def get_private_folder(self):
return self.__priv_dir
def __parse_sitemap(self, text):
path = os.path.join(self.__md_dir, 'sitemap.txt')
with io.open(path, 'w', encoding='utf-8') as _:
_.write(text)
return self.sitemap_parser.parse(path)
def __create_md_file(self, name, contents):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
_.write(contents)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
def __create_src_file(self, name, symbols):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
for symbol in symbols:
_.write('%s\n' % symbol)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
return path
def __remove_src_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __remove_md_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __touch_src_file(self, name):
path = os.path.join(self.__md_dir, name)
touch(path)
def __remove_tmp_dirs(self):
shutil.rmtree(self.__md_dir, ignore_errors=True)
shutil.rmtree(self.__priv_dir, ignore_errors=True)
shutil.rmtree(self.__src_dir, ignore_errors=True)
shutil.rmtree(self.__output_dir, ignore_errors=True)
shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True)
def test_basic(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
pages = doc_tree.get_pages()
# We do not care about ordering
self.assertSetEqual(
set(pages.keys()),
set([u'index.markdown',
u'section.markdown']))
index = pages.get('index.markdown')
self.assertEqual(index.title, u'My documentation')
def test_basic_incremental(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
# Building from scratch, all pages are stale
self.assertSetEqual(
set(doc_tree.get_stale_pages()),
set([u'index.markdown',
u'section.markdown']))
doc_tree.persist()
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
# Nothing changed, no page is stale
self.assertSetEqual(
set(doc_tree.get_stale_pages()),
set({}))
# But we still have our pages
self.assertSetEqual(
set(doc_tree.get_pages()),
set([u'index.markdown',
u'section.markdown']))
touch(os.path.join(self.__md_dir, u'section.markdown'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
self.assertSetEqual(
set(doc_tree.get_stale_pages()),
set([u'section.markdown']))
def __assert_extension_names(self, doc_tree, name_map):
pages = doc_tree.get_pages()
for name, ext_name in name_map.items():
page = pages[name]
self.assertEqual(ext_name, page.extension_name)
def __assert_stale(self, doc_tree, expected_stale):
stale_pages = doc_tree.get_stale_pages()
for pagename in expected_stale:
self.assertIn(pagename, stale_pages)
stale_pages.pop(pagename)
self.assertEqual(len(stale_pages), 0)
def __create_test_layout(self):
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_x.markdown\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
sources = []
sources.append(self.__create_src_file(
'source_a.test',
['symbol_1',
'symbol_2']))
sources.append(self.__create_src_file(
'source_b.test',
['symbol_3',
'symbol_4']))
self.test_ext.index = 'test-index.markdown'
self.test_ext.sources = sources
self.test_ext.setup()
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'core_page.markdown',
(u'# My non-extension page\n'))
self.__create_md_file(
'test-index.markdown',
(u'# My test index\n'))
self.__create_md_file(
'test-section.markdown',
(u'# My test section\n'
'\n'
'Linking to [a generated page](source_a.test)\n'))
self.__create_md_file(
'page_x.markdown',
(u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
self.__create_md_file(
'page_y.markdown',
(u'# Page Y\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
return doc_tree, sitemap
def __update_test_layout(self, doc_tree, sitemap):
self.test_ext.reset()
self.test_ext.setup()
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
return doc_tree
def test_extension_basic(self):
doc_tree, _ = self.__create_test_layout()
self.__assert_extension_names(
doc_tree,
{u'index.markdown': 'core',
u'test-index': 'test-extension',
u'test-section.markdown': 'test-extension',
u'source_a.test': 'test-extension',
u'source_b.test': 'test-extension',
u'page_x.markdown': 'test-extension',
u'page_y.markdown': 'test-extension',
u'core_page.markdown': 'core'})
all_pages = doc_tree.get_pages()
self.assertEqual(len(all_pages), 8)
self.__assert_stale(doc_tree, all_pages)
self.assertNotIn('source_a.test', all_pages['test-index'].subpages)
self.assertIn('source_a.test',
all_pages['test-section.markdown'].subpages)
def test_extension_override(self):
self.__create_md_file(
'source_a.test.markdown',
(u'# My override\n'))
doc_tree, _ = self.__create_test_layout()
page = doc_tree.get_pages()['source_a.test']
self.assertEqual(
page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
self.assertEqual(
os.path.basename(page.source_file),
'source_a.test.markdown')
self.assertEqual(
cmark.ast_to_html(page.ast, None),
u'<h1>My override</h1>\n')
def test_parse_yaml(self):
inp = (u'index.markdown\n')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'---\n'
'title: A random title\n'
'symbols: [symbol_1, symbol_2]\n'
'...\n'
'# My documentation\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
pages = doc_tree.get_pages()
page = pages.get('index.markdown')
self.assertEqual(
cmark.ast_to_html(page.ast, None),
u'<h1>My documentation</h1>\n')
self.assertEqual(page.title, u'A random title')
self.assertEqual(
page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
def test_empty_link_resolution(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'
'\n'
'[](index.markdown)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(
self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents,
u'<h1>My section</h1>\n'
'<p><a href="index.html">My documentation</a></p>\n')
def test_labeled_link_resolution(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'
'\n'
'[a label](index.markdown)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(
self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents,
u'<h1>My section</h1>\n'
'<p><a href="index.html">a label</a></p>\n')
def test_anchored_link_resolution(self):
inp = (u'index.markdown\n'
'\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file(
'index.markdown',
(u'# My documentation\n'))
self.__create_md_file(
'section.markdown',
(u'# My section\n'
'\n'
'[](index.markdown#subsection)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(
self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents,
u'<h1>My section</h1>\n'
'<p><a href="index.html#subsection">My documentation</a></p>\n')
# pylint: disable=too-many-statements
def test_extension_incremental(self):
doc_tree, sitemap = self.__create_test_layout()
doc_tree.persist()
# Here we touch source_a.test, as its symbols were
# all contained in a generated page, only that page
# should now be stale
self.__touch_src_file('source_a.test')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['source_a.test']))
doc_tree.persist()
# We now touch source_b.test, which symbols are contained
# both in a generated page and a user-provided one.
# We expect both pages to be stale
self.__touch_src_file('source_b.test')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test',
'page_x.markdown']))
doc_tree.persist()
# This one is trickier: we unlist symbol_3 from
# page_x, which means the symbol should now be
# documented in the generated page for source_b.test.
# We expect both pages to be stale, and make sure
# they contain the right symbols
self.__create_md_file(
'page_x.markdown',
(u'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet())
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
doc_tree.persist()
# Let's make sure the opposite use case works as well,
# we relocate symbol_3 in page_x , both page_x and
# the generated page for source_b.test should be stale
# and the symbols should be back to their original
# layout.
self.__create_md_file(
'page_x.markdown',
(u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
doc_tree.persist()
# We now move the definition of symbol_3 to source_a.test,
# we thus expect the generated page for source_a.test to be
# stale because its source changed, same for source_b.test,
# and page_x.markdown should be stale as well because the
# definition of symbol_3 may have changed. The final
# symbol layout should not have changed however.
self.__create_src_file(
'source_a.test',
['symbol_1',
'symbol_2',
'symbol_3'])
self.__create_src_file(
'source_b.test',
['symbol_4'])
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_a.test',
'source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
source_a_page = doc_tree.get_pages()['source_a.test']
self.assertEqual(
source_a_page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
doc_tree.persist()
# And we rollback again
self.__create_src_file(
'source_a.test',
['symbol_1',
'symbol_2'])
self.__create_src_file(
'source_b.test',
['symbol_3',
'symbol_4'])
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['source_a.test',
'source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
source_a_page = doc_tree.get_pages()['source_a.test']
self.assertEqual(
source_a_page.symbol_names,
OrderedSet(['symbol_1',
'symbol_2']))
doc_tree.persist()
# Now we'll try removing page_x altogether
self.__remove_md_file('page_x.markdown')
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
new_sitemap = self.__parse_sitemap(inp)
doc_tree = self.__update_test_layout(doc_tree, new_sitemap)
self.__assert_stale(doc_tree,
set(['source_b.test']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
doc_tree.persist()
# And rollback again
self.__create_md_file(
'page_x.markdown',
(u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree,
set(['page_x.markdown',
'source_b.test']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(
source_b_page.symbol_names,
OrderedSet(['symbol_4']))
doc_tree.persist()
class TestLinkResolver(unittest.TestCase):
def setUp(self):
here = os.path.dirname(__file__)
self.__priv_dir = os.path.abspath(os.path.join(here, "tmp-private"))
self.__remove_tmp_dirs()
os.mkdir(self.__priv_dir)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
def __remove_tmp_dirs(self):
shutil.rmtree(self.__priv_dir, ignore_errors=True)
def test_incremental(self):
param = ParameterSymbol(type_tokens=[Link(None, "test-struct", "test-struct")])
func = self.doc_database.get_or_create_symbol(
FunctionSymbol, unique_name="test-symbol", filename="text_b.x", parameters=[param]
)
func.resolve_links(self.link_resolver)
self.assertEqual(param.get_type_link().get_link(), None)
struct = self.doc_database.get_or_create_symbol(StructSymbol, unique_name="test-struct", filename="test_a.x")
struct.resolve_links(self.link_resolver)
func.resolve_links(self.link_resolver)
# Not in a page but still
self.assertEqual(param.get_type_link().get_link(), "test-struct")
self.doc_database.persist()
self.doc_database.close()
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
param = ParameterSymbol(type_tokens=[Link(None, "test-struct", "test-struct")])
func = self.doc_database.get_or_create_symbol(
FunctionSymbol, unique_name="test-symbol", filename="text_b.x", parameters=[param]
)
func.resolve_links(self.link_resolver)
self.assertEqual(param.get_type_link().get_link(), "test-struct")
class TestDocTree(unittest.TestCase):
def setUp(self):
here = os.path.dirname(__file__)
self.__md_dir = os.path.abspath(
os.path.join(here, 'tmp-markdown-files'))
self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private'))
self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files'))
self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output'))
self.__remove_tmp_dirs()
os.mkdir(self.__md_dir)
os.mkdir(self.__priv_dir)
os.mkdir(self.__src_dir)
os.mkdir(self.get_generated_doc_folder())
self.include_paths = OrderedSet([self.__md_dir])
self.include_paths.add(self.get_generated_doc_folder())
# Using the real doc database is too costly, tests should be lightning
# fast (and they are)
self.doc_database = DocDatabase()
self.doc_database.setup(self.__priv_dir)
self.link_resolver = LinkResolver(self.doc_database)
self.change_tracker = ChangeTracker()
self.sitemap_parser = SitemapParser()
self.test_ext = TestExtension(self)
self.core_ext = CoreExtension(self)
def tearDown(self):
self.__remove_tmp_dirs()
del self.test_ext
del self.core_ext
def get_generated_doc_folder(self):
return os.path.join(self.__priv_dir, 'generated')
def get_base_doc_folder(self):
return self.__md_dir
def get_private_folder(self):
return self.__priv_dir
def __parse_sitemap(self, text):
path = os.path.join(self.__md_dir, 'sitemap.txt')
with io.open(path, 'w', encoding='utf-8') as _:
_.write(text)
return self.sitemap_parser.parse(path)
def __create_md_file(self, name, contents):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
_.write(contents)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
def __create_src_file(self, name, symbols):
path = os.path.join(self.__md_dir, name)
with open(path, 'w') as _:
for symbol in symbols:
_.write('%s\n' % symbol)
# Just making sure we don't hit a race condition,
# in real world situations it is assumed users
# will not update source files twice in the same
# microsecond
touch(path)
return path
def __remove_src_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __remove_md_file(self, name):
path = os.path.join(self.__md_dir, name)
os.unlink(path)
def __touch_src_file(self, name):
path = os.path.join(self.__md_dir, name)
touch(path)
def __remove_tmp_dirs(self):
shutil.rmtree(self.__md_dir, ignore_errors=True)
shutil.rmtree(self.__priv_dir, ignore_errors=True)
shutil.rmtree(self.__src_dir, ignore_errors=True)
shutil.rmtree(self.__output_dir, ignore_errors=True)
shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True)
def test_basic(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown', (u'# My section\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
pages = doc_tree.get_pages()
# We do not care about ordering
self.assertSetEqual(set(pages.keys()),
set([u'index.markdown', u'section.markdown']))
index = pages.get('index.markdown')
self.assertEqual(index.title, u'My documentation')
def test_basic_incremental(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown', (u'# My section\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
# Building from scratch, all pages are stale
self.assertSetEqual(set(doc_tree.get_stale_pages()),
set([u'index.markdown', u'section.markdown']))
doc_tree.persist()
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
# Nothing changed, no page is stale
self.assertSetEqual(set(doc_tree.get_stale_pages()), set({}))
# But we still have our pages
self.assertSetEqual(set(doc_tree.get_pages()),
set([u'index.markdown', u'section.markdown']))
touch(os.path.join(self.__md_dir, u'section.markdown'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
self.assertSetEqual(set(doc_tree.get_stale_pages()),
set([u'section.markdown']))
def __assert_extension_names(self, doc_tree, name_map):
pages = doc_tree.get_pages()
for name, ext_name in name_map.items():
page = pages[name]
self.assertEqual(ext_name, page.extension_name)
def __assert_stale(self, doc_tree, expected_stale):
stale_pages = doc_tree.get_stale_pages()
for pagename in expected_stale:
self.assertIn(pagename, stale_pages)
stale_pages.pop(pagename)
self.assertEqual(len(stale_pages), 0)
def __create_test_layout(self):
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_x.markdown\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
sources = []
sources.append(
self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2']))
sources.append(
self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4']))
self.test_ext.index = 'test-index.markdown'
self.test_ext.sources = sources
self.test_ext.setup()
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('core_page.markdown',
(u'# My non-extension page\n'))
self.__create_md_file('test-index.markdown', (u'# My test index\n'))
self.__create_md_file(
'test-section.markdown',
(u'# My test section\n'
'\n'
'Linking to [a generated page](source_a.test)\n'))
self.__create_md_file('page_x.markdown', (u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
self.__create_md_file('page_y.markdown', (u'# Page Y\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
return doc_tree, sitemap
def __update_test_layout(self, doc_tree, sitemap):
self.test_ext.reset()
self.test_ext.setup()
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
return doc_tree
def test_extension_basic(self):
doc_tree, _ = self.__create_test_layout()
self.__assert_extension_names(
doc_tree, {
u'index.markdown': 'core',
u'test-index': 'test-extension',
u'test-section.markdown': 'test-extension',
u'source_a.test': 'test-extension',
u'source_b.test': 'test-extension',
u'page_x.markdown': 'test-extension',
u'page_y.markdown': 'test-extension',
u'core_page.markdown': 'core'
})
all_pages = doc_tree.get_pages()
self.assertEqual(len(all_pages), 8)
self.__assert_stale(doc_tree, all_pages)
self.assertNotIn('source_a.test', all_pages['test-index'].subpages)
self.assertIn('source_a.test',
all_pages['test-section.markdown'].subpages)
def test_extension_override(self):
self.__create_md_file('source_a.test.markdown', (u'# My override\n'))
doc_tree, _ = self.__create_test_layout()
page = doc_tree.get_pages()['source_a.test']
self.assertEqual(page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
self.assertEqual(os.path.basename(page.source_file),
'source_a.test.markdown')
out, _ = cmark.ast_to_html(page.ast, None)
self.assertEqual(out, u'<h1>My override</h1>\n')
def test_parse_yaml(self):
inp = (u'index.markdown\n')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown',
(u'---\n'
'title: A random title\n'
'symbols: [symbol_1, symbol_2]\n'
'...\n'
'# My documentation\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
pages = doc_tree.get_pages()
page = pages.get('index.markdown')
out, _ = cmark.ast_to_html(page.ast, None)
self.assertEqual(out, u'<h1>My documentation</h1>\n')
self.assertEqual(page.title, u'A random title')
self.assertEqual(page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
def test_empty_link_resolution(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown', (u'# My section\n'
'\n'
'[](index.markdown)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents, u'<h1>My section</h1>\n'
'<p><a href="index.html">My documentation</a></p>\n')
def test_labeled_link_resolution(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown',
(u'# My section\n'
'\n'
'[a label](index.markdown)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents, u'<h1>My section</h1>\n'
'<p><a href="index.html">a label</a></p>\n')
def test_anchored_link_resolution(self):
inp = (u'index.markdown\n' '\tsection.markdown')
sitemap = self.__parse_sitemap(inp)
self.__create_md_file('index.markdown', (u'# My documentation\n'))
self.__create_md_file('section.markdown',
(u'# My section\n'
'\n'
'[](index.markdown#subsection)\n'))
doc_tree = DocTree(self.__priv_dir, self.include_paths)
doc_tree.parse_sitemap(self.change_tracker, sitemap)
doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
doc_tree.format(self.link_resolver, self.__output_dir,
{self.core_ext.extension_name: self.core_ext})
pages = doc_tree.get_pages()
page = pages.get('section.markdown')
self.assertEqual(
page.formatted_contents, u'<h1>My section</h1>\n'
'<p><a href="index.html#subsection">My documentation</a></p>\n')
# pylint: disable=too-many-statements
def test_extension_incremental(self):
doc_tree, sitemap = self.__create_test_layout()
doc_tree.persist()
# Here we touch source_a.test, as its symbols were
# all contained in a generated page, only that page
# should now be stale
self.__touch_src_file('source_a.test')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['source_a.test']))
doc_tree.persist()
# We now touch source_b.test, which symbols are contained
# both in a generated page and a user-provided one.
# We expect both pages to be stale
self.__touch_src_file('source_b.test')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['source_b.test',
'page_x.markdown']))
doc_tree.persist()
# This one is trickier: we unlist symbol_3 from
# page_x, which means the symbol should now be
# documented in the generated page for source_b.test.
# We expect both pages to be stale, and make sure
# they contain the right symbols
self.__create_md_file('page_x.markdown', (u'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet())
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
doc_tree.persist()
# Let's make sure the opposite use case works as well,
# we relocate symbol_3 in page_x , both page_x and
# the generated page for source_b.test should be stale
# and the symbols should be back to their original
# layout.
self.__create_md_file('page_x.markdown', (u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['source_b.test',
'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
doc_tree.persist()
# We now move the definition of symbol_3 to source_a.test,
# we thus expect the generated page for source_a.test to be
# stale because its source changed, same for source_b.test,
# and page_x.markdown should be stale as well because the
# definition of symbol_3 may have changed. The final
# symbol layout should not have changed however.
self.__create_src_file('source_a.test',
['symbol_1', 'symbol_2', 'symbol_3'])
self.__create_src_file('source_b.test', ['symbol_4'])
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(
doc_tree,
set(['source_a.test', 'source_b.test', 'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
source_a_page = doc_tree.get_pages()['source_a.test']
self.assertEqual(source_a_page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
doc_tree.persist()
# And we rollback again
self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2'])
self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4'])
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(
doc_tree,
set(['source_a.test', 'source_b.test', 'page_x.markdown']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
source_a_page = doc_tree.get_pages()['source_a.test']
self.assertEqual(source_a_page.symbol_names,
OrderedSet(['symbol_1', 'symbol_2']))
doc_tree.persist()
# Now we'll try removing page_x altogether
self.__remove_md_file('page_x.markdown')
inp = (u'index.markdown\n'
'\ttest-index\n'
'\t\ttest-section.markdown\n'
'\t\t\tsource_a.test\n'
'\t\tpage_y.markdown\n'
'\tcore_page.markdown\n')
new_sitemap = self.__parse_sitemap(inp)
doc_tree = self.__update_test_layout(doc_tree, new_sitemap)
self.__assert_stale(doc_tree, set(['source_b.test']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names,
OrderedSet(['symbol_4', 'symbol_3']))
doc_tree.persist()
# And rollback again
self.__create_md_file('page_x.markdown', (u'---\n'
'symbols: [symbol_3]\n'
'...\n'
'# Page X\n'))
doc_tree = self.__update_test_layout(doc_tree, sitemap)
self.__assert_stale(doc_tree, set(['page_x.markdown',
'source_b.test']))
page_x = doc_tree.get_pages()['page_x.markdown']
self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3']))
source_b_page = doc_tree.get_pages()['source_b.test']
self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4']))
doc_tree.persist()
def test_index_override_incremental(self):
doc_tree, sitemap = self.__create_test_layout()
doc_tree.persist()
index_page = doc_tree.get_pages()['test-index']
self.assertIn('source_b.test', index_page.subpages)
self.__touch_src_file('test-index.markdown')
doc_tree = self.__update_test_layout(doc_tree, sitemap)
index_page = doc_tree.get_pages()['test-index']
self.assertIn('source_b.test', index_page.subpages)
doc_tree.persist()
class DocRepo(object):
"""
Banana banana
"""
formatted_signal = Signal()
def __init__(self):
self.output = None
self.doc_tree = None
self.change_tracker = None
self.output_format = None
self.include_paths = None
self.extensions = {}
self.tag_validators = {}
self.link_resolver = None
self.incremental = False
self.doc_database = None
self.config = None
self.project_name = None
self.project_version = None
self.sitemap_path = None
if os.name == 'nt':
self.datadir = os.path.join(os.path.dirname(__file__), '..',
'share')
else:
self.datadir = "/usr/share"
self.__conf_file = None
self.__extension_classes = {
CoreExtension.extension_name: CoreExtension
}
self.__index_file = None
self.__root_page = None
self.__base_doc_folder = None
self.__private_folder = None
self.__dry = False
self.__load_extensions()
self.__create_arg_parser()
def register_tag_validator(self, validator):
"""
Banana banana
"""
self.tag_validators[validator.name] = validator
def format_symbol(self, symbol_name):
"""
Banana banana
"""
sym = self.doc_database.get_symbol(symbol_name)
if not sym:
return None
sym.update_children_comments()
return sym.detailed_description
def patch_page(self, symbol, raw_comment):
"""
Banana banana
"""
pages = self.doc_tree.get_pages_for_symbol(symbol.unique_name)
if not pages:
return False
old_comment = symbol.comment
# pylint: disable=no-member
new_comment = self.raw_comment_parser.parse_comment(
raw_comment, old_comment.filename, old_comment.lineno,
old_comment.lineno + raw_comment.count('\n'), self.include_paths)
if new_comment is None:
return False
if new_comment.name != symbol.unique_name:
return False
pages = pages.values()
symbol.comment = new_comment
for page in pages:
formatter = self.__get_formatter(page.extension_name)
formatter.patch_page(page, symbol, self.output)
return True
def persist(self):
"""
Banana banana
"""
if self.__dry:
return
info('Persisting database and private files', 'persisting')
self.doc_tree.persist()
self.doc_database.persist()
pickle.dump(
self.change_tracker,
open(os.path.join(self.get_private_folder(), 'change_tracker.p'),
'wb'))
self.__dump_deps_file()
def finalize(self):
"""
Banana banana
"""
self.formatted_signal.clear()
if self.doc_database is not None:
info('Closing database')
self.doc_database.close()
# pylint: disable=no-self-use
def get_private_folder(self):
"""
Banana banana
"""
return self.__private_folder
def setup(self):
"""
Banana banana
"""
configurable_classes = all_subclasses(Configurable)
configured = set()
for subclass in configurable_classes:
if subclass.parse_config not in configured:
subclass.parse_config(self, self.config)
configured.add(subclass.parse_config)
self.__parse_config()
self.doc_tree = DocTree(self.get_private_folder(), self.include_paths)
for extension in self.extensions.values():
info('Setting up %s' % extension.extension_name)
extension.setup()
self.doc_database.flush()
sitemap = SitemapParser().parse(self.sitemap_path)
self.doc_tree.parse_sitemap(self.change_tracker, sitemap)
info("Resolving symbols", 'resolution')
self.doc_tree.resolve_symbols(self.doc_database, self.link_resolver)
self.doc_database.flush()
def format(self):
"""
Banana banana
"""
if not self.output:
return
self.doc_tree.format(self.link_resolver, self.output, self.extensions)
self.config.dump(conf_file=os.path.join(self.output, 'hotdoc.json'))
self.formatted_signal(self)
def __dump_deps_file(self):
dest = self.config.get('deps_file_dest')
target = self.config.get('deps_file_target')
if dest is None:
info("Not dumping deps file")
return
info("Dumping deps file to %s with target %s" % (dest, target))
destdir = os.path.dirname(dest)
if not os.path.exists(destdir):
os.makedirs(destdir)
empty_targets = []
with io.open(dest, 'w', encoding='utf-8') as _:
_.write(u'%s: ' % target)
if self.config:
for dep in self.config.get_dependencies():
empty_targets.append(dep)
_.write(u'%s ' % dep)
if self.doc_tree:
for page in self.doc_tree.get_pages().values():
if not page.generated:
empty_targets.append(page.source_file)
_.write(u'%s ' % page.source_file)
for empty_target in empty_targets:
_.write(u'\n\n%s:' % empty_target)
def __add_default_tags(self, _, comment):
for validator in self.tag_validators.values():
if validator.default and validator.name not in comment.tags:
comment.tags[validator.name] = \
Tag(name=validator.name,
description=validator.default)
def __setup_database(self):
self.doc_database = DocDatabase()
self.doc_database.comment_added_signal.connect(self.__add_default_tags)
self.doc_database.comment_updated_signal.connect(
self.__add_default_tags)
self.doc_database.setup(self.get_private_folder())
self.link_resolver = LinkResolver(self.doc_database)
def __create_change_tracker(self):
try:
self.change_tracker = \
pickle.load(open(os.path.join(self.get_private_folder(),
'change_tracker.p'), 'rb'))
if self.change_tracker.hard_dependencies_are_stale():
raise IOError
self.incremental = True
info("Building incrementally")
# pylint: disable=broad-except
except Exception:
info("Building from scratch")
shutil.rmtree(self.get_private_folder(), ignore_errors=True)
if self.output:
shutil.rmtree(self.output, ignore_errors=True)
self.change_tracker = ChangeTracker()
def __get_formatter(self, extension_name):
"""
Banana banana
"""
ext = self.extensions.get(extension_name)
if ext:
return ext.get_formatter(self.output_format)
return None
def __check_initial_args(self, args):
if args.version:
print VERSION
elif args.makefile_path:
here = os.path.dirname(__file__)
path = os.path.join(here, '..', 'utils', 'hotdoc.mk')
print os.path.abspath(path)
elif args.get_conf_path:
key = args.get_conf_path
path = self.config.get_path(key, rel_to_cwd=True)
if path is not None:
print path
elif args.get_conf_key:
key = args.get_conf_key
value = self.config.get(args.get_conf_key, None)
if value is not None:
print value
elif args.get_private_folder:
print os.path.relpath(self.__private_folder,
self.config.get_invoke_dir())
elif args.has_extension:
ext_name = args.has_extension
print ext_name in self.__extension_classes
elif args.list_extensions:
for ext_name in self.__extension_classes:
print ext_name
else:
self.parser.print_usage()
def __create_arg_parser(self):
self.parser = \
argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,)
configurable_classes = all_subclasses(Configurable)
seen = set()
for subclass in configurable_classes:
if subclass.add_arguments not in seen:
subclass.add_arguments(self.parser)
seen.add(subclass.add_arguments)
self.parser.add_argument('command',
action="store",
choices=('run', 'conf', 'help'),
nargs="?")
self.parser.add_argument('--dry',
help='Dry run, nothing will be output',
dest='dry',
action='store_true')
self.parser.add_argument('--conf-file',
help='Path to the config file',
dest='conf_file')
self.parser.add_argument('--output-conf-file',
help='Path where to save the updated conf'
' file',
dest='output_conf_file')
self.parser.add_argument('--version',
help="Print version and exit",
action="store_true")
self.parser.add_argument('--makefile-path',
help="Print path to includable "
"Makefile and exit",
action="store_true")
self.parser.add_argument('--deps-file-dest',
help='Where to output the dependencies file')
self.parser.add_argument('--deps-file-target',
help='Name of the dependencies target',
default='doc.stamp.d')
self.parser.add_argument("-i",
"--index",
action="store",
dest="index",
help="location of the "
"index file")
self.parser.add_argument("--sitemap",
action="store",
dest="sitemap",
help="Location of the sitemap file")
self.parser.add_argument("--project-name",
action="store",
dest="project_name",
help="Name of the documented project")
self.parser.add_argument("--project-version",
action="store",
dest="project_version",
help="Version of the documented project")
self.parser.add_argument("-o",
"--output",
action="store",
dest="output",
help="where to output the rendered "
"documentation")
self.parser.add_argument("--get-conf-key",
action="store",
help="print the value for a configuration "
"key")
self.parser.add_argument("--get-conf-path",
action="store",
help="print the value for a configuration "
"path")
self.parser.add_argument("--get-private-folder",
action="store_true",
help="get the path to hotdoc's private "
"folder")
self.parser.add_argument("--output-format",
action="store",
dest="output_format",
help="format for the "
"output")
self.parser.add_argument("--has-extension",
action="store",
dest="has_extension",
help="Check if a given "
"extension is available")
self.parser.add_argument("--list-extensions",
action="store_true",
dest="list_extensions",
help="Print "
"available extensions")
self.parser.add_argument("-",
action="store_true",
help="Separator to allow finishing a list"
" of arguments before a command",
dest="whatever")
def load_command_line(self, args):
"""
Loads the repo from command line arguments
"""
args = self.parser.parse_args(args)
self.__load_config(args)
cmd = args.command
if cmd == 'help':
self.parser.print_help()
sys.exit(0)
elif cmd is None:
self.__check_initial_args(args)
sys.exit(0)
exit_now = False
save_config = False
if cmd == 'run':
self.__dry = bool(args.dry)
elif cmd == 'conf':
save_config = True
exit_now = True
if save_config:
self.config.dump(args.output_conf_file)
if exit_now:
sys.exit(0)
def load_conf_file(self, conf_file, overrides):
"""
Load the project from a configuration file and key-value
overides.
"""
if conf_file is None and os.path.exists('hotdoc.json'):
conf_file = 'hotdoc.json'
self.__conf_file = conf_file
if conf_file and not os.path.exists(conf_file):
error('invalid-config',
"No configuration file was found at %s" % conf_file)
actual_args = {}
defaults = {'output_format': 'html'}
for key, value in overrides.items():
if key in ('cmd', 'conf_file', 'dry'):
continue
if value != self.parser.get_default(key):
actual_args[key] = value
if self.parser.get_default(key) is not None:
defaults[key] = value
self.config = ConfigParser(command_line_args=actual_args,
conf_file=conf_file,
defaults=defaults)
index = self.config.get_index()
if index:
hash_obj = hashlib.md5(self.config.get_index())
priv_name = 'hotdoc-private-' + hash_obj.hexdigest()
else:
priv_name = 'hotdoc-private'
self.__private_folder = os.path.abspath(priv_name)
def __load_extensions(self):
extension_classes = get_all_extension_classes(sort=True)
for subclass in extension_classes:
self.__extension_classes[subclass.extension_name] = subclass
# pylint: disable=no-self-use
def __load_config(self, args):
"""
Banana banana
"""
cli = dict(vars(args))
self.load_conf_file(args.conf_file, cli)
def __setup_private_folder(self):
folder = self.get_private_folder()
if os.path.exists(folder):
if not os.path.isdir(folder):
error('setup-issue',
'%s exists but is not a directory' % folder)
else:
os.mkdir(folder)
def __create_extensions(self):
for ext_class in self.__extension_classes.values():
ext = ext_class(self)
self.extensions[ext.extension_name] = ext
def get_base_doc_folder(self):
"""Get the folder in which the main index was located
"""
return self.__base_doc_folder
def __parse_config(self):
"""
Banana banana
"""
output = self.config.get_path('output') or None
self.sitemap_path = self.config.get_path('sitemap')
if self.sitemap_path is None:
error('invalid-config', 'No sitemap was provided')
if output is not None:
self.output = os.path.abspath(output)
else:
self.output = None
self.project_name = self.config.get('project_name', None)
self.project_version = self.config.get('project_version', None)
self.output_format = self.config.get('output_format')
if self.output_format not in ["html"]:
error('invalid-config',
'Unsupported output format : %s' % self.output_format)
self.__index_file = self.config.get_index()
if self.__index_file is None:
error('invalid-config', 'index is required')
if not os.path.exists(self.__index_file):
error('invalid-config',
'The provided index "%s" does not exist' % self.__index_file)
cmd_line_includes = self.config.get_paths('include_paths')
self.__base_doc_folder = os.path.dirname(self.__index_file)
self.include_paths = OrderedSet([self.__base_doc_folder])
self.include_paths |= OrderedSet(cmd_line_includes)
self.__create_change_tracker()
self.__setup_private_folder()
self.__setup_database()
self.__create_extensions()
if self.__conf_file:
self.change_tracker.add_hard_dependency(self.__conf_file)
