class TestLinkResolver(unittest.TestCase): def setUp(self): here = os.path.dirname(__file__) self.__priv_dir = os.path.abspath(os.path.join(here, "tmp-private")) self.__remove_tmp_dirs() os.mkdir(self.__priv_dir) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) def __remove_tmp_dirs(self): shutil.rmtree(self.__priv_dir, ignore_errors=True) def test_incremental(self): param = ParameterSymbol(type_tokens=[Link(None, "test-struct", "test-struct")]) func = self.doc_database.get_or_create_symbol( FunctionSymbol, unique_name="test-symbol", filename="text_b.x", parameters=[param] ) func.resolve_links(self.link_resolver) self.assertEqual(param.get_type_link().get_link(), None) struct = self.doc_database.get_or_create_symbol(StructSymbol, unique_name="test-struct", filename="test_a.x") struct.resolve_links(self.link_resolver) func.resolve_links(self.link_resolver) # Not in a page but still self.assertEqual(param.get_type_link().get_link(), "test-struct") self.doc_database.persist() self.doc_database.close() self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) param = ParameterSymbol(type_tokens=[Link(None, "test-struct", "test-struct")]) func = self.doc_database.get_or_create_symbol( FunctionSymbol, unique_name="test-symbol", filename="text_b.x", parameters=[param] ) func.resolve_links(self.link_resolver) self.assertEqual(param.get_type_link().get_link(), "test-struct")
class DocRepo(object): """ Banana banana """ formatted_signal = Signal() def __init__(self): self.output = None self.doc_tree = None self.change_tracker = None self.output_format = None self.include_paths = None self.extensions = {} self.tag_validators = {} self.link_resolver = None self.incremental = False self.doc_database = None self.config = None self.project_name = None self.project_version = None self.sitemap_path = None if os.name == 'nt': self.datadir = os.path.join( os.path.dirname(__file__), '..', 'share') else: self.datadir = "/usr/share" self.__conf_file = None self.__extension_classes = { CoreExtension.extension_name: CoreExtension} self.__index_file = None self.__root_page = None self.__base_doc_folder = None self.__private_folder = None self.__dry = False self.__load_extensions() self.__create_arg_parser() def register_tag_validator(self, validator): """ Banana banana """ self.tag_validators[validator.name] = validator def format_symbol(self, symbol_name): """ Banana banana """ sym = self.doc_database.get_symbol(symbol_name) if not sym: return None sym.update_children_comments() return sym.detailed_description def patch_page(self, symbol, raw_comment): """ Banana banana """ pages = self.doc_tree.get_pages_for_symbol(symbol.unique_name) if not pages: return False old_comment = symbol.comment # pylint: disable=no-member new_comment = self.raw_comment_parser.parse_comment( raw_comment, old_comment.filename, old_comment.lineno, old_comment.lineno + raw_comment.count('\n'), self.include_paths) if new_comment is None: return False if new_comment.name != symbol.unique_name: return False pages = pages.values() symbol.comment = new_comment for page in pages: formatter = self.__get_formatter(page.extension_name) formatter.patch_page(page, symbol, self.output) return True def persist(self): """ Banana banana """ if self.__dry: return info('Persisting database and private files', 'persisting') self.doc_tree.persist() self.doc_database.persist() pickle.dump(self.change_tracker, open(os.path.join(self.get_private_folder(), 'change_tracker.p'), 'wb')) self.__dump_deps_file() def finalize(self): """ Banana banana """ self.formatted_signal.clear() if self.doc_database is not None: info('Closing database') self.doc_database.close() # pylint: disable=no-self-use def get_private_folder(self): """ Banana banana """ return self.__private_folder def setup(self): """ Banana banana """ configurable_classes = all_subclasses(Configurable) configured = set() for subclass in configurable_classes: if subclass.parse_config not in configured: subclass.parse_config(self, self.config) configured.add(subclass.parse_config) self.__parse_config() self.doc_tree = DocTree(self.get_private_folder(), self.include_paths) for extension in self.extensions.values(): info('Setting up %s' % extension.extension_name) extension.setup() self.doc_database.flush() sitemap = SitemapParser().parse(self.sitemap_path) self.doc_tree.parse_sitemap(self.change_tracker, sitemap) info("Resolving symbols", 'resolution') self.doc_tree.resolve_symbols(self.doc_database, self.link_resolver) self.doc_database.flush() def format(self): """ Banana banana """ if not self.output: return self.doc_tree.format(self.link_resolver, self.output, self.extensions) self.config.dump(conf_file=os.path.join(self.output, 'hotdoc.json')) self.formatted_signal(self) def __dump_deps_file(self): dest = self.config.get('deps_file_dest') target = self.config.get('deps_file_target') if dest is None: info("Not dumping deps file") return info("Dumping deps file to %s with target %s" % (dest, target)) destdir = os.path.dirname(dest) if not os.path.exists(destdir): os.makedirs(destdir) empty_targets = [] with io.open(dest, 'w', encoding='utf-8') as _: _.write(u'%s: ' % target) if self.config: for dep in self.config.get_dependencies(): empty_targets.append(dep) _.write(u'%s ' % dep) if self.doc_tree: for page in self.doc_tree.get_pages().values(): if not page.generated: empty_targets.append(page.source_file) _.write(u'%s ' % page.source_file) for empty_target in empty_targets: _.write(u'\n\n%s:' % empty_target) def __add_default_tags(self, _, comment): for validator in self.tag_validators.values(): if validator.default and validator.name not in comment.tags: comment.tags[validator.name] = \ Tag(name=validator.name, description=validator.default) def __setup_database(self): self.doc_database = DocDatabase() self.doc_database.comment_added_signal.connect(self.__add_default_tags) self.doc_database.comment_updated_signal.connect( self.__add_default_tags) self.doc_database.setup(self.get_private_folder()) self.link_resolver = LinkResolver(self.doc_database) def __create_change_tracker(self): try: self.change_tracker = \ pickle.load(open(os.path.join(self.get_private_folder(), 'change_tracker.p'), 'rb')) if self.change_tracker.hard_dependencies_are_stale(): raise IOError self.incremental = True info("Building incrementally") # pylint: disable=broad-except except Exception: info("Building from scratch") shutil.rmtree(self.get_private_folder(), ignore_errors=True) if self.output: shutil.rmtree(self.output, ignore_errors=True) self.change_tracker = ChangeTracker() def __get_formatter(self, extension_name): """ Banana banana """ ext = self.extensions.get(extension_name) if ext: return ext.get_formatter(self.output_format) return None def __check_initial_args(self, args): if args.version: print VERSION elif args.makefile_path: here = os.path.dirname(__file__) path = os.path.join(here, '..', 'utils', 'hotdoc.mk') print os.path.abspath(path) elif args.get_conf_path: key = args.get_conf_path path = self.config.get_path(key, rel_to_cwd=True) if path is not None: print path elif args.get_conf_key: key = args.get_conf_key value = self.config.get(args.get_conf_key, None) if value is not None: print value elif args.get_private_folder: print os.path.relpath(self.__private_folder, self.config.get_invoke_dir()) elif args.has_extension: ext_name = args.has_extension print ext_name in self.__extension_classes elif args.list_extensions: for ext_name in self.__extension_classes: print ext_name else: self.parser.print_usage() def __create_arg_parser(self): self.parser = \ argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter,) configurable_classes = all_subclasses(Configurable) seen = set() for subclass in configurable_classes: if subclass.add_arguments not in seen: subclass.add_arguments(self.parser) seen.add(subclass.add_arguments) self.parser.add_argument('command', action="store", choices=('run', 'conf', 'help'), nargs="?") self.parser.add_argument('--dry', help='Dry run, nothing will be output', dest='dry', action='store_true') self.parser.add_argument('--conf-file', help='Path to the config file', dest='conf_file') self.parser.add_argument('--output-conf-file', help='Path where to save the updated conf' ' file', dest='output_conf_file') self.parser.add_argument('--version', help="Print version and exit", action="store_true") self.parser.add_argument('--makefile-path', help="Print path to includable " "Makefile and exit", action="store_true") self.parser.add_argument('--deps-file-dest', help='Where to output the dependencies file') self.parser.add_argument('--deps-file-target', help='Name of the dependencies target', default='doc.stamp.d') self.parser.add_argument("-i", "--index", action="store", dest="index", help="location of the " "index file") self.parser.add_argument("--sitemap", action="store", dest="sitemap", help="Location of the sitemap file") self.parser.add_argument("--project-name", action="store", dest="project_name", help="Name of the documented project") self.parser.add_argument("--project-version", action="store", dest="project_version", help="Version of the documented project") self.parser.add_argument("-o", "--output", action="store", dest="output", help="where to output the rendered " "documentation") self.parser.add_argument("--extra-extensions-paths", action="append", dest="extra_extension_paths", default=[], help="Extra paths to lookup extensions in") self.parser.add_argument("--get-conf-key", action="store", help="print the value for a configuration " "key") self.parser.add_argument("--get-conf-path", action="store", help="print the value for a configuration " "path") self.parser.add_argument("--get-private-folder", action="store_true", help="get the path to hotdoc's private " "folder") self.parser.add_argument("--output-format", action="store", dest="output_format", help="format for the " "output") self.parser.add_argument("--has-extension", action="store", dest="has_extension", help="Check if a given " "extension is available") self.parser.add_argument("--list-extensions", action="store_true", dest="list_extensions", help="Print " "available extensions") self.parser.add_argument('--include-paths', help='paths to look up included files in', dest='include_paths', action='append', default=[]) self.parser.add_argument("-", action="store_true", help="Separator to allow finishing a list" " of arguments before a command", dest="whatever") def load_command_line(self, args): """ Loads the repo from command line arguments """ args = self.parser.parse_args(args) self.__load_config(args) cmd = args.command if cmd == 'help': self.parser.print_help() sys.exit(0) elif cmd is None: self.__check_initial_args(args) sys.exit(0) exit_now = False save_config = False if cmd == 'run': self.__dry = bool(args.dry) elif cmd == 'conf': save_config = True exit_now = True if save_config: self.config.dump(args.output_conf_file) if exit_now: sys.exit(0) def load_conf_file(self, conf_file, overrides): """ Load the project from a configuration file and key-value overides. """ if conf_file is None and os.path.exists('hotdoc.json'): conf_file = 'hotdoc.json' self.__conf_file = conf_file if conf_file and not os.path.exists(conf_file): error('invalid-config', "No configuration file was found at %s" % conf_file) actual_args = {} defaults = {'output_format': 'html'} for key, value in overrides.items(): if key in ('cmd', 'conf_file', 'dry'): continue if value != self.parser.get_default(key): actual_args[key] = value if self.parser.get_default(key) is not None: defaults[key] = value self.config = ConfigParser(command_line_args=actual_args, conf_file=conf_file, defaults=defaults) index = self.config.get_index() if index: hash_obj = hashlib.md5(self.config.get_index()) priv_name = 'hotdoc-private-' + hash_obj.hexdigest() else: priv_name = 'hotdoc-private' self.__private_folder = os.path.abspath(priv_name) def __load_extensions(self): extension_classes = get_installed_extension_classes(True) for subclass in extension_classes: self.__extension_classes[subclass.extension_name] = subclass # pylint: disable=no-self-use def __load_config(self, args): """ Banana banana """ cli = dict(vars(args)) self.load_conf_file(args.conf_file, cli) def __setup_private_folder(self): folder = self.get_private_folder() if os.path.exists(folder): if not os.path.isdir(folder): error('setup-issue', '%s exists but is not a directory' % folder) else: os.mkdir(folder) def __create_extensions(self, extra_paths): for ext_class in self.__extension_classes.values(): ext = ext_class(self) self.extensions[ext.extension_name] = ext extra_classes = get_extra_extension_classes(extra_paths) self.__extension_classes.update(extra_classes) for ext_class in extra_classes.values(): ext = ext_class(self) self.extensions[ext.extension_name] = ext def get_base_doc_folder(self): """Get the folder in which the main index was located """ return self.__base_doc_folder def __parse_config(self): """ Banana banana """ output = self.config.get_path('output') or None self.sitemap_path = self.config.get_path('sitemap') if self.sitemap_path is None: error('invalid-config', 'No sitemap was provided') if output is not None: self.output = os.path.abspath(output) else: self.output = None self.project_name = self.config.get('project_name', None) self.project_version = self.config.get('project_version', None) self.output_format = self.config.get('output_format') if self.output_format not in ["html"]: error('invalid-config', 'Unsupported output format : %s' % self.output_format) self.__index_file = self.config.get_index() if self.__index_file is None: error('invalid-config', 'index is required') if not os.path.exists(self.__index_file): error('invalid-config', 'The provided index "%s" does not exist' % self.__index_file) cmd_line_includes = self.config.get_paths('include_paths') self.__base_doc_folder = os.path.dirname(self.__index_file) self.include_paths = OrderedSet([self.__base_doc_folder]) self.include_paths |= OrderedSet(cmd_line_includes) self.__create_change_tracker() self.__setup_private_folder() self.__setup_database() self.__create_extensions( self.config.get_paths('extra_extension_paths')) if self.__conf_file: self.change_tracker.add_hard_dependency(self.__conf_file)
class TestDocTree(unittest.TestCase): def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath(os.path.join( here, 'tmp-markdown-files')) self.__priv_dir = os.path.abspath(os.path.join( here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join( here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join( here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.__priv_dir) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.test_ext = TestExtension(self) self.core_ext = CoreExtension(self) def tearDown(self): self.__remove_tmp_dirs() del self.test_ext del self.core_ext def get_generated_doc_folder(self): return os.path.join(self.__priv_dir, 'generated') def get_base_doc_folder(self): return self.__md_dir def get_private_folder(self): return self.__priv_dir def __parse_sitemap(self, text): path = os.path.join(self.__md_dir, 'sitemap.txt') with io.open(path, 'w', encoding='utf-8') as _: _.write(text) return self.sitemap_parser.parse(path) def __create_md_file(self, name, contents): path = os.path.join(self.__md_dir, name) with open(path, 'w') as _: _.write(contents) # Just making sure we don't hit a race condition, # in real world situations it is assumed users # will not update source files twice in the same # microsecond touch(path) def __create_src_file(self, name, symbols): path = os.path.join(self.__md_dir, name) with open(path, 'w') as _: for symbol in symbols: _.write('%s\n' % symbol) # Just making sure we don't hit a race condition, # in real world situations it is assumed users # will not update source files twice in the same # microsecond touch(path) return path def __remove_src_file(self, name): path = os.path.join(self.__md_dir, name) os.unlink(path) def __remove_md_file(self, name): path = os.path.join(self.__md_dir, name) os.unlink(path) def __touch_src_file(self, name): path = os.path.join(self.__md_dir, name) touch(path) def __remove_tmp_dirs(self): shutil.rmtree(self.__md_dir, ignore_errors=True) shutil.rmtree(self.__priv_dir, ignore_errors=True) shutil.rmtree(self.__src_dir, ignore_errors=True) shutil.rmtree(self.__output_dir, ignore_errors=True) shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True) def test_basic(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) pages = doc_tree.get_pages() # We do not care about ordering self.assertSetEqual( set(pages.keys()), set([u'index.markdown', u'section.markdown'])) index = pages.get('index.markdown') self.assertEqual(index.title, u'My documentation') def test_basic_incremental(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) # Building from scratch, all pages are stale self.assertSetEqual( set(doc_tree.get_stale_pages()), set([u'index.markdown', u'section.markdown'])) doc_tree.persist() doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) # Nothing changed, no page is stale self.assertSetEqual( set(doc_tree.get_stale_pages()), set({})) # But we still have our pages self.assertSetEqual( set(doc_tree.get_pages()), set([u'index.markdown', u'section.markdown'])) touch(os.path.join(self.__md_dir, u'section.markdown')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) self.assertSetEqual( set(doc_tree.get_stale_pages()), set([u'section.markdown'])) def __assert_extension_names(self, doc_tree, name_map): pages = doc_tree.get_pages() for name, ext_name in name_map.items(): page = pages[name] self.assertEqual(ext_name, page.extension_name) def __assert_stale(self, doc_tree, expected_stale): stale_pages = doc_tree.get_stale_pages() for pagename in expected_stale: self.assertIn(pagename, stale_pages) stale_pages.pop(pagename) self.assertEqual(len(stale_pages), 0) def __create_test_layout(self): inp = (u'index.markdown\n' '\ttest-index\n' '\t\ttest-section.markdown\n' '\t\t\tsource_a.test\n' '\t\tpage_x.markdown\n' '\t\tpage_y.markdown\n' '\tcore_page.markdown\n') sources = [] sources.append(self.__create_src_file( 'source_a.test', ['symbol_1', 'symbol_2'])) sources.append(self.__create_src_file( 'source_b.test', ['symbol_3', 'symbol_4'])) self.test_ext.index = 'test-index.markdown' self.test_ext.sources = sources self.test_ext.setup() sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'core_page.markdown', (u'# My non-extension page\n')) self.__create_md_file( 'test-index.markdown', (u'# My test index\n')) self.__create_md_file( 'test-section.markdown', (u'# My test section\n' '\n' 'Linking to [a generated page](source_a.test)\n')) self.__create_md_file( 'page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) self.__create_md_file( 'page_y.markdown', (u'# Page Y\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) return doc_tree, sitemap def __update_test_layout(self, doc_tree, sitemap): self.test_ext.reset() self.test_ext.setup() doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) return doc_tree def test_extension_basic(self): doc_tree, _ = self.__create_test_layout() self.__assert_extension_names( doc_tree, {u'index.markdown': 'core', u'test-index': 'test-extension', u'test-section.markdown': 'test-extension', u'source_a.test': 'test-extension', u'source_b.test': 'test-extension', u'page_x.markdown': 'test-extension', u'page_y.markdown': 'test-extension', u'core_page.markdown': 'core'}) all_pages = doc_tree.get_pages() self.assertEqual(len(all_pages), 8) self.__assert_stale(doc_tree, all_pages) self.assertNotIn('source_a.test', all_pages['test-index'].subpages) self.assertIn('source_a.test', all_pages['test-section.markdown'].subpages) def test_extension_override(self): self.__create_md_file( 'source_a.test.markdown', (u'# My override\n')) doc_tree, _ = self.__create_test_layout() page = doc_tree.get_pages()['source_a.test'] self.assertEqual( page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) self.assertEqual( os.path.basename(page.source_file), 'source_a.test.markdown') self.assertEqual( cmark.ast_to_html(page.ast, None), u'<h1>My override</h1>\n') def test_parse_yaml(self): inp = (u'index.markdown\n') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'---\n' 'title: A random title\n' 'symbols: [symbol_1, symbol_2]\n' '...\n' '# My documentation\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) pages = doc_tree.get_pages() page = pages.get('index.markdown') self.assertEqual( cmark.ast_to_html(page.ast, None), u'<h1>My documentation</h1>\n') self.assertEqual(page.title, u'A random title') self.assertEqual( page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) def test_empty_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n' '\n' '[](index.markdown)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format( self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html">My documentation</a></p>\n') def test_labeled_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n' '\n' '[a label](index.markdown)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format( self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html">a label</a></p>\n') def test_anchored_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file( 'index.markdown', (u'# My documentation\n')) self.__create_md_file( 'section.markdown', (u'# My section\n' '\n' '[](index.markdown#subsection)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format( self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html#subsection">My documentation</a></p>\n') # pylint: disable=too-many-statements def test_extension_incremental(self): doc_tree, sitemap = self.__create_test_layout() doc_tree.persist() # Here we touch source_a.test, as its symbols were # all contained in a generated page, only that page # should now be stale self.__touch_src_file('source_a.test') doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test'])) doc_tree.persist() # We now touch source_b.test, which symbols are contained # both in a generated page and a user-provided one. # We expect both pages to be stale self.__touch_src_file('source_b.test') doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) doc_tree.persist() # This one is trickier: we unlist symbol_3 from # page_x, which means the symbol should now be # documented in the generated page for source_b.test. # We expect both pages to be stale, and make sure # they contain the right symbols self.__create_md_file( 'page_x.markdown', (u'# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet()) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4', 'symbol_3'])) doc_tree.persist() # Let's make sure the opposite use case works as well, # we relocate symbol_3 in page_x , both page_x and # the generated page for source_b.test should be stale # and the symbols should be back to their original # layout. self.__create_md_file( 'page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) doc_tree.persist() # We now move the definition of symbol_3 to source_a.test, # we thus expect the generated page for source_a.test to be # stale because its source changed, same for source_b.test, # and page_x.markdown should be stale as well because the # definition of symbol_3 may have changed. The final # symbol layout should not have changed however. self.__create_src_file( 'source_a.test', ['symbol_1', 'symbol_2', 'symbol_3']) self.__create_src_file( 'source_b.test', ['symbol_4']) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test', 'source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) source_a_page = doc_tree.get_pages()['source_a.test'] self.assertEqual( source_a_page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) doc_tree.persist() # And we rollback again self.__create_src_file( 'source_a.test', ['symbol_1', 'symbol_2']) self.__create_src_file( 'source_b.test', ['symbol_3', 'symbol_4']) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test', 'source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) source_a_page = doc_tree.get_pages()['source_a.test'] self.assertEqual( source_a_page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) doc_tree.persist() # Now we'll try removing page_x altogether self.__remove_md_file('page_x.markdown') inp = (u'index.markdown\n' '\ttest-index\n' '\t\ttest-section.markdown\n' '\t\t\tsource_a.test\n' '\t\tpage_y.markdown\n' '\tcore_page.markdown\n') new_sitemap = self.__parse_sitemap(inp) doc_tree = self.__update_test_layout(doc_tree, new_sitemap) self.__assert_stale(doc_tree, set(['source_b.test'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4', 'symbol_3'])) doc_tree.persist() # And rollback again self.__create_md_file( 'page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['page_x.markdown', 'source_b.test'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual( source_b_page.symbol_names, OrderedSet(['symbol_4'])) doc_tree.persist()
class TestDocTree(unittest.TestCase): def setUp(self): here = os.path.dirname(__file__) self.__md_dir = os.path.abspath( os.path.join(here, 'tmp-markdown-files')) self.__priv_dir = os.path.abspath(os.path.join(here, 'tmp-private')) self.__src_dir = os.path.abspath(os.path.join(here, 'tmp-src-files')) self.__output_dir = os.path.abspath(os.path.join(here, 'tmp-output')) self.__remove_tmp_dirs() os.mkdir(self.__md_dir) os.mkdir(self.__priv_dir) os.mkdir(self.__src_dir) os.mkdir(self.get_generated_doc_folder()) self.include_paths = OrderedSet([self.__md_dir]) self.include_paths.add(self.get_generated_doc_folder()) # Using the real doc database is too costly, tests should be lightning # fast (and they are) self.doc_database = DocDatabase() self.doc_database.setup(self.__priv_dir) self.link_resolver = LinkResolver(self.doc_database) self.change_tracker = ChangeTracker() self.sitemap_parser = SitemapParser() self.test_ext = TestExtension(self) self.core_ext = CoreExtension(self) def tearDown(self): self.__remove_tmp_dirs() del self.test_ext del self.core_ext def get_generated_doc_folder(self): return os.path.join(self.__priv_dir, 'generated') def get_base_doc_folder(self): return self.__md_dir def get_private_folder(self): return self.__priv_dir def __parse_sitemap(self, text): path = os.path.join(self.__md_dir, 'sitemap.txt') with io.open(path, 'w', encoding='utf-8') as _: _.write(text) return self.sitemap_parser.parse(path) def __create_md_file(self, name, contents): path = os.path.join(self.__md_dir, name) with open(path, 'w') as _: _.write(contents) # Just making sure we don't hit a race condition, # in real world situations it is assumed users # will not update source files twice in the same # microsecond touch(path) def __create_src_file(self, name, symbols): path = os.path.join(self.__md_dir, name) with open(path, 'w') as _: for symbol in symbols: _.write('%s\n' % symbol) # Just making sure we don't hit a race condition, # in real world situations it is assumed users # will not update source files twice in the same # microsecond touch(path) return path def __remove_src_file(self, name): path = os.path.join(self.__md_dir, name) os.unlink(path) def __remove_md_file(self, name): path = os.path.join(self.__md_dir, name) os.unlink(path) def __touch_src_file(self, name): path = os.path.join(self.__md_dir, name) touch(path) def __remove_tmp_dirs(self): shutil.rmtree(self.__md_dir, ignore_errors=True) shutil.rmtree(self.__priv_dir, ignore_errors=True) shutil.rmtree(self.__src_dir, ignore_errors=True) shutil.rmtree(self.__output_dir, ignore_errors=True) shutil.rmtree(self.get_generated_doc_folder(), ignore_errors=True) def test_basic(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'# My documentation\n')) self.__create_md_file('section.markdown', (u'# My section\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) pages = doc_tree.get_pages() # We do not care about ordering self.assertSetEqual(set(pages.keys()), set([u'index.markdown', u'section.markdown'])) index = pages.get('index.markdown') self.assertEqual(index.title, u'My documentation') def test_basic_incremental(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'# My documentation\n')) self.__create_md_file('section.markdown', (u'# My section\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) # Building from scratch, all pages are stale self.assertSetEqual(set(doc_tree.get_stale_pages()), set([u'index.markdown', u'section.markdown'])) doc_tree.persist() doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) # Nothing changed, no page is stale self.assertSetEqual(set(doc_tree.get_stale_pages()), set({})) # But we still have our pages self.assertSetEqual(set(doc_tree.get_pages()), set([u'index.markdown', u'section.markdown'])) touch(os.path.join(self.__md_dir, u'section.markdown')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) self.assertSetEqual(set(doc_tree.get_stale_pages()), set([u'section.markdown'])) def __assert_extension_names(self, doc_tree, name_map): pages = doc_tree.get_pages() for name, ext_name in name_map.items(): page = pages[name] self.assertEqual(ext_name, page.extension_name) def __assert_stale(self, doc_tree, expected_stale): stale_pages = doc_tree.get_stale_pages() for pagename in expected_stale: self.assertIn(pagename, stale_pages) stale_pages.pop(pagename) self.assertEqual(len(stale_pages), 0) def __create_test_layout(self): inp = (u'index.markdown\n' '\ttest-index\n' '\t\ttest-section.markdown\n' '\t\t\tsource_a.test\n' '\t\tpage_x.markdown\n' '\t\tpage_y.markdown\n' '\tcore_page.markdown\n') sources = [] sources.append( self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2'])) sources.append( self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4'])) self.test_ext.index = 'test-index.markdown' self.test_ext.sources = sources self.test_ext.setup() sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'# My documentation\n')) self.__create_md_file('core_page.markdown', (u'# My non-extension page\n')) self.__create_md_file('test-index.markdown', (u'# My test index\n')) self.__create_md_file( 'test-section.markdown', (u'# My test section\n' '\n' 'Linking to [a generated page](source_a.test)\n')) self.__create_md_file('page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) self.__create_md_file('page_y.markdown', (u'# Page Y\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) return doc_tree, sitemap def __update_test_layout(self, doc_tree, sitemap): self.test_ext.reset() self.test_ext.setup() doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) return doc_tree def test_extension_basic(self): doc_tree, _ = self.__create_test_layout() self.__assert_extension_names( doc_tree, { u'index.markdown': 'core', u'test-index': 'test-extension', u'test-section.markdown': 'test-extension', u'source_a.test': 'test-extension', u'source_b.test': 'test-extension', u'page_x.markdown': 'test-extension', u'page_y.markdown': 'test-extension', u'core_page.markdown': 'core' }) all_pages = doc_tree.get_pages() self.assertEqual(len(all_pages), 8) self.__assert_stale(doc_tree, all_pages) self.assertNotIn('source_a.test', all_pages['test-index'].subpages) self.assertIn('source_a.test', all_pages['test-section.markdown'].subpages) def test_extension_override(self): self.__create_md_file('source_a.test.markdown', (u'# My override\n')) doc_tree, _ = self.__create_test_layout() page = doc_tree.get_pages()['source_a.test'] self.assertEqual(page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) self.assertEqual(os.path.basename(page.source_file), 'source_a.test.markdown') out, _ = cmark.ast_to_html(page.ast, None) self.assertEqual(out, u'<h1>My override</h1>\n') def test_parse_yaml(self): inp = (u'index.markdown\n') sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'---\n' 'title: A random title\n' 'symbols: [symbol_1, symbol_2]\n' '...\n' '# My documentation\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) pages = doc_tree.get_pages() page = pages.get('index.markdown') out, _ = cmark.ast_to_html(page.ast, None) self.assertEqual(out, u'<h1>My documentation</h1>\n') self.assertEqual(page.title, u'A random title') self.assertEqual(page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) def test_empty_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'# My documentation\n')) self.__create_md_file('section.markdown', (u'# My section\n' '\n' '[](index.markdown)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format(self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html">My documentation</a></p>\n') def test_labeled_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'# My documentation\n')) self.__create_md_file('section.markdown', (u'# My section\n' '\n' '[a label](index.markdown)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format(self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html">a label</a></p>\n') def test_anchored_link_resolution(self): inp = (u'index.markdown\n' '\tsection.markdown') sitemap = self.__parse_sitemap(inp) self.__create_md_file('index.markdown', (u'# My documentation\n')) self.__create_md_file('section.markdown', (u'# My section\n' '\n' '[](index.markdown#subsection)\n')) doc_tree = DocTree(self.__priv_dir, self.include_paths) doc_tree.parse_sitemap(self.change_tracker, sitemap) doc_tree.resolve_symbols(self.doc_database, self.link_resolver) doc_tree.format(self.link_resolver, self.__output_dir, {self.core_ext.extension_name: self.core_ext}) pages = doc_tree.get_pages() page = pages.get('section.markdown') self.assertEqual( page.formatted_contents, u'<h1>My section</h1>\n' '<p><a href="index.html#subsection">My documentation</a></p>\n') # pylint: disable=too-many-statements def test_extension_incremental(self): doc_tree, sitemap = self.__create_test_layout() doc_tree.persist() # Here we touch source_a.test, as its symbols were # all contained in a generated page, only that page # should now be stale self.__touch_src_file('source_a.test') doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_a.test'])) doc_tree.persist() # We now touch source_b.test, which symbols are contained # both in a generated page and a user-provided one. # We expect both pages to be stale self.__touch_src_file('source_b.test') doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) doc_tree.persist() # This one is trickier: we unlist symbol_3 from # page_x, which means the symbol should now be # documented in the generated page for source_b.test. # We expect both pages to be stale, and make sure # they contain the right symbols self.__create_md_file('page_x.markdown', (u'# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet()) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4', 'symbol_3'])) doc_tree.persist() # Let's make sure the opposite use case works as well, # we relocate symbol_3 in page_x , both page_x and # the generated page for source_b.test should be stale # and the symbols should be back to their original # layout. self.__create_md_file('page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4'])) doc_tree.persist() # We now move the definition of symbol_3 to source_a.test, # we thus expect the generated page for source_a.test to be # stale because its source changed, same for source_b.test, # and page_x.markdown should be stale as well because the # definition of symbol_3 may have changed. The final # symbol layout should not have changed however. self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2', 'symbol_3']) self.__create_src_file('source_b.test', ['symbol_4']) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale( doc_tree, set(['source_a.test', 'source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4'])) source_a_page = doc_tree.get_pages()['source_a.test'] self.assertEqual(source_a_page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) doc_tree.persist() # And we rollback again self.__create_src_file('source_a.test', ['symbol_1', 'symbol_2']) self.__create_src_file('source_b.test', ['symbol_3', 'symbol_4']) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale( doc_tree, set(['source_a.test', 'source_b.test', 'page_x.markdown'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4'])) source_a_page = doc_tree.get_pages()['source_a.test'] self.assertEqual(source_a_page.symbol_names, OrderedSet(['symbol_1', 'symbol_2'])) doc_tree.persist() # Now we'll try removing page_x altogether self.__remove_md_file('page_x.markdown') inp = (u'index.markdown\n' '\ttest-index\n' '\t\ttest-section.markdown\n' '\t\t\tsource_a.test\n' '\t\tpage_y.markdown\n' '\tcore_page.markdown\n') new_sitemap = self.__parse_sitemap(inp) doc_tree = self.__update_test_layout(doc_tree, new_sitemap) self.__assert_stale(doc_tree, set(['source_b.test'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4', 'symbol_3'])) doc_tree.persist() # And rollback again self.__create_md_file('page_x.markdown', (u'---\n' 'symbols: [symbol_3]\n' '...\n' '# Page X\n')) doc_tree = self.__update_test_layout(doc_tree, sitemap) self.__assert_stale(doc_tree, set(['page_x.markdown', 'source_b.test'])) page_x = doc_tree.get_pages()['page_x.markdown'] self.assertEqual(page_x.symbol_names, OrderedSet(['symbol_3'])) source_b_page = doc_tree.get_pages()['source_b.test'] self.assertEqual(source_b_page.symbol_names, OrderedSet(['symbol_4'])) doc_tree.persist() def test_index_override_incremental(self): doc_tree, sitemap = self.__create_test_layout() doc_tree.persist() index_page = doc_tree.get_pages()['test-index'] self.assertIn('source_b.test', index_page.subpages) self.__touch_src_file('test-index.markdown') doc_tree = self.__update_test_layout(doc_tree, sitemap) index_page = doc_tree.get_pages()['test-index'] self.assertIn('source_b.test', index_page.subpages) doc_tree.persist()
class DocRepo(object): """ Banana banana """ formatted_signal = Signal() def __init__(self): self.output = None self.doc_tree = None self.change_tracker = None self.output_format = None self.include_paths = None self.extensions = {} self.tag_validators = {} self.link_resolver = None self.incremental = False self.doc_database = None self.config = None self.project_name = None self.project_version = None self.sitemap_path = None if os.name == 'nt': self.datadir = os.path.join(os.path.dirname(__file__), '..', 'share') else: self.datadir = "/usr/share" self.__conf_file = None self.__extension_classes = { CoreExtension.extension_name: CoreExtension } self.__index_file = None self.__root_page = None self.__base_doc_folder = None self.__private_folder = None self.__dry = False self.__load_extensions() self.__create_arg_parser() def register_tag_validator(self, validator): """ Banana banana """ self.tag_validators[validator.name] = validator def format_symbol(self, symbol_name): """ Banana banana """ sym = self.doc_database.get_symbol(symbol_name) if not sym: return None sym.update_children_comments() return sym.detailed_description def patch_page(self, symbol, raw_comment): """ Banana banana """ pages = self.doc_tree.get_pages_for_symbol(symbol.unique_name) if not pages: return False old_comment = symbol.comment # pylint: disable=no-member new_comment = self.raw_comment_parser.parse_comment( raw_comment, old_comment.filename, old_comment.lineno, old_comment.lineno + raw_comment.count('\n'), self.include_paths) if new_comment is None: return False if new_comment.name != symbol.unique_name: return False pages = pages.values() symbol.comment = new_comment for page in pages: formatter = self.__get_formatter(page.extension_name) formatter.patch_page(page, symbol, self.output) return True def persist(self): """ Banana banana """ if self.__dry: return info('Persisting database and private files', 'persisting') self.doc_tree.persist() self.doc_database.persist() pickle.dump( self.change_tracker, open(os.path.join(self.get_private_folder(), 'change_tracker.p'), 'wb')) self.__dump_deps_file() def finalize(self): """ Banana banana """ self.formatted_signal.clear() if self.doc_database is not None: info('Closing database') self.doc_database.close() # pylint: disable=no-self-use def get_private_folder(self): """ Banana banana """ return self.__private_folder def setup(self): """ Banana banana """ configurable_classes = all_subclasses(Configurable) configured = set() for subclass in configurable_classes: if subclass.parse_config not in configured: subclass.parse_config(self, self.config) configured.add(subclass.parse_config) self.__parse_config() self.doc_tree = DocTree(self.get_private_folder(), self.include_paths) for extension in self.extensions.values(): info('Setting up %s' % extension.extension_name) extension.setup() self.doc_database.flush() sitemap = SitemapParser().parse(self.sitemap_path) self.doc_tree.parse_sitemap(self.change_tracker, sitemap) info("Resolving symbols", 'resolution') self.doc_tree.resolve_symbols(self.doc_database, self.link_resolver) self.doc_database.flush() def format(self): """ Banana banana """ if not self.output: return self.doc_tree.format(self.link_resolver, self.output, self.extensions) self.config.dump(conf_file=os.path.join(self.output, 'hotdoc.json')) self.formatted_signal(self) def __dump_deps_file(self): dest = self.config.get('deps_file_dest') target = self.config.get('deps_file_target') if dest is None: info("Not dumping deps file") return info("Dumping deps file to %s with target %s" % (dest, target)) destdir = os.path.dirname(dest) if not os.path.exists(destdir): os.makedirs(destdir) empty_targets = [] with io.open(dest, 'w', encoding='utf-8') as _: _.write(u'%s: ' % target) if self.config: for dep in self.config.get_dependencies(): empty_targets.append(dep) _.write(u'%s ' % dep) if self.doc_tree: for page in self.doc_tree.get_pages().values(): if not page.generated: empty_targets.append(page.source_file) _.write(u'%s ' % page.source_file) for empty_target in empty_targets: _.write(u'\n\n%s:' % empty_target) def __add_default_tags(self, _, comment): for validator in self.tag_validators.values(): if validator.default and validator.name not in comment.tags: comment.tags[validator.name] = \ Tag(name=validator.name, description=validator.default) def __setup_database(self): self.doc_database = DocDatabase() self.doc_database.comment_added_signal.connect(self.__add_default_tags) self.doc_database.comment_updated_signal.connect( self.__add_default_tags) self.doc_database.setup(self.get_private_folder()) self.link_resolver = LinkResolver(self.doc_database) def __create_change_tracker(self): try: self.change_tracker = \ pickle.load(open(os.path.join(self.get_private_folder(), 'change_tracker.p'), 'rb')) if self.change_tracker.hard_dependencies_are_stale(): raise IOError self.incremental = True info("Building incrementally") # pylint: disable=broad-except except Exception: info("Building from scratch") shutil.rmtree(self.get_private_folder(), ignore_errors=True) if self.output: shutil.rmtree(self.output, ignore_errors=True) self.change_tracker = ChangeTracker() def __get_formatter(self, extension_name): """ Banana banana """ ext = self.extensions.get(extension_name) if ext: return ext.get_formatter(self.output_format) return None def __check_initial_args(self, args): if args.version: print VERSION elif args.makefile_path: here = os.path.dirname(__file__) path = os.path.join(here, '..', 'utils', 'hotdoc.mk') print os.path.abspath(path) elif args.get_conf_path: key = args.get_conf_path path = self.config.get_path(key, rel_to_cwd=True) if path is not None: print path elif args.get_conf_key: key = args.get_conf_key value = self.config.get(args.get_conf_key, None) if value is not None: print value elif args.get_private_folder: print os.path.relpath(self.__private_folder, self.config.get_invoke_dir()) elif args.has_extension: ext_name = args.has_extension print ext_name in self.__extension_classes elif args.list_extensions: for ext_name in self.__extension_classes: print ext_name else: self.parser.print_usage() def __create_arg_parser(self): self.parser = \ argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter,) configurable_classes = all_subclasses(Configurable) seen = set() for subclass in configurable_classes: if subclass.add_arguments not in seen: subclass.add_arguments(self.parser) seen.add(subclass.add_arguments) self.parser.add_argument('command', action="store", choices=('run', 'conf', 'help'), nargs="?") self.parser.add_argument('--dry', help='Dry run, nothing will be output', dest='dry', action='store_true') self.parser.add_argument('--conf-file', help='Path to the config file', dest='conf_file') self.parser.add_argument('--output-conf-file', help='Path where to save the updated conf' ' file', dest='output_conf_file') self.parser.add_argument('--version', help="Print version and exit", action="store_true") self.parser.add_argument('--makefile-path', help="Print path to includable " "Makefile and exit", action="store_true") self.parser.add_argument('--deps-file-dest', help='Where to output the dependencies file') self.parser.add_argument('--deps-file-target', help='Name of the dependencies target', default='doc.stamp.d') self.parser.add_argument("-i", "--index", action="store", dest="index", help="location of the " "index file") self.parser.add_argument("--sitemap", action="store", dest="sitemap", help="Location of the sitemap file") self.parser.add_argument("--project-name", action="store", dest="project_name", help="Name of the documented project") self.parser.add_argument("--project-version", action="store", dest="project_version", help="Version of the documented project") self.parser.add_argument("-o", "--output", action="store", dest="output", help="where to output the rendered " "documentation") self.parser.add_argument("--get-conf-key", action="store", help="print the value for a configuration " "key") self.parser.add_argument("--get-conf-path", action="store", help="print the value for a configuration " "path") self.parser.add_argument("--get-private-folder", action="store_true", help="get the path to hotdoc's private " "folder") self.parser.add_argument("--output-format", action="store", dest="output_format", help="format for the " "output") self.parser.add_argument("--has-extension", action="store", dest="has_extension", help="Check if a given " "extension is available") self.parser.add_argument("--list-extensions", action="store_true", dest="list_extensions", help="Print " "available extensions") self.parser.add_argument("-", action="store_true", help="Separator to allow finishing a list" " of arguments before a command", dest="whatever") def load_command_line(self, args): """ Loads the repo from command line arguments """ args = self.parser.parse_args(args) self.__load_config(args) cmd = args.command if cmd == 'help': self.parser.print_help() sys.exit(0) elif cmd is None: self.__check_initial_args(args) sys.exit(0) exit_now = False save_config = False if cmd == 'run': self.__dry = bool(args.dry) elif cmd == 'conf': save_config = True exit_now = True if save_config: self.config.dump(args.output_conf_file) if exit_now: sys.exit(0) def load_conf_file(self, conf_file, overrides): """ Load the project from a configuration file and key-value overides. """ if conf_file is None and os.path.exists('hotdoc.json'): conf_file = 'hotdoc.json' self.__conf_file = conf_file if conf_file and not os.path.exists(conf_file): error('invalid-config', "No configuration file was found at %s" % conf_file) actual_args = {} defaults = {'output_format': 'html'} for key, value in overrides.items(): if key in ('cmd', 'conf_file', 'dry'): continue if value != self.parser.get_default(key): actual_args[key] = value if self.parser.get_default(key) is not None: defaults[key] = value self.config = ConfigParser(command_line_args=actual_args, conf_file=conf_file, defaults=defaults) index = self.config.get_index() if index: hash_obj = hashlib.md5(self.config.get_index()) priv_name = 'hotdoc-private-' + hash_obj.hexdigest() else: priv_name = 'hotdoc-private' self.__private_folder = os.path.abspath(priv_name) def __load_extensions(self): extension_classes = get_all_extension_classes(sort=True) for subclass in extension_classes: self.__extension_classes[subclass.extension_name] = subclass # pylint: disable=no-self-use def __load_config(self, args): """ Banana banana """ cli = dict(vars(args)) self.load_conf_file(args.conf_file, cli) def __setup_private_folder(self): folder = self.get_private_folder() if os.path.exists(folder): if not os.path.isdir(folder): error('setup-issue', '%s exists but is not a directory' % folder) else: os.mkdir(folder) def __create_extensions(self): for ext_class in self.__extension_classes.values(): ext = ext_class(self) self.extensions[ext.extension_name] = ext def get_base_doc_folder(self): """Get the folder in which the main index was located """ return self.__base_doc_folder def __parse_config(self): """ Banana banana """ output = self.config.get_path('output') or None self.sitemap_path = self.config.get_path('sitemap') if self.sitemap_path is None: error('invalid-config', 'No sitemap was provided') if output is not None: self.output = os.path.abspath(output) else: self.output = None self.project_name = self.config.get('project_name', None) self.project_version = self.config.get('project_version', None) self.output_format = self.config.get('output_format') if self.output_format not in ["html"]: error('invalid-config', 'Unsupported output format : %s' % self.output_format) self.__index_file = self.config.get_index() if self.__index_file is None: error('invalid-config', 'index is required') if not os.path.exists(self.__index_file): error('invalid-config', 'The provided index "%s" does not exist' % self.__index_file) cmd_line_includes = self.config.get_paths('include_paths') self.__base_doc_folder = os.path.dirname(self.__index_file) self.include_paths = OrderedSet([self.__base_doc_folder]) self.include_paths |= OrderedSet(cmd_line_includes) self.__create_change_tracker() self.__setup_private_folder() self.__setup_database() self.__create_extensions() if self.__conf_file: self.change_tracker.add_hard_dependency(self.__conf_file)