def test_generate_empty_functional(): """ Functional test for index generation of empty API. Header is plain text while content is compressed. """ project_name = 'some-name' log = [] logger = lambda part, message: log.append((part, message)) sut = SphinxInventory(logger=logger, project_name=project_name) output = PersistentStringIO() sut._openFileForWriting = lambda path: closing(output) sut.generate(subjects=[], basepath='base-path') expected_log = [( 'sphinx', 'Generating objects inventory at base-path/objects.inv' )] assert expected_log == log expected_ouput = """# Sphinx inventory version 2 # Project: some-name # Version: 2.0 # The rest of this file is compressed with zlib. x\x9c\x03\x00\x00\x00\x00\x01""" assert expected_ouput == output.getvalue()
def __init__(self, options: Optional[Values] = None): self.allobjects: Dict[str, Documentable] = {} self.rootobjects: List[Documentable] = [] self.violations = 0 """The number of docstring problems found. This is used to determine whether to fail the build when using the --warnings-as-errors option, so it should only be increased for problems that the user can fix. """ if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.projectname = 'my project' self.docstring_syntax_errors: Set[str] = set() """FullNames of objects for which the docstring failed to parse.""" self.verboselevel = 0 self.needsnl = False self.once_msgs: Set[Tuple[str, str]] = set() self.unprocessed_modules: Set[Module] = set() self.module_count = 0 self.processing_modules: List[str] = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg)
def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [ ] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg)
def test_generate_empty_functional(): """ Functional test for index generation of empty API. Header is plain text while content is compressed. """ project_name = 'some-name' log = [] logger = lambda section, message, thresh=0: log.append(( section, message, thresh)) sut = SphinxInventory(logger=logger, project_name=project_name) output = PersistentStringIO() sut._openFileForWriting = lambda path: closing(output) sut.generate(subjects=[], basepath='base-path') expected_log = [( 'sphinx', 'Generating objects inventory at base-path/objects.inv', 0 )] assert expected_log == log expected_ouput = """# Sphinx inventory version 2 # Project: some-name # Version: 2.0 # The rest of this file is compressed with zlib. x\x9c\x03\x00\x00\x00\x00\x01""" assert expected_ouput == output.getvalue()
def __init__(self, options=None): self.allobjects = OrderedDict() self.rootobjects = [] self.warnings = {} self.packages = [] if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.docstring_syntax_errors = set() """FullNames of objects for which the docstring failed to parse.""" self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg)
def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] self.moresystems = [] self.subsystems = [] self.urlprefix = '' if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [ ] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() # Once pickle support is removed, System should be # initialized with project name so that we can reuse intersphinx instance for # object.inv generation. self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def test_EpydocLinker_resolve_identifier_xref_intersphinx_relative_id( ) -> None: """ Return the link from inventory using short names, by resolving them based on the imports done in the module. """ system = model.System() inventory = SphinxInventory(system.msg) inventory._links['ext_package.ext_module'] = ('http://tm.tld', 'some.html') system.intersphinx = inventory target = model.Module(system, 'ignore-name') # Here we set up the target module as it would have this import. # from ext_package import ext_module ext_package = model.Module(system, 'ext_package') target.contents['ext_module'] = model.Module(system, 'ext_module', parent=ext_package) sut = epydoc2stan._EpydocLinker(target) # This is called for the L{ext_module<Pretty Text>} markup. url = sut.resolve_identifier('ext_module') url_xref = sut.resolve_identifier_xref('ext_module', 0) assert "http://tm.tld/some.html" == url assert "http://tm.tld/some.html" == url_xref
def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] self.moresystems = [] self.subsystems = [] self.urlprefix = '' if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() # Once pickle support is removed, System should be # initialized with project name so that we can reuse intersphinx instance for # object.inv generation. self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
def test_EpydocLinker_translate_identifier_xref_intersphinx_relative_id(): """ Return the link from inventory using short names, by resolving them based on the imports done in the module. """ system = model.System() inventory = SphinxInventory(system.msg, "some-project") inventory._links["ext_package.ext_module"] = ("http://tm.tld", "some.html") system.intersphinx = inventory target = model.Module(system, "ignore-name", "ignore-docstring") # Here we set up the target module as it would have this import. # from ext_package import ext_module ext_package = model.Module(system, "ext_package", "ignore-docstring") target.contents["ext_module"] = model.Module(system, "ext_module", "ignore-docstring", parent=ext_package) sut = epydoc2stan._EpydocLinker(target) # This is called for the L{ext_module<Pretty Text>} markup. result = sut.translate_identifier_xref("ext_module", "Pretty Text") expected = '<a href="http://tm.tld/some.html"><code>Pretty Text</code></a>' assert expected == result
def test_parseInventory_empty( inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return empty dict for empty input. """ result = inv_reader_nolog._parseInventory('http://base.tld', '') assert {} == result
def test_getLink_found(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return the link from internal state. """ inv_reader_nolog._links['some.name'] = ('http://base.tld', 'some/url.php') assert 'http://base.tld/some/url.php' == inv_reader_nolog.getLink( 'some.name')
def test_parseInventory_single_line( inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return a dict with a single member. """ result = inv_reader_nolog._parseInventory( 'http://base.tld', 'some.attr py:attr -1 some.html De scription') assert {'some.attr': ('http://base.tld', 'some.html')} == result
def test_getLink_self_anchor(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return the link with anchor as target name when link end with $. """ inv_reader_nolog._links['some.name'] = ('http://base.tld', 'some/url.php#$') assert 'http://base.tld/some/url.php#some.name' == inv_reader_nolog.getLink( 'some.name')
def test_initialization(): """ Is initialized with logger and project name. """ logger = object() name = object() sut = SphinxInventory(logger=logger, project_name=name) assert logger is sut.info assert name is sut.project_name
def __setstate__(self, state): if "abbrevmapping" not in state: state["abbrevmapping"] = {} # this is so very, very evil. # see doc/extreme-pickling-pain.txt for more. def lookup(name): for system in [self] + self.moresystems + self.subsystems: if name in system.allobjects: return system.allobjects[name] raise KeyError(name) self.__dict__.update(state) for system in [self] + self.moresystems + self.subsystems: if "allobjects" not in system.__dict__: return for system in [self] + self.moresystems + self.subsystems: for obj in system.orderedallobjects: for k, v in obj.__dict__.copy().iteritems(): if k.startswith("$"): del obj.__dict__[k] obj.__dict__[k[1:]] = lookup(v) elif k.startswith("@"): n = [] for vv in v: if vv is None: n.append(None) else: n.append(lookup(vv)) del obj.__dict__[k] obj.__dict__[k[1:]] = n elif k.startswith("!"): n = {} for kk, vv in v.iteritems(): n[kk] = lookup(vv) del obj.__dict__[k] obj.__dict__[k[1:]] = n self.intersphinx = SphinxInventory( logger=self.msg, project_name=self.projectname )
def test_update_functional(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Functional test for updating from an empty inventory. """ payload = (b'some.module1 py:module -1 module1.html -\n' b'other.module2 py:module 0 module2.html Other description\n') # Patch URL loader to avoid hitting the system. content = b"""# Sphinx inventory version 2 # Project: some-name # Version: 2.0 # The rest of this file is compressed with zlib. """ + zlib.compress(payload) url = 'http://some.url/api/objects.inv' inv_reader_nolog.update({url: content}, url) assert 'http://some.url/api/module1.html' == inv_reader_nolog.getLink( 'some.module1') assert 'http://some.url/api/module2.html' == inv_reader_nolog.getLink( 'other.module2')
def test_EpydocLinker_look_for_intersphinx_hit(): """ Return the link from inventory based on first package name. """ system = model.System() inventory = SphinxInventory(system.msg, "some-project") inventory._links["base.module.other"] = ("http://tm.tld", "some.html") system.intersphinx = inventory target = model.Module(system, "ignore-name", "ignore-docstring") sut = epydoc2stan._EpydocLinker(target) result = sut.look_for_intersphinx("base.module.other") assert "http://tm.tld/some.html" == result
def test_EpydocLinker_look_for_intersphinx_hit() -> None: """ Return the link from inventory based on first package name. """ system = model.System() inventory = SphinxInventory(system.msg) inventory._links['base.module.other'] = ('http://tm.tld', 'some.html') system.intersphinx = inventory target = model.Module(system, 'ignore-name') sut = epydoc2stan._EpydocLinker(target) result = sut.look_for_intersphinx('base.module.other') assert 'http://tm.tld/some.html' == result
def test_getPayload_empty(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return empty string. """ content = b"""# Sphinx inventory version 2 # Project: some-name # Version: 2.0 # The rest of this file is compressed with zlib. x\x9c\x03\x00\x00\x00\x00\x01""" result = inv_reader_nolog._getPayload('http://base.ignore', content) assert '' == result
def test_getPayload_content(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return content as string. """ payload = "first_line\nsecond line\nit's a snake: \U0001F40D" content = b"""# Ignored line # Project: some-name # Version: 2.0 # commented line. """ + zlib.compress(payload.encode('utf-8')) result = inv_reader_nolog._getPayload('http://base.ignore', content) assert payload == result
def test_EpydocLinker_resolve_identifier_xref_intersphinx_absolute_id(): """ Returns the link from Sphinx inventory based on a cross reference ID specified in absolute dotted path and with a custom pretty text for the URL. """ system = model.System() inventory = SphinxInventory(system.msg) inventory._links['base.module.other'] = ('http://tm.tld', 'some.html') system.intersphinx = inventory target = model.Module(system, 'ignore-name') sut = epydoc2stan._EpydocLinker(target) url = sut.resolve_identifier_xref('base.module.other') assert "http://tm.tld/some.html" == url
def test_xref_link_intersphinx() -> None: """A linked name that is documented in another project is linked using an absolute URL (retrieved via Intersphinx). """ mod = fromText(''' def func(): """This is a thin wrapper around L{external.func}.""" ''', modname='test') system = mod.system inventory = SphinxInventory(system.msg) inventory._links['external.func'] = ('https://example.net', 'lib.html#func') system.intersphinx = inventory html = docstring2html(mod.contents['func']) assert 'href="https://example.net/lib.html#func"' in html
def test_EpydocLinker_translate_identifier_xref_intersphinx(): """ Return the link from inventory. """ system = model.System() inventory = SphinxInventory(system.msg, 'some-project') inventory._links['base.module.other'] = ('http://tm.tld', 'some.html') system.intersphinx = inventory target = model.Module(system, 'ignore-name', 'ignore-docstring') sut = epydoc2stan._EpydocLinker(target) result = sut.translate_identifier_xref( 'base.module.other', 'base.module.pretty') expected = ( '<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>' ) assert expected == result
def test_EpydocLinker_translate_identifier_xref_intersphinx_absolute_id(): """ Returns the link from Sphinx inventory based on a cross reference ID specified in absolute dotted path and with a custom pretty text for the URL. """ system = model.System() inventory = SphinxInventory(system.msg, "some-project") inventory._links["base.module.other"] = ("http://tm.tld", "some.html") system.intersphinx = inventory target = model.Module(system, "ignore-name", "ignore-docstring") sut = epydoc2stan._EpydocLinker(target) result = sut.translate_identifier_xref("base.module.other", "base.module.pretty") expected = '<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>' assert expected == result
def test_EpydocLinker_translate_identifier_xref_intersphinx_absolute_id(): """ Returns the link from Sphinx inventory based on a cross reference ID specified in absolute dotted path and with a custom pretty text for the URL. """ system = model.System() inventory = SphinxInventory(system.msg) inventory._links['base.module.other'] = ('http://tm.tld', 'some.html') system.intersphinx = inventory target = model.Module(system, 'ignore-name', 'ignore-docstring') sut = epydoc2stan._EpydocLinker(target) result = sut.translate_identifier_xref('base.module.other', 'base.module.pretty') expected = ( '<a href="http://tm.tld/some.html"><code>base.module.pretty</code></a>' ) assert expected == flatten(result)
def __setstate__(self, state): if 'abbrevmapping' not in state: state['abbrevmapping'] = {} # this is so very, very evil. # see doc/extreme-pickling-pain.txt for more. def lookup(name): for system in [self] + self.moresystems + self.subsystems: if name in system.allobjects: return system.allobjects[name] raise KeyError, name self.__dict__.update(state) for system in [self] + self.moresystems + self.subsystems: if 'allobjects' not in system.__dict__: return for system in [self] + self.moresystems + self.subsystems: for obj in system.orderedallobjects: for k, v in obj.__dict__.copy().iteritems(): if k.startswith('$'): del obj.__dict__[k] obj.__dict__[k[1:]] = lookup(v) elif k.startswith('@'): n = [] for vv in v: if vv is None: n.append(None) else: n.append(lookup(vv)) del obj.__dict__[k] obj.__dict__[k[1:]] = n elif k.startswith('!'): n = {} for kk, vv in v.iteritems(): n[kk] = lookup(vv) del obj.__dict__[k] obj.__dict__[k[1:]] = n self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname)
class System(object): """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # not done here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder sourcebase = None def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] self.moresystems = [] self.subsystems = [] self.urlprefix = '' if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [ ] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() # Once pickle support is removed, System should be # initialized with project name so that we can reuse intersphinx instance for # object.inv generation. self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def verbosity(self, section=None): if isinstance(section, str): section = (section, ) delta = max( [self.options.verbosity_details.get(sect, 0) for sect in section]) return self.options.verbosity + delta def progress(self, section, i, n, msg): if n is None: i = str(i) else: i = '%s/%s' % (i, n) if self.verbosity(section) == 0 and sys.stdout.isatty(): print('\r' + i, msg, end='') sys.stdout.flush() if i == n: self.needsnl = False print() else: self.needsnl = True def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False): if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print() print(msg, end='') if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print() def objForFullName(self, fullName): for system in [self] + self.moresystems: if fullName in system.allobjects: return system.allobjects[fullName] return None def _warning(self, current, type, detail): if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print(fn, type, detail) self.warnings.setdefault(type, []).append((fn, detail)) def objectsOfType(self, cls): """Iterate over all instances of C{cls} present in the system. """ for o in self.orderedallobjects: if isinstance(o, cls): yield o def privacyClass(self, ob): if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def __getstate__(self): d = self.__dict__.copy() del d['intersphinx'] return d def __setstate__(self, state): if 'abbrevmapping' not in state: state['abbrevmapping'] = {} # this is so very, very evil. # see doc/extreme-pickling-pain.txt for more. def lookup(name): for system in [self] + self.moresystems + self.subsystems: if name in system.allobjects: return system.allobjects[name] raise KeyError(name) self.__dict__.update(state) for system in [self] + self.moresystems + self.subsystems: if 'allobjects' not in system.__dict__: return for system in [self] + self.moresystems + self.subsystems: for obj in system.orderedallobjects: for k, v in obj.__dict__.copy().iteritems(): if k.startswith('$'): del obj.__dict__[k] obj.__dict__[k[1:]] = lookup(v) elif k.startswith('@'): n = [] for vv in v: if vv is None: n.append(None) else: n.append(lookup(vv)) del obj.__dict__[k] obj.__dict__[k[1:]] = n elif k.startswith('!'): n = {} for kk, vv in v.iteritems(): n[kk] = lookup(vv) del obj.__dict__[k] obj.__dict__[k[1:]] = n self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def addObject(self, obj): """Add C{object} to the system.""" if obj.parent: obj.parent.orderedcontents.append(obj) obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) self.orderedallobjects.append(obj) if obj.fullName() in self.allobjects: self.handleDuplicate(obj) else: self.allobjects[obj.fullName()] = obj # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an # "--add-package" argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod): if self.sourcebase is None: mod.sourceHref = None return projBaseDir = mod.system.options.projectbasedirectory if projBaseDir is not None: mod.sourceHref = (self.sourcebase + mod.filepath[len(projBaseDir):]) return trailing = [] dir, fname = os.path.split(mod.filepath) while os.path.exists(os.path.join(dir, '.svn')): dir, dirname = os.path.split(dir) trailing.append(dirname) # now trailing[-1] would be 'Divmod' in the above example del trailing[-1] trailing.reverse() trailing.append(fname) mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing) def addModule(self, modpath, modname, parentPackage=None): mod = self.Module(self, modname, None, parentPackage) self.addObject(mod) self.progress("addModule", len(self.orderedallobjects), None, "modules and packages discovered") mod.filepath = modpath self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod) def ensureModule(self, module_full_name): if module_full_name in self.allobjects: return self.allobjects[module_full_name] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, None, parent_package) self.addObject(module) return module def ensurePackage(self, package_full_name): if package_full_name in self.allobjects: return self.allobjects[package_full_name] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, None, parent_package) self.addObject(package) return package def _introspectThing(self, thing, parent, parentMod): for k, v in thing.__dict__.iteritems(): if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))): f = self.Function(self, k, v.__doc__, parent) f.parentMod = parentMod f.decorators = None f.argspec = ((), None, None, ()) self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, v.__doc__, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, py_mod, module_full_name): module = self.ensureModule(module_full_name) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) print(py_mod) def addPackage(self, dirpath, parentPackage=None): if not os.path.exists(dirpath): raise Exception("package path %r does not exist!" % (dirpath, )) if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) package.filepath = dirpath self.setSourceHref(package) for fname in os.listdir(dirpath): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package, path): for (suffix, mode, type) in imp.get_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if type == imp.C_EXTENSION: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = "%s.%s" % (package.fullName(), module_name) else: module_full_name = module_name py_mod = imp.load_module(module_full_name, open(path, 'rb'), path, (suffix, mode, type)) self.introspectModule(py_mod, module_full_name) elif type == imp.PY_SOURCE: self.addModule(path, module_name, package) break def handleDuplicate(self, obj): '''This is called when we see two objects with the same .fullName(), for example: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fn = obj.fullName() while (fn + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[obj.fullName()] self._warning(obj.parent, "duplicate", prev) def remove(o): del self.allobjects[o.fullName()] for c in o.orderedcontents: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o): self.allobjects[o.fullName()] = o for c in o.orderedcontents: readd(c) readd(prev) self.allobjects[obj.fullName()] = obj return obj def getProcessedModule(self, modname): mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__').parent if not isinstance(mod, Module): return None if mod.state == UNPROCESSED: self.processModule(mod) return mod def processModule(self, mod): assert mod.state == UNPROCESSED mod.state = PROCESSING if getattr(mod, 'filepath', None) is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.filepath) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s" % (self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) self.progress( 'process', self.module_count - len(self.unprocessed_modules), self.module_count, "modules processed %s warnings" % (sum(len(v) for v in self.warnings.itervalues()), )) def process(self): while self.unprocessed_modules: mod = iter(self.unprocessed_modules).next() self.processModule(mod) def fetchIntersphinxInventories(self): """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(url)
def test_getLink_not_found(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Return None if link does not exists. """ assert None is inv_reader_nolog.getLink('no.such.name')
class System(object): """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # not done here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder sourcebase = None def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] self.moresystems = [] self.subsystems = [] self.urlprefix = '' if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() # Once pickle support is removed, System should be # initialized with project name so that we can reuse intersphinx instance for # object.inv generation. self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def verbosity(self, section=None): if isinstance(section, str): section = (section,) delta = max([self.options.verbosity_details.get(sect, 0) for sect in section]) return self.options.verbosity + delta def progress(self, section, i, n, msg): if n is None: i = str(i) else: i = '%s/%s'%(i,n) if self.verbosity(section) == 0 and sys.stdout.isatty(): print '\r'+i, msg, sys.stdout.flush() if i == n: self.needsnl = False print else: self.needsnl = True def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False): if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print print msg, if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print def objForFullName(self, fullName): for system in [self] + self.moresystems: if fullName in system.allobjects: return system.allobjects[fullName] return None def _warning(self, current, type, detail): if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print fn, type, detail self.warnings.setdefault(type, []).append((fn, detail)) def objectsOfType(self, cls): """Iterate over all instances of C{cls} present in the system. """ for o in self.orderedallobjects: if isinstance(o, cls): yield o def privacyClass(self, ob): if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def __getstate__(self): d = self.__dict__.copy() del d['intersphinx'] return d def __setstate__(self, state): if 'abbrevmapping' not in state: state['abbrevmapping'] = {} # this is so very, very evil. # see doc/extreme-pickling-pain.txt for more. def lookup(name): for system in [self] + self.moresystems + self.subsystems: if name in system.allobjects: return system.allobjects[name] raise KeyError, name self.__dict__.update(state) for system in [self] + self.moresystems + self.subsystems: if 'allobjects' not in system.__dict__: return for system in [self] + self.moresystems + self.subsystems: for obj in system.orderedallobjects: for k, v in obj.__dict__.copy().iteritems(): if k.startswith('$'): del obj.__dict__[k] obj.__dict__[k[1:]] = lookup(v) elif k.startswith('@'): n = [] for vv in v: if vv is None: n.append(None) else: n.append(lookup(vv)) del obj.__dict__[k] obj.__dict__[k[1:]] = n elif k.startswith('!'): n = {} for kk, vv in v.iteritems(): n[kk] = lookup(vv) del obj.__dict__[k] obj.__dict__[k[1:]] = n self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def addObject(self, obj): """Add C{object} to the system.""" if obj.parent: obj.parent.orderedcontents.append(obj) obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) self.orderedallobjects.append(obj) if obj.fullName() in self.allobjects: self.handleDuplicate(obj) else: self.allobjects[obj.fullName()] = obj # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an # "--add-package" argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod): if self.sourcebase is None: mod.sourceHref = None return projBaseDir = mod.system.options.projectbasedirectory if projBaseDir is not None: mod.sourceHref = ( self.sourcebase + mod.filepath[len(projBaseDir):]) return trailing = [] dir, fname = os.path.split(mod.filepath) while os.path.exists(os.path.join(dir, '.svn')): dir, dirname = os.path.split(dir) trailing.append(dirname) # now trailing[-1] would be 'Divmod' in the above example del trailing[-1] trailing.reverse() trailing.append(fname) mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing) def addModule(self, modpath, modname, parentPackage=None): mod = self.Module(self, modname, None, parentPackage) self.addObject(mod) self.progress( "addModule", len(self.orderedallobjects), None, "modules and packages discovered") mod.filepath = modpath self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod) def ensureModule(self, module_full_name): if module_full_name in self.allobjects: return self.allobjects[module_full_name] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, None, parent_package) self.addObject(module) return module def ensurePackage(self, package_full_name): if package_full_name in self.allobjects: return self.allobjects[package_full_name] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, None, parent_package) self.addObject(package) return package def _introspectThing(self, thing, parent, parentMod): for k, v in thing.__dict__.iteritems(): if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))): f = self.Function(self, k, v.__doc__, parent) f.parentMod = parentMod f.decorators = None f.argspec = ((), None, None, ()) self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, v.__doc__, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, py_mod, module_full_name): module = self.ensureModule(module_full_name) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) print py_mod def addPackage(self, dirpath, parentPackage=None): if not os.path.exists(dirpath): raise Exception("package path %r does not exist!" %(dirpath,)) if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) package.filepath = dirpath self.setSourceHref(package) for fname in os.listdir(dirpath): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package, path): for (suffix, mode, type) in imp.get_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if type == imp.C_EXTENSION: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = "%s.%s" % ( package.fullName(), module_name) else: module_full_name = module_name py_mod = imp.load_module( module_full_name, open(path, 'rb'), path, (suffix, mode, type)) self.introspectModule(py_mod, module_full_name) elif type == imp.PY_SOURCE: self.addModule(path, module_name, package) break def handleDuplicate(self, obj): '''This is called when we see two objects with the same .fullName(), for example: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fn = obj.fullName() while (fn + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[obj.fullName()] self._warning(obj.parent, "duplicate", prev) def remove(o): del self.allobjects[o.fullName()] for c in o.orderedcontents: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o): self.allobjects[o.fullName()] = o for c in o.orderedcontents: readd(c) readd(prev) self.allobjects[obj.fullName()] = obj return obj def getProcessedModule(self, modname): mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__').parent if not isinstance(mod, Module): return None if mod.state == UNPROCESSED: self.processModule(mod) return mod def processModule(self, mod): assert mod.state == UNPROCESSED mod.state = PROCESSING if getattr(mod, 'filepath', None) is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.filepath) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s"%(self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) self.progress( 'process', self.module_count - len(self.unprocessed_modules), self.module_count, "modules processed %s warnings"%( sum(len(v) for v in self.warnings.itervalues()),)) def process(self): while self.unprocessed_modules: mod = iter(self.unprocessed_modules).next() self.processModule(mod) def fetchIntersphinxInventories(self): """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(url)
def make_SphinxInventory(logger=object()): """ Return a SphinxInventory. """ return SphinxInventory(logger=logger, project_name='project_name')
def main(args): import cPickle options, args = parse_args(args) exitcode = 0 if options.configfile: readConfigFile(options) try: # step 1: make/find the system if options.systemclass: systemclass = findClassFromDottedName(options.systemclass, '--system-class') if not issubclass(systemclass, model.System): msg = "%s is not a subclass of model.System" error(msg, systemclass) else: systemclass = zopeinterface.ZopeInterfaceSystem if options.inputpickle: system = cPickle.load(open(options.inputpickle, 'rb')) if options.systemclass: if type(system) is not systemclass: cls = type(system) msg = ("loaded pickle has class %s.%s, differing " "from explicitly requested %s") error(msg, cls.__module__, cls.__name__, options.systemclass) else: system = systemclass() # Once pickle support is removed, always instantiate System with # options and make fetchIntersphinxInventories private in __init__. system.options = options system.fetchIntersphinxInventories() system.urlprefix = '' if options.moresystems: moresystems = [] for fnamepref in options.moresystems: fname, prefix = fnamepref.split(':', 1) moresystems.append(cPickle.load(open(fname, 'rb'))) moresystems[-1].urlprefix = prefix moresystems[-1].options = system.options moresystems[-1].subsystems.append(system) system.moresystems = moresystems system.sourcebase = options.htmlsourcebase if options.abbrevmapping: for thing in options.abbrevmapping.split(','): k, v = thing.split('=') system.abbrevmapping[k] = v # step 1.5: check that we're actually going to accomplish something here if options.auto: options.server = True options.edit = True for fn in os.listdir('.'): if os.path.isdir(fn) and \ os.path.exists(os.path.join(fn, '__init__.py')): options.packages.append(fn) elif fn.endswith('.py') and fn != 'setup.py': options.modules.append(fn) args = list(args) + options.modules + options.packages if options.makehtml == MAKE_HTML_DEFAULT: if not options.outputpickle and not options.testing \ and not options.server and not options.makeintersphinx: options.makehtml = True else: options.makehtml = False if options.buildtime: try: system.buildtime = datetime.datetime.strptime( options.buildtime, BUILDTIME_FORMAT) except ValueError, e: error(e) # step 2: add any packages and modules if args: prependedpackage = None if options.prependedpackage: for m in options.prependedpackage.split('.'): prependedpackage = system.Package( system, m, None, prependedpackage) system.addObject(prependedpackage) initmodule = system.Module(system, '__init__', None, prependedpackage) system.addObject(initmodule) for path in args: path = os.path.abspath(path) if path in system.packages: continue if os.path.isdir(path): system.msg('addPackage', 'adding directory ' + path) system.addPackage(path, prependedpackage) else: system.msg('addModuleFromPath', 'adding module ' + path) system.addModuleFromPath(prependedpackage, path) system.packages.append(path) # step 3: move the system to the desired state if not system.packages: error("The system does not contain any code, did you " "forget an --add-package?") system.process() if system.options.livecheck: error("write this") if system.options.projectname is None: name = '/'.join([ro.name for ro in system.rootobjects]) system.msg( 'warning', 'WARNING: guessing '+name+' for project name', thresh=-1) system.projectname = name else: system.projectname = system.options.projectname # step 4: save the system, if desired if options.outputpickle: system.msg('', 'saving output pickle to ' + options.outputpickle) del system.options # don't persist the options f = open(options.outputpickle, 'wb') cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL) f.close() system.options = options # step 5: make html, if desired if options.makehtml: options.makeintersphinx = True if options.htmlwriter: writerclass = findClassFromDottedName( options.htmlwriter, '--html-writer') else: from pydoctor import templatewriter writerclass = templatewriter.TemplateWriter system.msg('html', 'writing html to %s using %s.%s'%( options.htmloutput, writerclass.__module__, writerclass.__name__)) writer = writerclass(options.htmloutput) writer.system = system writer.prepOutputDirectory() system.epytextproblems = [] if options.htmlsubjects: subjects = [] for fn in options.htmlsubjects: subjects.append(system.allobjects[fn]) elif options.htmlsummarypages: writer.writeModuleIndex(system) subjects = [] else: writer.writeModuleIndex(system) subjects = system.rootobjects writer.writeIndividualFiles(subjects, options.htmlfunctionpages) if system.epytextproblems: def p(msg): system.msg(('epytext', 'epytext-summary'), msg, thresh=-1, topthresh=1) p("these %s objects' docstrings are not proper epytext:" %(len(system.epytextproblems),)) exitcode = 2 for fn in system.epytextproblems: p(' '+fn) if options.outputpickle: system.msg( '', 'saving output pickle to ' + options.outputpickle) # save again, with epytextproblems del system.options # don't persist the options f = open(options.outputpickle, 'wb') cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL) f.close() system.options = options if options.makeintersphinx: if not options.makehtml: subjects = system.rootobjects # Generate Sphinx inventory. sphinx_inventory = SphinxInventory( logger=system.msg, project_name=system.projectname, ) if not os.path.exists(options.htmloutput): os.makedirs(options.htmloutput) sphinx_inventory.generate( subjects=subjects, basepath=options.htmloutput, ) # Finally, if we should serve html, lets serve some html. if options.server: from pydoctor.server import ( EditingPyDoctorResource, PyDoctorResource) from pydoctor.epydoc2stan import doc2stan from twisted.web.server import Site from twisted.web.resource import Resource from twisted.web.vhost import VHostMonsterResource from twisted.internet import reactor if options.edit: if not options.nocheck: system.msg( "server", "Checking formatting of docstrings.") included_obs = [ ob for ob in system.orderedallobjects if ob.isVisible] for i, ob in enumerate(included_obs): system.progress( "server", i+1, len(included_obs), "docstrings checked, found %s problems" % ( len(system.epytextproblems))) doc2stan(ob, docstring=ob.docstring) root = EditingPyDoctorResource(system) else: root = PyDoctorResource(system) if options.facing_path: options.local_only = True realroot = Resource() cur = realroot realroot.putChild('vhost', VHostMonsterResource()) segments = options.facing_path.split('/') for segment in segments[:-1]: next = Resource() cur.putChild(segment, next) cur = next cur.putChild(segments[-1], root) root = realroot system.msg( "server", "Setting up server at http://localhost:%d/" % options.server_port) if options.auto: def wb_open(): import webbrowser webbrowser.open( 'http://localhost:%d/' % options.server_port) reactor.callWhenRunning(wb_open) from twisted.python import log log.startLogging(sys.stdout) site = Site(root) if options.local_only: interface = 'localhost' else: interface = '' reactor.listenTCP(options.server_port, site, interface=interface) reactor.run()
class System: """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # Not assigned here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder defaultBuilder: Type[ASTBuilder] sourcebase = None def __init__(self, options=None): self.allobjects = {} self.rootobjects = [] self.warnings = {} self.packages = [] if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.docstring_syntax_errors = set() """FullNames of objects for which the docstring failed to parse.""" self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg) def verbosity(self, section=None): if isinstance(section, str): section = (section, ) delta = max( self.options.verbosity_details.get(sect, 0) for sect in section) return self.options.verbosity + delta def progress(self, section, i, n, msg): if n is None: i = str(i) else: i = f'{i}/{n}' if self.verbosity(section) == 0 and sys.stdout.isatty(): print('\r' + i, msg, end='') sys.stdout.flush() if i == n: self.needsnl = False print() else: self.needsnl = True def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False): if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print() print(msg, end='') if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print('') def objForFullName(self, fullName): return self.allobjects.get(fullName) def _warning(self, current, message, detail): if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print(fn, message, detail) self.warnings.setdefault(message, []).append((fn, detail)) def objectsOfType(self, cls): """Iterate over all instances of C{cls} present in the system. """ for o in self.allobjects.values(): if isinstance(o, cls): yield o def privacyClass(self, ob): if ob.kind is None: return PrivacyClass.HIDDEN if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def addObject(self, obj): """Add C{object} to the system.""" fullName = obj.fullName() if obj.parent and obj.parent.fullName() != fullName: obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) if fullName in self.allobjects: self.handleDuplicate(obj) else: self.allobjects[fullName] = obj # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an # "--add-package" argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod): if self.sourcebase is None: mod.sourceHref = None else: projBaseDir = mod.system.options.projectbasedirectory mod.sourceHref = (self.sourcebase + mod.filepath[len(projBaseDir):]) def addModule(self, modpath, modname, parentPackage=None): mod = self.Module(self, modname, parentPackage) self.addObject(mod) self.progress("addModule", len(self.allobjects), None, "modules and packages discovered") mod.filepath = modpath self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod) def ensureModule(self, module_full_name): if module_full_name in self.allobjects: return self.allobjects[module_full_name] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, parent_package) self.addObject(module) return module def ensurePackage(self, package_full_name): if package_full_name in self.allobjects: return self.allobjects[package_full_name] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, parent_package) self.addObject(package) return package def _introspectThing(self, thing, parent, parentMod): for k, v in thing.__dict__.items(): if isinstance(v, (types.BuiltinFunctionType, types.FunctionType)): f = self.Function(self, k, parent) f.parentMod = parentMod f.docstring = v.__doc__ f.decorators = None f.argspec = ((), None, None, ()) self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod c.docstring = v.__doc__ self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, py_mod, module_full_name): module = self.ensureModule(module_full_name) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) def addPackage(self, dirpath, parentPackage=None): if not os.path.exists(dirpath): raise Exception(f"package path {dirpath!r} does not exist!") if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) package.filepath = dirpath self.setSourceHref(package) for fname in sorted(os.listdir(dirpath)): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package, path): for (suffix, mode, impl) in imp.get_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if impl == imp.C_EXTENSION: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = f'{package.fullName()}.{module_name}' else: module_full_name = module_name py_mod = imp.load_module(module_full_name, open(path, 'rb'), path, (suffix, mode, impl)) self.introspectModule(py_mod, module_full_name) elif impl == imp.PY_SOURCE: self.addModule(path, module_name, package) break def handleDuplicate(self, obj): '''This is called when we see two objects with the same .fullName(), for example: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fullName = obj.fullName() while (fullName + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[fullName] self._warning(obj.parent, "duplicate", prev) def remove(o): del self.allobjects[o.fullName()] oc = list(o.contents.values()) for c in oc: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o): self.allobjects[o.fullName()] = o for c in o.contents.values(): readd(c) readd(prev) self.allobjects[fullName] = obj def getProcessedModule(self, modname): mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__').parent if not isinstance(mod, Module): return None if mod.state is ProcessingState.UNPROCESSED: self.processModule(mod) return mod def processModule(self, mod): assert mod.state is ProcessingState.UNPROCESSED mod.state = ProcessingState.PROCESSING if getattr(mod, 'filepath', None) is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.filepath) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s" % (self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = ProcessingState.PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) num_warnings = sum(len(v) for v in self.warnings.values()) self.progress('process', self.module_count - len(self.unprocessed_modules), self.module_count, f"modules processed {num_warnings} warnings") def process(self): while self.unprocessed_modules: mod = next(iter(self.unprocessed_modules)) self.processModule(mod) def fetchIntersphinxInventories(self, cache): """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(cache, url)
def main(args): import cPickle options, args = parse_args(args) exitcode = 0 if options.configfile: readConfigFile(options) cache = prepareCache(clearCache=options.clear_intersphinx_cache, enableCache=options.enable_intersphinx_cache, cachePath=options.intersphinx_cache_path, maxAge=options.intersphinx_cache_max_age) try: # step 1: make/find the system if options.systemclass: systemclass = findClassFromDottedName(options.systemclass, '--system-class') if not issubclass(systemclass, model.System): msg = "%s is not a subclass of model.System" error(msg, systemclass) else: systemclass = zopeinterface.ZopeInterfaceSystem if options.inputpickle: system = cPickle.load(open(options.inputpickle, 'rb')) if options.systemclass: if type(system) is not systemclass: cls = type(system) msg = ("loaded pickle has class %s.%s, differing " "from explicitly requested %s") error(msg, cls.__module__, cls.__name__, options.systemclass) else: system = systemclass() # Once pickle support is removed, always instantiate System with # options and make fetchIntersphinxInventories private in __init__. system.options = options system.fetchIntersphinxInventories(cache) system.urlprefix = '' if options.moresystems: moresystems = [] for fnamepref in options.moresystems: fname, prefix = fnamepref.split(':', 1) moresystems.append(cPickle.load(open(fname, 'rb'))) moresystems[-1].urlprefix = prefix moresystems[-1].options = system.options moresystems[-1].subsystems.append(system) system.moresystems = moresystems system.sourcebase = options.htmlsourcebase if options.abbrevmapping: for thing in options.abbrevmapping.split(','): k, v = thing.split('=') system.abbrevmapping[k] = v # step 1.5: check that we're actually going to accomplish something here args = list(args) + options.modules + options.packages if options.makehtml == MAKE_HTML_DEFAULT: if not options.outputpickle and not options.testing \ and not options.makeintersphinx: options.makehtml = True else: options.makehtml = False # Support source date epoch: # https://reproducible-builds.org/specs/source-date-epoch/ try: system.buildtime = datetime.datetime.utcfromtimestamp( int(os.environ['SOURCE_DATE_EPOCH'])) except ValueError as e: error(e) except KeyError: pass if options.buildtime: try: system.buildtime = datetime.datetime.strptime( options.buildtime, BUILDTIME_FORMAT) except ValueError as e: error(e) # step 2: add any packages and modules if args: prependedpackage = None if options.prependedpackage: for m in options.prependedpackage.split('.'): prependedpackage = system.Package(system, m, None, prependedpackage) system.addObject(prependedpackage) initmodule = system.Module(system, '__init__', None, prependedpackage) system.addObject(initmodule) for path in args: path = os.path.abspath(path) if path in system.packages: continue if os.path.isdir(path): system.msg('addPackage', 'adding directory ' + path) system.addPackage(path, prependedpackage) else: system.msg('addModuleFromPath', 'adding module ' + path) system.addModuleFromPath(prependedpackage, path) system.packages.append(path) # step 3: move the system to the desired state if not system.packages: error("The system does not contain any code, did you " "forget an --add-package?") system.process() if system.options.livecheck: error("write this") if system.options.projectname is None: name = '/'.join([ro.name for ro in system.rootobjects]) system.msg('warning', 'WARNING: guessing ' + name + ' for project name', thresh=-1) system.projectname = name else: system.projectname = system.options.projectname # step 4: save the system, if desired if options.outputpickle: system.msg('', 'saving output pickle to ' + options.outputpickle) del system.options # don't persist the options f = open(options.outputpickle, 'wb') cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL) f.close() system.options = options # step 5: make html, if desired if options.makehtml: options.makeintersphinx = True if options.htmlwriter: writerclass = findClassFromDottedName(options.htmlwriter, '--html-writer') else: from pydoctor import templatewriter writerclass = templatewriter.TemplateWriter system.msg( 'html', 'writing html to %s using %s.%s' % (options.htmloutput, writerclass.__module__, writerclass.__name__)) writer = writerclass(options.htmloutput) writer.system = system writer.prepOutputDirectory() system.epytextproblems = [] if options.htmlsubjects: subjects = [] for fn in options.htmlsubjects: subjects.append(system.allobjects[fn]) elif options.htmlsummarypages: writer.writeModuleIndex(system) subjects = [] else: writer.writeModuleIndex(system) subjects = system.rootobjects writer.writeIndividualFiles(subjects, options.htmlfunctionpages) if system.epytextproblems: def p(msg): system.msg(('epytext', 'epytext-summary'), msg, thresh=-1, topthresh=1) p("these %s objects' docstrings are not proper epytext:" % (len(system.epytextproblems), )) exitcode = 2 for fn in system.epytextproblems: p(' ' + fn) if options.outputpickle: system.msg('', 'saving output pickle to ' + options.outputpickle) # save again, with epytextproblems del system.options # don't persist the options f = open(options.outputpickle, 'wb') cPickle.dump(system, f, cPickle.HIGHEST_PROTOCOL) f.close() system.options = options if options.makeintersphinx: if not options.makehtml: subjects = system.rootobjects # Generate Sphinx inventory. sphinx_inventory = SphinxInventory( logger=system.msg, project_name=system.projectname, ) if not os.path.exists(options.htmloutput): os.makedirs(options.htmloutput) sphinx_inventory.generate( subjects=subjects, basepath=options.htmloutput, ) except: if options.pdb: import pdb pdb.post_mortem(sys.exc_traceback) raise return exitcode
class System: """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # Not assigned here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder defaultBuilder: Type[ASTBuilder] sourcebase: Optional[str] = None def __init__(self, options: Optional[Values] = None): self.allobjects: Dict[str, Documentable] = {} self.rootobjects: List[Documentable] = [] self.violations = 0 """The number of docstring problems found. This is used to determine whether to fail the build when using the --warnings-as-errors option, so it should only be increased for problems that the user can fix. """ if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.projectname = 'my project' self.docstring_syntax_errors: Set[str] = set() """FullNames of objects for which the docstring failed to parse.""" self.verboselevel = 0 self.needsnl = False self.once_msgs: Set[Tuple[str, str]] = set() self.unprocessed_modules: Set[Module] = set() self.module_count = 0 self.processing_modules: List[str] = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg) @property def root_names(self) -> Collection[str]: """The top-level package/module names in this system.""" return { obj.name for obj in self.rootobjects if isinstance(obj, (Module, Package)) } def verbosity(self, section: Union[str, Iterable[str]]) -> int: if isinstance(section, str): section = (section, ) delta: int = max( self.options.verbosity_details.get(sect, 0) for sect in section) base: int = self.options.verbosity return base + delta def progress(self, section: str, i: int, n: Optional[int], msg: str) -> None: if n is None: d = str(i) else: d = f'{i}/{n}' if self.verbosity(section) == 0 and sys.stdout.isatty(): print('\r' + d, msg, end='') sys.stdout.flush() if d == n: self.needsnl = False print() else: self.needsnl = True def msg(self, section: str, msg: str, thresh: int = 0, topthresh: int = 100, nonl: bool = False, wantsnl: bool = True, once: bool = False) -> None: if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh < 0: # Apidoc build messages are generated using negative threshold # and we have separate reporting for them, # on top of the logging system. self.violations += 1 if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print() print(msg, end='') if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print('') def objForFullName(self, fullName: str) -> Optional[Documentable]: return self.allobjects.get(fullName) def find_object(self, full_name: str) -> Optional[Documentable]: """Look up an object using a potentially outdated full name. A name can become outdated if the object is reparented: L{objForFullName()} will only be able to find it under its new name, but we might still have references to the old name. @param full_name: The fully qualified name of the object. @return: The object, or L{None} if the name is external (it does not match any of the roots of this system). @raise LookupError: If the object is not found, while its name does match one of the roots of this system. """ obj = self.objForFullName(full_name) if obj is not None: return obj # The object might have been reparented, in which case there will # be an alias at the original location; look for it using expandName(). name_parts = full_name.split('.', 1) for root_obj in self.rootobjects: if root_obj.name == name_parts[0]: obj = self.objForFullName(root_obj.expandName(name_parts[1])) if obj is not None: return obj raise LookupError(full_name) return None def _warning(self, current: Optional[Documentable], message: str, detail: str) -> None: if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print(fn, message, detail) def objectsOfType(self, cls: Type[T]) -> Iterator[T]: """Iterate over all instances of C{cls} present in the system. """ for o in self.allobjects.values(): if isinstance(o, cls): yield o def privacyClass(self, ob: Documentable) -> PrivacyClass: if ob.kind is None: return PrivacyClass.HIDDEN if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def addObject(self, obj: Documentable) -> None: """Add C{object} to the system.""" if obj.parent: obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) first = self.allobjects.setdefault(obj.fullName(), obj) if obj is not first: self.handleDuplicate(obj) # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod: _ModuleT, source_path: Path) -> None: if self.sourcebase is None: mod.sourceHref = None else: projBaseDir = mod.system.options.projectbasedirectory relative = source_path.relative_to(projBaseDir).as_posix() mod.sourceHref = f'{self.sourcebase}/{relative}' def addModule(self, modpath: Path, modname: str, parentPackage: Optional[_PackageT] = None) -> None: mod = self.Module(self, modname, parentPackage, modpath) self.addObject(mod) self.progress("addModule", len(self.allobjects), None, "modules and packages discovered") self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod, modpath) def ensureModule(self, module_full_name: str, modpath: Path) -> _ModuleT: try: module = self.allobjects[module_full_name] assert isinstance(module, Module) except KeyError: pass else: return module parent_package: Optional[_PackageT] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, parent_package, modpath) self.addObject(module) return module def ensurePackage(self, package_full_name: str) -> _PackageT: if package_full_name in self.allobjects: package = self.allobjects[package_full_name] assert isinstance(package, Package) return package parent_package: Optional[_PackageT] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, parent_package) self.addObject(package) return package def _introspectThing(self, thing: object, parent: Documentable, parentMod: _ModuleT) -> None: for k, v in thing.__dict__.items(): if (isinstance(v, (types.BuiltinFunctionType, types.FunctionType)) # In PyPy 7.3.1, functions from extensions are not # instances of the above abstract types. or v.__class__.__name__ == 'builtin_function_or_method'): f = self.Function(self, k, parent) f.parentMod = parentMod f.docstring = v.__doc__ f.decorators = None f.signature = Signature() self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod c.docstring = v.__doc__ self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, path: Path, module_full_name: str) -> None: spec = importlib.util.spec_from_file_location(module_full_name, path) py_mod = importlib.util.module_from_spec(spec) loader = spec.loader assert isinstance(loader, importlib.abc.Loader), loader loader.exec_module(py_mod) module = self.ensureModule(module_full_name, path) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) def addPackage(self, dirpath: str, parentPackage: Optional[_PackageT] = None) -> None: if not os.path.exists(dirpath): raise Exception(f"package path {dirpath!r} does not exist!") if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) for fname in sorted(os.listdir(dirpath)): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package: Optional[_PackageT], path: str) -> None: for suffix in importlib.machinery.all_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if suffix in importlib.machinery.EXTENSION_SUFFIXES: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = f'{package.fullName()}.{module_name}' else: module_full_name = module_name self.introspectModule(Path(path), module_full_name) elif suffix in importlib.machinery.SOURCE_SUFFIXES: self.addModule(Path(path), module_name, package) break def handleDuplicate(self, obj: Documentable) -> None: '''This is called when we see two objects with the same .fullName(), for example:: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fullName = obj.fullName() while (fullName + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[fullName] self._warning(obj.parent, "duplicate", str(prev)) def remove(o: Documentable) -> None: del self.allobjects[o.fullName()] oc = list(o.contents.values()) for c in oc: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o: Documentable) -> None: self.allobjects[o.fullName()] = o for c in o.contents.values(): readd(c) readd(prev) self.allobjects[fullName] = obj def getProcessedModule(self, modname: str) -> Optional[_ModuleT]: mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__') if not isinstance(mod, Module): return None if mod.state is ProcessingState.UNPROCESSED: self.processModule(mod) assert mod.state in (ProcessingState.PROCESSING, ProcessingState.PROCESSED) return mod def processModule(self, mod: _ModuleT) -> None: assert mod.state is ProcessingState.UNPROCESSED mod.state = ProcessingState.PROCESSING if mod.source_path is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.source_path) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s" % (self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = ProcessingState.PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) self.progress('process', self.module_count - len(self.unprocessed_modules), self.module_count, f"modules processed, {self.violations} warnings") def process(self) -> None: while self.unprocessed_modules: mod = next(iter(self.unprocessed_modules)) self.processModule(mod) self.postProcess() def postProcess(self) -> None: """Called when there are no more unprocessed modules. Analysis of relations between documentables can be done here, without the risk of drawing incorrect conclusions because modules were not fully processed yet. """ pass def fetchIntersphinxInventories(self, cache: CacheT) -> None: """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(cache, url)