def test_update_functional(inv_reader_nolog: sphinx.SphinxInventory) -> None: """ Functional test for updating from an empty inventory. """ payload = (b'some.module1 py:module -1 module1.html -\n' b'other.module2 py:module 0 module2.html Other description\n') # Patch URL loader to avoid hitting the system. content = b"""# Sphinx inventory version 2 # Project: some-name # Version: 2.0 # The rest of this file is compressed with zlib. """ + zlib.compress(payload) url = 'http://some.url/api/objects.inv' inv_reader_nolog.update({url: content}, url) assert 'http://some.url/api/module1.html' == inv_reader_nolog.getLink( 'some.module1') assert 'http://some.url/api/module2.html' == inv_reader_nolog.getLink( 'other.module2')
class System(object): """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # not done here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder sourcebase = None def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] self.moresystems = [] self.subsystems = [] self.urlprefix = '' if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [ ] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() # Once pickle support is removed, System should be # initialized with project name so that we can reuse intersphinx instance for # object.inv generation. self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def verbosity(self, section=None): if isinstance(section, str): section = (section, ) delta = max( [self.options.verbosity_details.get(sect, 0) for sect in section]) return self.options.verbosity + delta def progress(self, section, i, n, msg): if n is None: i = str(i) else: i = '%s/%s' % (i, n) if self.verbosity(section) == 0 and sys.stdout.isatty(): print('\r' + i, msg, end='') sys.stdout.flush() if i == n: self.needsnl = False print() else: self.needsnl = True def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False): if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print() print(msg, end='') if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print() def objForFullName(self, fullName): for system in [self] + self.moresystems: if fullName in system.allobjects: return system.allobjects[fullName] return None def _warning(self, current, type, detail): if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print(fn, type, detail) self.warnings.setdefault(type, []).append((fn, detail)) def objectsOfType(self, cls): """Iterate over all instances of C{cls} present in the system. """ for o in self.orderedallobjects: if isinstance(o, cls): yield o def privacyClass(self, ob): if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def __getstate__(self): d = self.__dict__.copy() del d['intersphinx'] return d def __setstate__(self, state): if 'abbrevmapping' not in state: state['abbrevmapping'] = {} # this is so very, very evil. # see doc/extreme-pickling-pain.txt for more. def lookup(name): for system in [self] + self.moresystems + self.subsystems: if name in system.allobjects: return system.allobjects[name] raise KeyError(name) self.__dict__.update(state) for system in [self] + self.moresystems + self.subsystems: if 'allobjects' not in system.__dict__: return for system in [self] + self.moresystems + self.subsystems: for obj in system.orderedallobjects: for k, v in obj.__dict__.copy().iteritems(): if k.startswith('$'): del obj.__dict__[k] obj.__dict__[k[1:]] = lookup(v) elif k.startswith('@'): n = [] for vv in v: if vv is None: n.append(None) else: n.append(lookup(vv)) del obj.__dict__[k] obj.__dict__[k[1:]] = n elif k.startswith('!'): n = {} for kk, vv in v.iteritems(): n[kk] = lookup(vv) del obj.__dict__[k] obj.__dict__[k[1:]] = n self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def addObject(self, obj): """Add C{object} to the system.""" if obj.parent: obj.parent.orderedcontents.append(obj) obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) self.orderedallobjects.append(obj) if obj.fullName() in self.allobjects: self.handleDuplicate(obj) else: self.allobjects[obj.fullName()] = obj # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an # "--add-package" argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod): if self.sourcebase is None: mod.sourceHref = None return projBaseDir = mod.system.options.projectbasedirectory if projBaseDir is not None: mod.sourceHref = (self.sourcebase + mod.filepath[len(projBaseDir):]) return trailing = [] dir, fname = os.path.split(mod.filepath) while os.path.exists(os.path.join(dir, '.svn')): dir, dirname = os.path.split(dir) trailing.append(dirname) # now trailing[-1] would be 'Divmod' in the above example del trailing[-1] trailing.reverse() trailing.append(fname) mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing) def addModule(self, modpath, modname, parentPackage=None): mod = self.Module(self, modname, None, parentPackage) self.addObject(mod) self.progress("addModule", len(self.orderedallobjects), None, "modules and packages discovered") mod.filepath = modpath self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod) def ensureModule(self, module_full_name): if module_full_name in self.allobjects: return self.allobjects[module_full_name] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, None, parent_package) self.addObject(module) return module def ensurePackage(self, package_full_name): if package_full_name in self.allobjects: return self.allobjects[package_full_name] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, None, parent_package) self.addObject(package) return package def _introspectThing(self, thing, parent, parentMod): for k, v in thing.__dict__.iteritems(): if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))): f = self.Function(self, k, v.__doc__, parent) f.parentMod = parentMod f.decorators = None f.argspec = ((), None, None, ()) self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, v.__doc__, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, py_mod, module_full_name): module = self.ensureModule(module_full_name) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) print(py_mod) def addPackage(self, dirpath, parentPackage=None): if not os.path.exists(dirpath): raise Exception("package path %r does not exist!" % (dirpath, )) if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) package.filepath = dirpath self.setSourceHref(package) for fname in os.listdir(dirpath): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package, path): for (suffix, mode, type) in imp.get_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if type == imp.C_EXTENSION: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = "%s.%s" % (package.fullName(), module_name) else: module_full_name = module_name py_mod = imp.load_module(module_full_name, open(path, 'rb'), path, (suffix, mode, type)) self.introspectModule(py_mod, module_full_name) elif type == imp.PY_SOURCE: self.addModule(path, module_name, package) break def handleDuplicate(self, obj): '''This is called when we see two objects with the same .fullName(), for example: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fn = obj.fullName() while (fn + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[obj.fullName()] self._warning(obj.parent, "duplicate", prev) def remove(o): del self.allobjects[o.fullName()] for c in o.orderedcontents: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o): self.allobjects[o.fullName()] = o for c in o.orderedcontents: readd(c) readd(prev) self.allobjects[obj.fullName()] = obj return obj def getProcessedModule(self, modname): mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__').parent if not isinstance(mod, Module): return None if mod.state == UNPROCESSED: self.processModule(mod) return mod def processModule(self, mod): assert mod.state == UNPROCESSED mod.state = PROCESSING if getattr(mod, 'filepath', None) is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.filepath) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s" % (self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) self.progress( 'process', self.module_count - len(self.unprocessed_modules), self.module_count, "modules processed %s warnings" % (sum(len(v) for v in self.warnings.itervalues()), )) def process(self): while self.unprocessed_modules: mod = iter(self.unprocessed_modules).next() self.processModule(mod) def fetchIntersphinxInventories(self): """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(url)
class System: """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # Not assigned here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder defaultBuilder: Type[ASTBuilder] sourcebase = None def __init__(self, options=None): self.allobjects = {} self.rootobjects = [] self.warnings = {} self.packages = [] if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.docstring_syntax_errors = set() """FullNames of objects for which the docstring failed to parse.""" self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg) def verbosity(self, section=None): if isinstance(section, str): section = (section, ) delta = max( self.options.verbosity_details.get(sect, 0) for sect in section) return self.options.verbosity + delta def progress(self, section, i, n, msg): if n is None: i = str(i) else: i = f'{i}/{n}' if self.verbosity(section) == 0 and sys.stdout.isatty(): print('\r' + i, msg, end='') sys.stdout.flush() if i == n: self.needsnl = False print() else: self.needsnl = True def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False): if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print() print(msg, end='') if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print('') def objForFullName(self, fullName): return self.allobjects.get(fullName) def _warning(self, current, message, detail): if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print(fn, message, detail) self.warnings.setdefault(message, []).append((fn, detail)) def objectsOfType(self, cls): """Iterate over all instances of C{cls} present in the system. """ for o in self.allobjects.values(): if isinstance(o, cls): yield o def privacyClass(self, ob): if ob.kind is None: return PrivacyClass.HIDDEN if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def addObject(self, obj): """Add C{object} to the system.""" fullName = obj.fullName() if obj.parent and obj.parent.fullName() != fullName: obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) if fullName in self.allobjects: self.handleDuplicate(obj) else: self.allobjects[fullName] = obj # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an # "--add-package" argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod): if self.sourcebase is None: mod.sourceHref = None else: projBaseDir = mod.system.options.projectbasedirectory mod.sourceHref = (self.sourcebase + mod.filepath[len(projBaseDir):]) def addModule(self, modpath, modname, parentPackage=None): mod = self.Module(self, modname, parentPackage) self.addObject(mod) self.progress("addModule", len(self.allobjects), None, "modules and packages discovered") mod.filepath = modpath self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod) def ensureModule(self, module_full_name): if module_full_name in self.allobjects: return self.allobjects[module_full_name] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, parent_package) self.addObject(module) return module def ensurePackage(self, package_full_name): if package_full_name in self.allobjects: return self.allobjects[package_full_name] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, parent_package) self.addObject(package) return package def _introspectThing(self, thing, parent, parentMod): for k, v in thing.__dict__.items(): if isinstance(v, (types.BuiltinFunctionType, types.FunctionType)): f = self.Function(self, k, parent) f.parentMod = parentMod f.docstring = v.__doc__ f.decorators = None f.argspec = ((), None, None, ()) self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod c.docstring = v.__doc__ self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, py_mod, module_full_name): module = self.ensureModule(module_full_name) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) def addPackage(self, dirpath, parentPackage=None): if not os.path.exists(dirpath): raise Exception(f"package path {dirpath!r} does not exist!") if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) package.filepath = dirpath self.setSourceHref(package) for fname in sorted(os.listdir(dirpath)): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package, path): for (suffix, mode, impl) in imp.get_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if impl == imp.C_EXTENSION: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = f'{package.fullName()}.{module_name}' else: module_full_name = module_name py_mod = imp.load_module(module_full_name, open(path, 'rb'), path, (suffix, mode, impl)) self.introspectModule(py_mod, module_full_name) elif impl == imp.PY_SOURCE: self.addModule(path, module_name, package) break def handleDuplicate(self, obj): '''This is called when we see two objects with the same .fullName(), for example: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fullName = obj.fullName() while (fullName + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[fullName] self._warning(obj.parent, "duplicate", prev) def remove(o): del self.allobjects[o.fullName()] oc = list(o.contents.values()) for c in oc: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o): self.allobjects[o.fullName()] = o for c in o.contents.values(): readd(c) readd(prev) self.allobjects[fullName] = obj def getProcessedModule(self, modname): mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__').parent if not isinstance(mod, Module): return None if mod.state is ProcessingState.UNPROCESSED: self.processModule(mod) return mod def processModule(self, mod): assert mod.state is ProcessingState.UNPROCESSED mod.state = ProcessingState.PROCESSING if getattr(mod, 'filepath', None) is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.filepath) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s" % (self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = ProcessingState.PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) num_warnings = sum(len(v) for v in self.warnings.values()) self.progress('process', self.module_count - len(self.unprocessed_modules), self.module_count, f"modules processed {num_warnings} warnings") def process(self): while self.unprocessed_modules: mod = next(iter(self.unprocessed_modules)) self.processModule(mod) def fetchIntersphinxInventories(self, cache): """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(cache, url)
class System: """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # Not assigned here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder defaultBuilder: Type[ASTBuilder] sourcebase: Optional[str] = None def __init__(self, options: Optional[Values] = None): self.allobjects: Dict[str, Documentable] = {} self.rootobjects: List[Documentable] = [] self.violations = 0 """The number of docstring problems found. This is used to determine whether to fail the build when using the --warnings-as-errors option, so it should only be increased for problems that the user can fix. """ if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.projectname = 'my project' self.docstring_syntax_errors: Set[str] = set() """FullNames of objects for which the docstring failed to parse.""" self.verboselevel = 0 self.needsnl = False self.once_msgs: Set[Tuple[str, str]] = set() self.unprocessed_modules: Set[Module] = set() self.module_count = 0 self.processing_modules: List[str] = [] self.buildtime = datetime.datetime.now() self.intersphinx = SphinxInventory(logger=self.msg) @property def root_names(self) -> Collection[str]: """The top-level package/module names in this system.""" return { obj.name for obj in self.rootobjects if isinstance(obj, (Module, Package)) } def verbosity(self, section: Union[str, Iterable[str]]) -> int: if isinstance(section, str): section = (section, ) delta: int = max( self.options.verbosity_details.get(sect, 0) for sect in section) base: int = self.options.verbosity return base + delta def progress(self, section: str, i: int, n: Optional[int], msg: str) -> None: if n is None: d = str(i) else: d = f'{i}/{n}' if self.verbosity(section) == 0 and sys.stdout.isatty(): print('\r' + d, msg, end='') sys.stdout.flush() if d == n: self.needsnl = False print() else: self.needsnl = True def msg(self, section: str, msg: str, thresh: int = 0, topthresh: int = 100, nonl: bool = False, wantsnl: bool = True, once: bool = False) -> None: if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh < 0: # Apidoc build messages are generated using negative threshold # and we have separate reporting for them, # on top of the logging system. self.violations += 1 if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print() print(msg, end='') if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print('') def objForFullName(self, fullName: str) -> Optional[Documentable]: return self.allobjects.get(fullName) def find_object(self, full_name: str) -> Optional[Documentable]: """Look up an object using a potentially outdated full name. A name can become outdated if the object is reparented: L{objForFullName()} will only be able to find it under its new name, but we might still have references to the old name. @param full_name: The fully qualified name of the object. @return: The object, or L{None} if the name is external (it does not match any of the roots of this system). @raise LookupError: If the object is not found, while its name does match one of the roots of this system. """ obj = self.objForFullName(full_name) if obj is not None: return obj # The object might have been reparented, in which case there will # be an alias at the original location; look for it using expandName(). name_parts = full_name.split('.', 1) for root_obj in self.rootobjects: if root_obj.name == name_parts[0]: obj = self.objForFullName(root_obj.expandName(name_parts[1])) if obj is not None: return obj raise LookupError(full_name) return None def _warning(self, current: Optional[Documentable], message: str, detail: str) -> None: if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print(fn, message, detail) def objectsOfType(self, cls: Type[T]) -> Iterator[T]: """Iterate over all instances of C{cls} present in the system. """ for o in self.allobjects.values(): if isinstance(o, cls): yield o def privacyClass(self, ob: Documentable) -> PrivacyClass: if ob.kind is None: return PrivacyClass.HIDDEN if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def addObject(self, obj: Documentable) -> None: """Add C{object} to the system.""" if obj.parent: obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) first = self.allobjects.setdefault(obj.fullName(), obj) if obj is not first: self.handleDuplicate(obj) # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod: _ModuleT, source_path: Path) -> None: if self.sourcebase is None: mod.sourceHref = None else: projBaseDir = mod.system.options.projectbasedirectory relative = source_path.relative_to(projBaseDir).as_posix() mod.sourceHref = f'{self.sourcebase}/{relative}' def addModule(self, modpath: Path, modname: str, parentPackage: Optional[_PackageT] = None) -> None: mod = self.Module(self, modname, parentPackage, modpath) self.addObject(mod) self.progress("addModule", len(self.allobjects), None, "modules and packages discovered") self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod, modpath) def ensureModule(self, module_full_name: str, modpath: Path) -> _ModuleT: try: module = self.allobjects[module_full_name] assert isinstance(module, Module) except KeyError: pass else: return module parent_package: Optional[_PackageT] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, parent_package, modpath) self.addObject(module) return module def ensurePackage(self, package_full_name: str) -> _PackageT: if package_full_name in self.allobjects: package = self.allobjects[package_full_name] assert isinstance(package, Package) return package parent_package: Optional[_PackageT] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, parent_package) self.addObject(package) return package def _introspectThing(self, thing: object, parent: Documentable, parentMod: _ModuleT) -> None: for k, v in thing.__dict__.items(): if (isinstance(v, (types.BuiltinFunctionType, types.FunctionType)) # In PyPy 7.3.1, functions from extensions are not # instances of the above abstract types. or v.__class__.__name__ == 'builtin_function_or_method'): f = self.Function(self, k, parent) f.parentMod = parentMod f.docstring = v.__doc__ f.decorators = None f.signature = Signature() self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod c.docstring = v.__doc__ self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, path: Path, module_full_name: str) -> None: spec = importlib.util.spec_from_file_location(module_full_name, path) py_mod = importlib.util.module_from_spec(spec) loader = spec.loader assert isinstance(loader, importlib.abc.Loader), loader loader.exec_module(py_mod) module = self.ensureModule(module_full_name, path) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) def addPackage(self, dirpath: str, parentPackage: Optional[_PackageT] = None) -> None: if not os.path.exists(dirpath): raise Exception(f"package path {dirpath!r} does not exist!") if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) for fname in sorted(os.listdir(dirpath)): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package: Optional[_PackageT], path: str) -> None: for suffix in importlib.machinery.all_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if suffix in importlib.machinery.EXTENSION_SUFFIXES: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = f'{package.fullName()}.{module_name}' else: module_full_name = module_name self.introspectModule(Path(path), module_full_name) elif suffix in importlib.machinery.SOURCE_SUFFIXES: self.addModule(Path(path), module_name, package) break def handleDuplicate(self, obj: Documentable) -> None: '''This is called when we see two objects with the same .fullName(), for example:: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fullName = obj.fullName() while (fullName + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[fullName] self._warning(obj.parent, "duplicate", str(prev)) def remove(o: Documentable) -> None: del self.allobjects[o.fullName()] oc = list(o.contents.values()) for c in oc: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o: Documentable) -> None: self.allobjects[o.fullName()] = o for c in o.contents.values(): readd(c) readd(prev) self.allobjects[fullName] = obj def getProcessedModule(self, modname: str) -> Optional[_ModuleT]: mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__') if not isinstance(mod, Module): return None if mod.state is ProcessingState.UNPROCESSED: self.processModule(mod) assert mod.state in (ProcessingState.PROCESSING, ProcessingState.PROCESSED) return mod def processModule(self, mod: _ModuleT) -> None: assert mod.state is ProcessingState.UNPROCESSED mod.state = ProcessingState.PROCESSING if mod.source_path is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.source_path) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s" % (self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = ProcessingState.PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) self.progress('process', self.module_count - len(self.unprocessed_modules), self.module_count, f"modules processed, {self.violations} warnings") def process(self) -> None: while self.unprocessed_modules: mod = next(iter(self.unprocessed_modules)) self.processModule(mod) self.postProcess() def postProcess(self) -> None: """Called when there are no more unprocessed modules. Analysis of relations between documentables can be done here, without the risk of drawing incorrect conclusions because modules were not fully processed yet. """ pass def fetchIntersphinxInventories(self, cache: CacheT) -> None: """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(cache, url)
class System(object): """A collection of related documentable objects. PyDoctor documents collections of objects, often the contents of a package. """ Class = Class Module = Module Package = Package Function = Function Attribute = Attribute # not done here for circularity reasons: #defaultBuilder = astbuilder.ASTBuilder sourcebase = None def __init__(self, options=None): self.allobjects = {} self.orderedallobjects = [] self.rootobjects = [] self.warnings = {} self.packages = [] self.moresystems = [] self.subsystems = [] self.urlprefix = '' if options: self.options = options else: from pydoctor.driver import parse_args self.options, _ = parse_args([]) self.options.verbosity = 3 self.abbrevmapping = {} self.projectname = 'my project' self.epytextproblems = [] # fullNames of objects that failed to epytext properly self.verboselevel = 0 self.needsnl = False self.once_msgs = set() self.unprocessed_modules = set() self.module_count = 0 self.processing_modules = [] self.buildtime = datetime.datetime.now() # Once pickle support is removed, System should be # initialized with project name so that we can reuse intersphinx instance for # object.inv generation. self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def verbosity(self, section=None): if isinstance(section, str): section = (section,) delta = max([self.options.verbosity_details.get(sect, 0) for sect in section]) return self.options.verbosity + delta def progress(self, section, i, n, msg): if n is None: i = str(i) else: i = '%s/%s'%(i,n) if self.verbosity(section) == 0 and sys.stdout.isatty(): print '\r'+i, msg, sys.stdout.flush() if i == n: self.needsnl = False print else: self.needsnl = True def msg(self, section, msg, thresh=0, topthresh=100, nonl=False, wantsnl=True, once=False): if once: if (section, msg) in self.once_msgs: return else: self.once_msgs.add((section, msg)) if thresh <= self.verbosity(section) <= topthresh: if self.needsnl and wantsnl: print print msg, if nonl: self.needsnl = True sys.stdout.flush() else: self.needsnl = False print def objForFullName(self, fullName): for system in [self] + self.moresystems: if fullName in system.allobjects: return system.allobjects[fullName] return None def _warning(self, current, type, detail): if current is not None: fn = current.fullName() else: fn = '<None>' if self.options.verbosity > 0: print fn, type, detail self.warnings.setdefault(type, []).append((fn, detail)) def objectsOfType(self, cls): """Iterate over all instances of C{cls} present in the system. """ for o in self.orderedallobjects: if isinstance(o, cls): yield o def privacyClass(self, ob): if ob.name.startswith('_') and \ not (ob.name.startswith('__') and ob.name.endswith('__')): return PrivacyClass.PRIVATE return PrivacyClass.VISIBLE def __getstate__(self): d = self.__dict__.copy() del d['intersphinx'] return d def __setstate__(self, state): if 'abbrevmapping' not in state: state['abbrevmapping'] = {} # this is so very, very evil. # see doc/extreme-pickling-pain.txt for more. def lookup(name): for system in [self] + self.moresystems + self.subsystems: if name in system.allobjects: return system.allobjects[name] raise KeyError, name self.__dict__.update(state) for system in [self] + self.moresystems + self.subsystems: if 'allobjects' not in system.__dict__: return for system in [self] + self.moresystems + self.subsystems: for obj in system.orderedallobjects: for k, v in obj.__dict__.copy().iteritems(): if k.startswith('$'): del obj.__dict__[k] obj.__dict__[k[1:]] = lookup(v) elif k.startswith('@'): n = [] for vv in v: if vv is None: n.append(None) else: n.append(lookup(vv)) del obj.__dict__[k] obj.__dict__[k[1:]] = n elif k.startswith('!'): n = {} for kk, vv in v.iteritems(): n[kk] = lookup(vv) del obj.__dict__[k] obj.__dict__[k[1:]] = n self.intersphinx = SphinxInventory(logger=self.msg, project_name=self.projectname) def addObject(self, obj): """Add C{object} to the system.""" if obj.parent: obj.parent.orderedcontents.append(obj) obj.parent.contents[obj.name] = obj else: self.rootobjects.append(obj) self.orderedallobjects.append(obj) if obj.fullName() in self.allobjects: self.handleDuplicate(obj) else: self.allobjects[obj.fullName()] = obj # if we assume: # # - that svn://divmod.org/trunk is checked out into ~/src/Divmod # # - that http://divmod.org/trac/browser/trunk is the trac URL to the # above directory # # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an # "--add-package" argument # # we want to work out the sourceHref for nevow.flat.ten. the answer # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py. # # we can work this out by finding that Divmod is the top of the svn # checkout, and posixpath.join-ing the parts of the filePath that # follows that. # # http://divmod.org/trac/browser/trunk # ~/src/Divmod/Nevow/nevow/flat/ten.py def setSourceHref(self, mod): if self.sourcebase is None: mod.sourceHref = None return projBaseDir = mod.system.options.projectbasedirectory if projBaseDir is not None: mod.sourceHref = ( self.sourcebase + mod.filepath[len(projBaseDir):]) return trailing = [] dir, fname = os.path.split(mod.filepath) while os.path.exists(os.path.join(dir, '.svn')): dir, dirname = os.path.split(dir) trailing.append(dirname) # now trailing[-1] would be 'Divmod' in the above example del trailing[-1] trailing.reverse() trailing.append(fname) mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing) def addModule(self, modpath, modname, parentPackage=None): mod = self.Module(self, modname, None, parentPackage) self.addObject(mod) self.progress( "addModule", len(self.orderedallobjects), None, "modules and packages discovered") mod.filepath = modpath self.unprocessed_modules.add(mod) self.module_count += 1 self.setSourceHref(mod) def ensureModule(self, module_full_name): if module_full_name in self.allobjects: return self.allobjects[module_full_name] if '.' in module_full_name: parent_name, module_name = module_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None module_name = module_full_name module = self.Module(self, module_name, None, parent_package) self.addObject(module) return module def ensurePackage(self, package_full_name): if package_full_name in self.allobjects: return self.allobjects[package_full_name] if '.' in package_full_name: parent_name, package_name = package_full_name.rsplit('.', 1) parent_package = self.ensurePackage(parent_name) else: parent_package = None package_name = package_full_name package = self.Package(self, package_name, None, parent_package) self.addObject(package) return package def _introspectThing(self, thing, parent, parentMod): for k, v in thing.__dict__.iteritems(): if isinstance(v, (types.BuiltinFunctionType, type(dict.keys))): f = self.Function(self, k, v.__doc__, parent) f.parentMod = parentMod f.decorators = None f.argspec = ((), None, None, ()) self.addObject(f) elif isinstance(v, type): c = self.Class(self, k, v.__doc__, parent) c.bases = [] c.baseobjects = [] c.rawbases = [] c.parentMod = parentMod self.addObject(c) self._introspectThing(v, c, parentMod) def introspectModule(self, py_mod, module_full_name): module = self.ensureModule(module_full_name) module.docstring = py_mod.__doc__ self._introspectThing(py_mod, module, module) print py_mod def addPackage(self, dirpath, parentPackage=None): if not os.path.exists(dirpath): raise Exception("package path %r does not exist!" %(dirpath,)) if not os.path.exists(os.path.join(dirpath, '__init__.py')): raise Exception("you must pass a package directory to " "addPackage") if parentPackage: prefix = parentPackage.fullName() + '.' else: prefix = '' package_name = os.path.basename(dirpath) package_full_name = prefix + package_name package = self.ensurePackage(package_full_name) package.filepath = dirpath self.setSourceHref(package) for fname in os.listdir(dirpath): fullname = os.path.join(dirpath, fname) if os.path.isdir(fullname): initname = os.path.join(fullname, '__init__.py') if os.path.exists(initname): self.addPackage(fullname, package) elif not fname.startswith('.'): self.addModuleFromPath(package, fullname) def addModuleFromPath(self, package, path): for (suffix, mode, type) in imp.get_suffixes(): if not path.endswith(suffix): continue module_name = os.path.basename(path[:-len(suffix)]) if type == imp.C_EXTENSION: if not self.options.introspect_c_modules: continue if package is not None: module_full_name = "%s.%s" % ( package.fullName(), module_name) else: module_full_name = module_name py_mod = imp.load_module( module_full_name, open(path, 'rb'), path, (suffix, mode, type)) self.introspectModule(py_mod, module_full_name) elif type == imp.PY_SOURCE: self.addModule(path, module_name, package) break def handleDuplicate(self, obj): '''This is called when we see two objects with the same .fullName(), for example: class C: if something: def meth(self): implementation 1 else: def meth(self): implementation 2 The default is that the second definition "wins". ''' i = 0 fn = obj.fullName() while (fn + ' ' + str(i)) in self.allobjects: i += 1 prev = self.allobjects[obj.fullName()] self._warning(obj.parent, "duplicate", prev) def remove(o): del self.allobjects[o.fullName()] for c in o.orderedcontents: remove(c) remove(prev) prev.name = obj.name + ' ' + str(i) def readd(o): self.allobjects[o.fullName()] = o for c in o.orderedcontents: readd(c) readd(prev) self.allobjects[obj.fullName()] = obj return obj def getProcessedModule(self, modname): mod = self.allobjects.get(modname) if mod is None: return None if isinstance(mod, Package): return self.getProcessedModule(modname + '.__init__').parent if not isinstance(mod, Module): return None if mod.state == UNPROCESSED: self.processModule(mod) return mod def processModule(self, mod): assert mod.state == UNPROCESSED mod.state = PROCESSING if getattr(mod, 'filepath', None) is None: return builder = self.defaultBuilder(self) ast = builder.parseFile(mod.filepath) if ast: self.processing_modules.append(mod.fullName()) self.msg("processModule", "processing %s"%(self.processing_modules), 1) builder.processModuleAST(ast, mod) mod.state = PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() self.unprocessed_modules.remove(mod) self.progress( 'process', self.module_count - len(self.unprocessed_modules), self.module_count, "modules processed %s warnings"%( sum(len(v) for v in self.warnings.itervalues()),)) def process(self): while self.unprocessed_modules: mod = iter(self.unprocessed_modules).next() self.processModule(mod) def fetchIntersphinxInventories(self): """ Download and parse intersphinx inventories based on configuration. """ for url in self.options.intersphinx: self.intersphinx.update(url)