def getApi(self): field = "api[%s]" % self.id apidata = self.project.getCache().read(field, self.getModificationTime()) if markdown is None: raise JasyError("Missing Markdown feature to convert package docs into HTML.") if apidata is None: apidata = ApiData(self.id) apidata.main["type"] = "Package" apidata.main["doc"] = markdown(self.getText()) self.project.getCache().store(field, apidata, self.getModificationTime()) return apidata
def getApi(self): field = "api[%s]" % self.__id apidata = self.__cache.read(field, self.__mtime) if apidata is None: apidata = ApiData(self.getTree(cleanup=False), self.__name) self.__cache.store(field, apidata, self.__mtime) return apidata
def collect(self, internals=False, privates=False): # # Collecting Original Data # logging.debug("Collecting Classes...") apiData = {} for project in self.session.getProjects(): classes = project.getClasses() for className in classes: apiData[className] = classes[className].getApi() # # Collecting Source Code # logging.debug("Highlighting Code...") highlighted = {} for project in self.session.getProjects(): classes = project.getClasses() for className in classes: highlighted[className] = classes[className].getHighlightedCode() # # Filter Internals/Privates # logging.debug("Filtering Items...") def isVisible(entry): if "visibility" in entry: visibility = entry["visibility"] if visibility == "private" and not privates: return False if visibility == "internal" and not internals: return False return True def filterInternalsPrivates(classApi, field): data = getattr(classApi, field, None) if data: for name in list(data): if not isVisible(data[name]): del data[name] for className in apiData: filterInternalsPrivates(apiData[className], "statics") filterInternalsPrivates(apiData[className], "members") # # Attaching Links to Source Code (Lines) # Building Documentation Summaries # logging.debug("Tweaking Output...") for className in apiData: classApi = apiData[className] constructData = getattr(classApi, "construct", None) if constructData is not None: if "line" in constructData: constructData["sourceLink"] = "source:%s~%s" % (className, constructData["line"]) for section in ("properties", "events", "statics", "members"): sectionData = getattr(classApi, section, None) if sectionData is not None: for name in sectionData: if "line" in sectionData[name]: sectionData[name]["sourceLink"] = "source:%s~%s" % (className, sectionData[name]["line"]) if "doc" in sectionData[name]: summary = extractSummary(sectionData[name]["doc"]) if summary is not None: sectionData[name]["summary"] = summary # # Including Mixins / IncludedBy # logging.debug("Resolving Mixins...") # Just used temporary to keep track of which classes are merged mergedClasses = set() def getApi(className): classApi = apiData[className] if className in mergedClasses: return classApi classIncludes = getattr(classApi, "includes", None) if classIncludes: for mixinName in classIncludes: mixinApi = apiData[mixinName] if not hasattr(mixinApi, "includedBy"): mixinApi.includedBy = set() mixinApi.includedBy.add(className) mergeMixin(className, mixinName, classApi, getApi(mixinName)) mergedClasses.add(className) return classApi for className in apiData: apiData[className] = getApi(className) # # Connection Interfaces / ImplementedBy # logging.debug("Connecting Interfaces...") for className in apiData: classApi = getApi(className) if not hasattr(classApi, "main"): continue classType = classApi.main["type"] if classType == "core.Class": classImplements = getattr(classApi, "implements", None) if classImplements: for interfaceName in classImplements: interfaceApi = apiData[interfaceName] implementedBy = getattr(interfaceApi, "implementedBy", None) if not implementedBy: implementedBy = interfaceApi.implementedBy = [] implementedBy.append(className) connectInterface(className, interfaceName, classApi, interfaceApi) # # Connecting Uses / UsedBy # logging.debug("Collecting Use Patterns...") # This matches all uses with the known classes and only keeps them if matched allClasses = set(list(apiData)) for className in apiData: uses = apiData[className].uses # Rebuild use list cleanUses = set() for use in uses: if use != className and use in allClasses: cleanUses.add(use) useEntry = apiData[use] if not hasattr(useEntry, "usedBy"): useEntry.usedBy = set() useEntry.usedBy.add(className) apiData[className].uses = cleanUses # # Merging Named Classes # logging.debug("Merging Named Classes...") for className in list(apiData): classApi = apiData[className] destName = classApi.main["name"] if destName is not None and destName != className: if destName in apiData: destApi = apiData[destName] destApi.main["from"].append(className) else: destApi = apiData[destName] = ApiData(destName) destApi.main = { "type" : "Extend", "name" : destName, "from" : [className], "doc" : "Extensions for %s" % destName } # If there is a "main" tag found in the class use its API description if "tags" in classApi.main and classApi.main["tags"] is not None and "main" in classApi.main["tags"]: if "doc" in classApi.main and destApi.main["doc"] == "Extensions for %s" % destName: destApi.main["doc"] = classApi.main["doc"] classApi.main["extension"] = True # Read existing data construct = getattr(classApi, "construct", None) statics = getattr(classApi, "statics", None) members = getattr(classApi, "members", None) if construct is not None: if hasattr(destApi, "construct"): logging.warn("Overriding constructor in extension %s by %s", destName, className) destApi.construct = copy.copy(construct) if statics is not None: if not hasattr(destApi, "statics"): destApi.statics = {} for staticName in statics: destApi.statics[staticName] = copy.copy(statics[staticName]) destApi.statics[staticName]["from"] = className destApi.statics[staticName]["fromLink"] = "static:%s~%s" % (className, staticName) if members is not None: if not hasattr(destApi, "members"): destApi.members = {} for memberName in members: destApi.members[memberName] = copy.copy(members[memberName]) destApi.members[memberName]["from"] = className destApi.members[memberName]["fromLink"] = "member:%s~%s" % (className, memberName) # # Collecting errors # for className in sorted(apiData): classApi = apiData[className] errors = [] if isErrornous(classApi.main): errors.append({ "kind": "Main", "name": None, "line": 1 }) if hasattr(classApi, "construct"): if isErrornous(classApi.construct): errors.append({ "kind": "Constructor", "name": None, "line": classApi.construct["line"] }) for section in ("statics", "members", "properties", "events"): items = getattr(classApi, section, {}) for itemName in items: item = items[itemName] if isErrornous(item): errors.append({ "kind": itemMap[section], "name": itemName, "line": item["line"] }) if errors: logging.warn("API documentation errors in %s", className) errorsSorted = sorted(errors, key=lambda entry: entry["line"]) for entry in errorsSorted: if entry["name"]: logging.warn("- %s: %s (line %s)", entry["kind"], entry["name"], entry["line"]) else: logging.warn("- %s (line %s)", entry["kind"], entry["line"]) classApi.errors = errorsSorted # # Building Search Index # logging.debug("Building Search Index") search = {} def addSearch(classApi, field): data = getattr(classApi, field, None) if data: for name in data: if not name in search: search[name] = set() search[name].add(className) for className in apiData: classApi = apiData[className] addSearch(classApi, "statics") addSearch(classApi, "members") addSearch(classApi, "properties") addSearch(classApi, "events") # # Post Process (dict to sorted list) # for className in sorted(apiData): classApi = apiData[className] convertTags(classApi.main) construct = getattr(classApi, "construct", None) if construct: convertFunction(construct) convertTags(construct) for section in ("statics", "members", "properties", "events"): items = getattr(classApi, section, None) if items: sortedList = [] for itemName in sorted(items): item = items[itemName] item["name"] = itemName if "type" in item and item["type"] == "Function": convertFunction(item) convertTags(item) sortedList.append(item) setattr(classApi, section, sortedList) # # Collecting Package Docs # logging.debug("Collecting Package Docs...") packages = set() for project in self.session.getProjects(): docs = project.getDocs() for packageName in docs: packages.add(packageName) apiData[packageName] = ApiData(packageName) apiData[packageName].main = { "type" : "Package", "name" : packageName, "doc" : docs[packageName] } for className in list(apiData): # Auto create API data for all packages in between splits = className.split(".") packageName = splits[0] for split in splits[1:]: if not packageName in apiData: packages.add(packageName) logging.debug("Creating missing package doc entry: %s" % packageName) apiData[packageName] = ApiData(packageName) apiData[packageName].main = { "type" : "Package", "name" : packageName } packageName = "%s.%s" % (packageName, split) # Register class inside package "content" lastPackage = ".".join(splits[:-1]) if lastPackage: if not hasattr(apiData[lastPackage], "content"): apiData[lastPackage].content = [] classPkgEntry = { "name": splits[-1], "link": className } classMain = apiData[className].main if "doc" in classMain and classMain["doc"]: summary = extractSummary(classMain["doc"]) if summary: classPkgEntry["summary"] = summary apiData[lastPackage].content.append(classPkgEntry) # Sort package content for packageName in packages: apiData[packageName].content.sort(key=lambda entry: entry["name"]) # # Writing API Index # logging.debug("Building Index...") index = {} for className in apiData: classApi = apiData[className] mainInfo = classApi.main # Create missing packages for className current = index for split in className.split("."): if not split in current: current[split] = {} current = current[split] # Store current type current["$type"] = mainInfo["type"] # # Return # return apiData, highlighted, index, search
def process(self, apiData, classFilter=None, internals=False, privates=False, printErrors=True): knownClasses = set(list(apiData)) # # Attaching Links to Source Code (Lines) # Building Documentation Summaries # info("Adding Source Links...") for className in apiData: classApi = apiData[className] constructData = getattr(classApi, "construct", None) if constructData is not None: if "line" in constructData: constructData["sourceLink"] = "source:%s~%s" % (className, constructData["line"]) for section in ("properties", "events", "statics", "members"): sectionData = getattr(classApi, section, None) if sectionData is not None: for name in sectionData: if "line" in sectionData[name]: sectionData[name]["sourceLink"] = "source:%s~%s" % (className, sectionData[name]["line"]) # # Including Mixins / IncludedBy # info("Resolving Mixins...") # Just used temporary to keep track of which classes are merged mergedClasses = set() def getApi(className): classApi = apiData[className] if className in mergedClasses: return classApi classIncludes = getattr(classApi, "includes", None) if classIncludes: for mixinName in classIncludes: if not mixinName in apiData: error("Invalid mixin %s in class %s", className, mixinName) continue mixinApi = apiData[mixinName] if not hasattr(mixinApi, "includedBy"): mixinApi.includedBy = set() mixinApi.includedBy.add(className) mergeMixin(className, mixinName, classApi, getApi(mixinName)) mergedClasses.add(className) return classApi for className in apiData: apiData[className] = getApi(className) # # Checking links # info("Checking Links...") additionalTypes = ("Call", "Identifier", "Map", "Integer", "Node", "Element") def checkInternalLink(link, className): match = internalLinkParse.match(link) if not match: return 'Invalid link "#%s"' % link if match.group(3) is not None: className = match.group(3) if not className in knownClasses and not className in apiData: return 'Invalid class in link "#%s"' % link # Accept all section/item values for named classes, # as it might be pretty complicated to verify this here. if not className in apiData: return True classApi = apiData[className] sectionName = match.group(2) itemName = match.group(5) if itemName is None: return True if sectionName is not None: if not sectionName in linkMap: return 'Invalid section in link "#%s"' % link section = getattr(classApi, linkMap[sectionName], None) if section is None: return 'Invalid section in link "#%s"' % link else: if itemName in section: return True return 'Invalid item in link "#%s"' % link for sectionName in ("statics", "members", "properties", "events"): section = getattr(classApi, sectionName, None) if section and itemName in section: return True return 'Invalid item link "#%s"' % link def checkLinksInItem(item): # Process types if "type" in item: if item["type"] == "Function": # Check param types if "params" in item: for paramName in item["params"]: paramEntry = item["params"][paramName] if "type" in paramEntry: for paramTypeEntry in paramEntry["type"]: if not paramTypeEntry["name"] in knownClasses and not paramTypeEntry["name"] in additionalTypes and not ("builtin" in paramTypeEntry or "pseudo" in paramTypeEntry): item["errornous"] = True error('- Invalid param type "%s" in %s at line %s', paramTypeEntry["name"], className, item["line"]) if not "pseudo" in paramTypeEntry and paramTypeEntry["name"] in knownClasses: paramTypeEntry["linkable"] = True # Check return types if "returns" in item: for returnTypeEntry in item["returns"]: if not returnTypeEntry["name"] in knownClasses and not returnTypeEntry["name"] in additionalTypes and not ("builtin" in returnTypeEntry or "pseudo" in returnTypeEntry): item["errornous"] = True error('- Invalid return type "%s" in %s at line %s', returnTypeEntry["name"], className, item["line"]) if not "pseudo" in returnTypeEntry and returnTypeEntry["name"] in knownClasses: returnTypeEntry["linkable"] = True elif not item["type"] in builtinTypes and not item["type"] in pseudoTypes and not item["type"] in additionalTypes: error('- Invalid type "%s" in %s at line %s', item["type"], className, item["line"]) # Process doc if "doc" in item: def processInternalLink(match): linkUrl = match.group(2) if linkUrl.startswith("#"): linkCheck = checkInternalLink(linkUrl[1:], className) if linkCheck is not True: item["errornous"] = True if sectionName: error("- %s in %s:%s~%s at line %s" % (linkCheck, sectionName, className, name, item["line"])) else: error("- %s in %s at line %s" % (linkCheck, className, item["line"])) linkExtract.sub(processInternalLink, item["doc"]) # Process APIs for className in apiData: classApi = apiData[className] sectionName = None constructData = getattr(classApi, "construct", None) if constructData is not None: checkLinksInItem(constructData) for sectionName in ("properties", "events", "statics", "members"): section = getattr(classApi, sectionName, None) if section is not None: for name in section: checkLinksInItem(section[name]) # # Filter Internals/Privates # info("Filtering Items...") def isVisible(entry): if "visibility" in entry: visibility = entry["visibility"] if visibility == "private" and not privates: return False if visibility == "internal" and not internals: return False return True def filterInternalsPrivates(classApi, field): data = getattr(classApi, field, None) if data: for name in list(data): if not isVisible(data[name]): del data[name] for className in apiData: filterInternalsPrivates(apiData[className], "statics") filterInternalsPrivates(apiData[className], "members") # # Connection Interfaces / ImplementedBy # info("Connecting Interfaces...") for className in apiData: classApi = getApi(className) if not hasattr(classApi, "main"): continue classType = classApi.main["type"] if classType == "core.Class": classImplements = getattr(classApi, "implements", None) if classImplements: for interfaceName in classImplements: interfaceApi = apiData[interfaceName] implementedBy = getattr(interfaceApi, "implementedBy", None) if not implementedBy: implementedBy = interfaceApi.implementedBy = [] implementedBy.append(className) connectInterface(className, interfaceName, classApi, interfaceApi) # # Merging Named Classes # info("Merging Named Classes...") indent() for className in list(apiData): classApi = apiData[className] destName = classApi.main["name"] if destName is not None and destName != className: debug("Extending class %s with %s", destName, className) if destName in apiData: destApi = apiData[destName] destApi.main["from"].append(className) else: destApi = apiData[destName] = ApiData(destName) destApi.main = { "type" : "Extend", "name" : destName, "from" : [className] } # If there is a "main" tag found in the class use its API description if "tags" in classApi.main and classApi.main["tags"] is not None and "main" in classApi.main["tags"]: if "doc" in classApi.main: destApi.main["doc"] = classApi.main["doc"] classApi.main["extension"] = True # Read existing data construct = getattr(classApi, "construct", None) statics = getattr(classApi, "statics", None) members = getattr(classApi, "members", None) if construct is not None: if hasattr(destApi, "construct"): warn("Overriding constructor in extension %s by %s", destName, className) destApi.construct = copy.copy(construct) if statics is not None: if not hasattr(destApi, "statics"): destApi.statics = {} for staticName in statics: destApi.statics[staticName] = copy.copy(statics[staticName]) destApi.statics[staticName]["from"] = className destApi.statics[staticName]["fromLink"] = "static:%s~%s" % (className, staticName) if members is not None: if not hasattr(destApi, "members"): destApi.members = {} for memberName in members: destApi.members[memberName] = copy.copy(members[memberName]) destApi.members[memberName]["from"] = className destApi.members[memberName]["fromLink"] = "member:%s~%s" % (className, memberName) outdent() # # Connecting Uses / UsedBy # info("Collecting Use Patterns...") # This matches all uses with the known classes and only keeps them if matched allClasses = set(list(apiData)) for className in apiData: uses = apiData[className].uses # Rebuild use list cleanUses = set() for use in uses: if use != className and use in allClasses: cleanUses.add(use) useEntry = apiData[use] if not hasattr(useEntry, "usedBy"): useEntry.usedBy = set() useEntry.usedBy.add(className) apiData[className].uses = cleanUses # # Collecting errors # info("Collecting Errors...") indent() for className in sorted(apiData): classApi = apiData[className] errors = [] if isErrornous(classApi.main): errors.append({ "kind": "Main", "name": None, "line": 1 }) if hasattr(classApi, "construct"): if isErrornous(classApi.construct): errors.append({ "kind": "Constructor", "name": None, "line": classApi.construct["line"] }) for section in ("statics", "members", "properties", "events"): items = getattr(classApi, section, {}) for itemName in items: item = items[itemName] if isErrornous(item): errors.append({ "kind": itemMap[section], "name": itemName, "line": item["line"] }) if errors: if printErrors: warn("Found errors in %s", className) errorsSorted = sorted(errors, key=lambda entry: entry["line"]) if printErrors: indent() for entry in errorsSorted: if entry["name"]: warn("%s: %s (line %s)", entry["kind"], entry["name"], entry["line"]) else: warn("%s (line %s)", entry["kind"], entry["line"]) outdent() classApi.errors = errorsSorted outdent() # # Building Search Index # info("Building Search Index...") search = {} def addSearch(classApi, field): data = getattr(classApi, field, None) if data: for name in data: if not name in search: search[name] = set() search[name].add(className) for className in apiData: classApi = apiData[className] addSearch(classApi, "statics") addSearch(classApi, "members") addSearch(classApi, "properties") addSearch(classApi, "events") # # Post Process (dict to sorted list) # info("Post Processing Data...") for className in sorted(apiData): classApi = apiData[className] convertTags(classApi.main) construct = getattr(classApi, "construct", None) if construct: convertFunction(construct) convertTags(construct) for section in ("statics", "members", "properties", "events"): items = getattr(classApi, section, None) if items: sortedList = [] for itemName in sorted(items): item = items[itemName] item["name"] = itemName if "type" in item and item["type"] == "Function": convertFunction(item) convertTags(item) sortedList.append(item) setattr(classApi, section, sortedList) # # Collecting Package Docs # info("Collecting Package Docs...") indent() # Inject existing package docs into api data for project in session.getProjects(): docs = project.getDocs() for packageName in docs: if self.isIncluded(packageName, classFilter): debug("Creating package documentation %s", packageName) apiData[packageName] = docs[packageName].getApi() # Fill missing package docs for className in sorted(apiData): splits = className.split(".") packageName = splits[0] for split in splits[1:]: if not packageName in apiData: warn("Missing package documentation %s", packageName) apiData[packageName] = ApiData(packageName) apiData[packageName].main = { "type" : "Package", "name" : packageName } packageName = "%s.%s" % (packageName, split) # Now register all classes in their parent namespace/package for className in sorted(apiData): splits = className.split(".") packageName = ".".join(splits[:-1]) if packageName: package = apiData[packageName] # debug("- Registering class %s in parent %s", className, packageName) entry = { "name" : splits[-1], "link" : className, } classMain = apiData[className].main if "doc" in classMain and classMain["doc"]: summary = extractSummary(classMain["doc"]) if summary: entry["summary"] = summary if "type" in classMain and classMain["type"]: entry["type"] = classMain["type"] if not hasattr(package, "content"): package.content = [entry] else: package.content.append(entry) outdent() # # Writing API Index # debug("Building Index...") index = {} for className in sorted(apiData): classApi = apiData[className] mainInfo = classApi.main # Create structure for className current = index for split in className.split("."): if not split in current: current[split] = {} current = current[split] # Store current type current["$type"] = mainInfo["type"] # Keep information if if hasattr(classApi, "content"): current["$content"] = True # # Return # return apiData, index, search
def getApi(self, highlight=True): field = "api[%s]-%s" % (self.id, highlight) apidata = self.project.getCache().read(field, self.mtime) if apidata is None: apidata = ApiData(self.id, highlight) tree = self.__getTree(context="api") indent() apidata.scanTree(tree) outdent() metaData = self.getMetaData() apidata.addAssets(metaData.assets) for require in metaData.requires: apidata.addUses(require) for optional in metaData.optionals: apidata.removeUses(optional) apidata.addSize(self.getSize()) apidata.addFields(self.getFields()) self.project.getCache().store(field, apidata, self.mtime) return apidata
def getApi(self): field = "api[%s]" % self.__id apidata = self.__cache.read(field, self.__mtime) if apidata is None: apidata = ApiData(self.__name) apidata.scanTree(self.getTree(cleanup=False)) metaData = self.getMetaData() apidata.addAssets(metaData.assets) for require in metaData.requires: apidata.addUses(require) for optional in metaData.optionals: apidata.removeUses(optional) apidata.addSize(self.getSize()) apidata.addPermutations(self.getPermutationKeys()) self.__cache.store(field, apidata, self.__mtime) return apidata