def getApi(self):
field = "api[%s]" % self.id
apidata = self.project.getCache().read(field, self.getModificationTime())
if markdown is None:
raise JasyError("Missing Markdown feature to convert package docs into HTML.")
if apidata is None:
apidata = ApiData(self.id)
apidata.main["type"] = "Package"
apidata.main["doc"] = markdown(self.getText())
self.project.getCache().store(field, apidata, self.getModificationTime())
return apidata
def getApi(self):
field = "api[%s]" % self.__id
apidata = self.__cache.read(field, self.__mtime)
if apidata is None:
apidata = ApiData(self.getTree(cleanup=False), self.__name)
self.__cache.store(field, apidata, self.__mtime)
return apidata
def collect(self, internals=False, privates=False):
#
# Collecting Original Data
#
logging.debug("Collecting Classes...")
apiData = {}
for project in self.session.getProjects():
classes = project.getClasses()
for className in classes:
apiData[className] = classes[className].getApi()
#
# Collecting Source Code
#
logging.debug("Highlighting Code...")
highlighted = {}
for project in self.session.getProjects():
classes = project.getClasses()
for className in classes:
highlighted[className] = classes[className].getHighlightedCode()
#
# Filter Internals/Privates
#
logging.debug("Filtering Items...")
def isVisible(entry):
if "visibility" in entry:
visibility = entry["visibility"]
if visibility == "private" and not privates:
return False
if visibility == "internal" and not internals:
return False
return True
def filterInternalsPrivates(classApi, field):
data = getattr(classApi, field, None)
if data:
for name in list(data):
if not isVisible(data[name]):
del data[name]
for className in apiData:
filterInternalsPrivates(apiData[className], "statics")
filterInternalsPrivates(apiData[className], "members")
#
# Attaching Links to Source Code (Lines)
# Building Documentation Summaries
#
logging.debug("Tweaking Output...")
for className in apiData:
classApi = apiData[className]
constructData = getattr(classApi, "construct", None)
if constructData is not None:
if "line" in constructData:
constructData["sourceLink"] = "source:%s~%s" % (className, constructData["line"])
for section in ("properties", "events", "statics", "members"):
sectionData = getattr(classApi, section, None)
if sectionData is not None:
for name in sectionData:
if "line" in sectionData[name]:
sectionData[name]["sourceLink"] = "source:%s~%s" % (className, sectionData[name]["line"])
if "doc" in sectionData[name]:
summary = extractSummary(sectionData[name]["doc"])
if summary is not None:
sectionData[name]["summary"] = summary
#
# Including Mixins / IncludedBy
#
logging.debug("Resolving Mixins...")
# Just used temporary to keep track of which classes are merged
mergedClasses = set()
def getApi(className):
classApi = apiData[className]
if className in mergedClasses:
return classApi
classIncludes = getattr(classApi, "includes", None)
if classIncludes:
for mixinName in classIncludes:
mixinApi = apiData[mixinName]
if not hasattr(mixinApi, "includedBy"):
mixinApi.includedBy = set()
mixinApi.includedBy.add(className)
mergeMixin(className, mixinName, classApi, getApi(mixinName))
mergedClasses.add(className)
return classApi
for className in apiData:
apiData[className] = getApi(className)
#
# Connection Interfaces / ImplementedBy
#
logging.debug("Connecting Interfaces...")
for className in apiData:
classApi = getApi(className)
if not hasattr(classApi, "main"):
continue
classType = classApi.main["type"]
if classType == "core.Class":
classImplements = getattr(classApi, "implements", None)
if classImplements:
for interfaceName in classImplements:
interfaceApi = apiData[interfaceName]
implementedBy = getattr(interfaceApi, "implementedBy", None)
if not implementedBy:
implementedBy = interfaceApi.implementedBy = []
implementedBy.append(className)
connectInterface(className, interfaceName, classApi, interfaceApi)
#
# Connecting Uses / UsedBy
#
logging.debug("Collecting Use Patterns...")
# This matches all uses with the known classes and only keeps them if matched
allClasses = set(list(apiData))
for className in apiData:
uses = apiData[className].uses
# Rebuild use list
cleanUses = set()
for use in uses:
if use != className and use in allClasses:
cleanUses.add(use)
useEntry = apiData[use]
if not hasattr(useEntry, "usedBy"):
useEntry.usedBy = set()
useEntry.usedBy.add(className)
apiData[className].uses = cleanUses
#
# Merging Named Classes
#
logging.debug("Merging Named Classes...")
for className in list(apiData):
classApi = apiData[className]
destName = classApi.main["name"]
if destName is not None and destName != className:
if destName in apiData:
destApi = apiData[destName]
destApi.main["from"].append(className)
else:
destApi = apiData[destName] = ApiData(destName)
destApi.main = {
"type" : "Extend",
"name" : destName,
"from" : [className],
"doc" : "Extensions for %s" % destName
}
# If there is a "main" tag found in the class use its API description
if "tags" in classApi.main and classApi.main["tags"] is not None and "main" in classApi.main["tags"]:
if "doc" in classApi.main and destApi.main["doc"] == "Extensions for %s" % destName:
destApi.main["doc"] = classApi.main["doc"]
classApi.main["extension"] = True
# Read existing data
construct = getattr(classApi, "construct", None)
statics = getattr(classApi, "statics", None)
members = getattr(classApi, "members", None)
if construct is not None:
if hasattr(destApi, "construct"):
logging.warn("Overriding constructor in extension %s by %s", destName, className)
destApi.construct = copy.copy(construct)
if statics is not None:
if not hasattr(destApi, "statics"):
destApi.statics = {}
for staticName in statics:
destApi.statics[staticName] = copy.copy(statics[staticName])
destApi.statics[staticName]["from"] = className
destApi.statics[staticName]["fromLink"] = "static:%s~%s" % (className, staticName)
if members is not None:
if not hasattr(destApi, "members"):
destApi.members = {}
for memberName in members:
destApi.members[memberName] = copy.copy(members[memberName])
destApi.members[memberName]["from"] = className
destApi.members[memberName]["fromLink"] = "member:%s~%s" % (className, memberName)
#
# Collecting errors
#
for className in sorted(apiData):
classApi = apiData[className]
errors = []
if isErrornous(classApi.main):
errors.append({
"kind": "Main",
"name": None,
"line": 1
})
if hasattr(classApi, "construct"):
if isErrornous(classApi.construct):
errors.append({
"kind": "Constructor",
"name": None,
"line": classApi.construct["line"]
})
for section in ("statics", "members", "properties", "events"):
items = getattr(classApi, section, {})
for itemName in items:
item = items[itemName]
if isErrornous(item):
errors.append({
"kind": itemMap[section],
"name": itemName,
"line": item["line"]
})
if errors:
logging.warn("API documentation errors in %s", className)
errorsSorted = sorted(errors, key=lambda entry: entry["line"])
for entry in errorsSorted:
if entry["name"]:
logging.warn("- %s: %s (line %s)", entry["kind"], entry["name"], entry["line"])
else:
logging.warn("- %s (line %s)", entry["kind"], entry["line"])
classApi.errors = errorsSorted
#
# Building Search Index
#
logging.debug("Building Search Index")
search = {}
def addSearch(classApi, field):
data = getattr(classApi, field, None)
if data:
for name in data:
if not name in search:
search[name] = set()
search[name].add(className)
for className in apiData:
classApi = apiData[className]
addSearch(classApi, "statics")
addSearch(classApi, "members")
addSearch(classApi, "properties")
addSearch(classApi, "events")
#
# Post Process (dict to sorted list)
#
for className in sorted(apiData):
classApi = apiData[className]
convertTags(classApi.main)
construct = getattr(classApi, "construct", None)
if construct:
convertFunction(construct)
convertTags(construct)
for section in ("statics", "members", "properties", "events"):
items = getattr(classApi, section, None)
if items:
sortedList = []
for itemName in sorted(items):
item = items[itemName]
item["name"] = itemName
if "type" in item and item["type"] == "Function":
convertFunction(item)
convertTags(item)
sortedList.append(item)
setattr(classApi, section, sortedList)
#
# Collecting Package Docs
#
logging.debug("Collecting Package Docs...")
packages = set()
for project in self.session.getProjects():
docs = project.getDocs()
for packageName in docs:
packages.add(packageName)
apiData[packageName] = ApiData(packageName)
apiData[packageName].main = {
"type" : "Package",
"name" : packageName,
"doc" : docs[packageName]
}
for className in list(apiData):
# Auto create API data for all packages in between
splits = className.split(".")
packageName = splits[0]
for split in splits[1:]:
if not packageName in apiData:
packages.add(packageName)
logging.debug("Creating missing package doc entry: %s" % packageName)
apiData[packageName] = ApiData(packageName)
apiData[packageName].main = {
"type" : "Package",
"name" : packageName
}
packageName = "%s.%s" % (packageName, split)
# Register class inside package "content"
lastPackage = ".".join(splits[:-1])
if lastPackage:
if not hasattr(apiData[lastPackage], "content"):
apiData[lastPackage].content = []
classPkgEntry = {
"name": splits[-1],
"link": className
}
classMain = apiData[className].main
if "doc" in classMain and classMain["doc"]:
summary = extractSummary(classMain["doc"])
if summary:
classPkgEntry["summary"] = summary
apiData[lastPackage].content.append(classPkgEntry)
# Sort package content
for packageName in packages:
apiData[packageName].content.sort(key=lambda entry: entry["name"])
#
# Writing API Index
#
logging.debug("Building Index...")
index = {}
for className in apiData:
classApi = apiData[className]
mainInfo = classApi.main
# Create missing packages for className
current = index
for split in className.split("."):
if not split in current:
current[split] = {}
current = current[split]
# Store current type
current["$type"] = mainInfo["type"]
#
# Return
#
return apiData, highlighted, index, search
def process(self, apiData, classFilter=None, internals=False, privates=False, printErrors=True):
knownClasses = set(list(apiData))
#
# Attaching Links to Source Code (Lines)
# Building Documentation Summaries
#
info("Adding Source Links...")
for className in apiData:
classApi = apiData[className]
constructData = getattr(classApi, "construct", None)
if constructData is not None:
if "line" in constructData:
constructData["sourceLink"] = "source:%s~%s" % (className, constructData["line"])
for section in ("properties", "events", "statics", "members"):
sectionData = getattr(classApi, section, None)
if sectionData is not None:
for name in sectionData:
if "line" in sectionData[name]:
sectionData[name]["sourceLink"] = "source:%s~%s" % (className, sectionData[name]["line"])
#
# Including Mixins / IncludedBy
#
info("Resolving Mixins...")
# Just used temporary to keep track of which classes are merged
mergedClasses = set()
def getApi(className):
classApi = apiData[className]
if className in mergedClasses:
return classApi
classIncludes = getattr(classApi, "includes", None)
if classIncludes:
for mixinName in classIncludes:
if not mixinName in apiData:
error("Invalid mixin %s in class %s", className, mixinName)
continue
mixinApi = apiData[mixinName]
if not hasattr(mixinApi, "includedBy"):
mixinApi.includedBy = set()
mixinApi.includedBy.add(className)
mergeMixin(className, mixinName, classApi, getApi(mixinName))
mergedClasses.add(className)
return classApi
for className in apiData:
apiData[className] = getApi(className)
#
# Checking links
#
info("Checking Links...")
additionalTypes = ("Call", "Identifier", "Map", "Integer", "Node", "Element")
def checkInternalLink(link, className):
match = internalLinkParse.match(link)
if not match:
return 'Invalid link "#%s"' % link
if match.group(3) is not None:
className = match.group(3)
if not className in knownClasses and not className in apiData:
return 'Invalid class in link "#%s"' % link
# Accept all section/item values for named classes,
# as it might be pretty complicated to verify this here.
if not className in apiData:
return True
classApi = apiData[className]
sectionName = match.group(2)
itemName = match.group(5)
if itemName is None:
return True
if sectionName is not None:
if not sectionName in linkMap:
return 'Invalid section in link "#%s"' % link
section = getattr(classApi, linkMap[sectionName], None)
if section is None:
return 'Invalid section in link "#%s"' % link
else:
if itemName in section:
return True
return 'Invalid item in link "#%s"' % link
for sectionName in ("statics", "members", "properties", "events"):
section = getattr(classApi, sectionName, None)
if section and itemName in section:
return True
return 'Invalid item link "#%s"' % link
def checkLinksInItem(item):
# Process types
if "type" in item:
if item["type"] == "Function":
# Check param types
if "params" in item:
for paramName in item["params"]:
paramEntry = item["params"][paramName]
if "type" in paramEntry:
for paramTypeEntry in paramEntry["type"]:
if not paramTypeEntry["name"] in knownClasses and not paramTypeEntry["name"] in additionalTypes and not ("builtin" in paramTypeEntry or "pseudo" in paramTypeEntry):
item["errornous"] = True
error('- Invalid param type "%s" in %s at line %s', paramTypeEntry["name"], className, item["line"])
if not "pseudo" in paramTypeEntry and paramTypeEntry["name"] in knownClasses:
paramTypeEntry["linkable"] = True
# Check return types
if "returns" in item:
for returnTypeEntry in item["returns"]:
if not returnTypeEntry["name"] in knownClasses and not returnTypeEntry["name"] in additionalTypes and not ("builtin" in returnTypeEntry or "pseudo" in returnTypeEntry):
item["errornous"] = True
error('- Invalid return type "%s" in %s at line %s', returnTypeEntry["name"], className, item["line"])
if not "pseudo" in returnTypeEntry and returnTypeEntry["name"] in knownClasses:
returnTypeEntry["linkable"] = True
elif not item["type"] in builtinTypes and not item["type"] in pseudoTypes and not item["type"] in additionalTypes:
error('- Invalid type "%s" in %s at line %s', item["type"], className, item["line"])
# Process doc
if "doc" in item:
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
error("- %s in %s:%s~%s at line %s" % (linkCheck, sectionName, className, name, item["line"]))
else:
error("- %s in %s at line %s" % (linkCheck, className, item["line"]))
linkExtract.sub(processInternalLink, item["doc"])
# Process APIs
for className in apiData:
classApi = apiData[className]
sectionName = None
constructData = getattr(classApi, "construct", None)
if constructData is not None:
checkLinksInItem(constructData)
for sectionName in ("properties", "events", "statics", "members"):
section = getattr(classApi, sectionName, None)
if section is not None:
for name in section:
checkLinksInItem(section[name])
#
# Filter Internals/Privates
#
info("Filtering Items...")
def isVisible(entry):
if "visibility" in entry:
visibility = entry["visibility"]
if visibility == "private" and not privates:
return False
if visibility == "internal" and not internals:
return False
return True
def filterInternalsPrivates(classApi, field):
data = getattr(classApi, field, None)
if data:
for name in list(data):
if not isVisible(data[name]):
del data[name]
for className in apiData:
filterInternalsPrivates(apiData[className], "statics")
filterInternalsPrivates(apiData[className], "members")
#
# Connection Interfaces / ImplementedBy
#
info("Connecting Interfaces...")
for className in apiData:
classApi = getApi(className)
if not hasattr(classApi, "main"):
continue
classType = classApi.main["type"]
if classType == "core.Class":
classImplements = getattr(classApi, "implements", None)
if classImplements:
for interfaceName in classImplements:
interfaceApi = apiData[interfaceName]
implementedBy = getattr(interfaceApi, "implementedBy", None)
if not implementedBy:
implementedBy = interfaceApi.implementedBy = []
implementedBy.append(className)
connectInterface(className, interfaceName, classApi, interfaceApi)
#
# Merging Named Classes
#
info("Merging Named Classes...")
indent()
for className in list(apiData):
classApi = apiData[className]
destName = classApi.main["name"]
if destName is not None and destName != className:
debug("Extending class %s with %s", destName, className)
if destName in apiData:
destApi = apiData[destName]
destApi.main["from"].append(className)
else:
destApi = apiData[destName] = ApiData(destName)
destApi.main = {
"type" : "Extend",
"name" : destName,
"from" : [className]
}
# If there is a "main" tag found in the class use its API description
if "tags" in classApi.main and classApi.main["tags"] is not None and "main" in classApi.main["tags"]:
if "doc" in classApi.main:
destApi.main["doc"] = classApi.main["doc"]
classApi.main["extension"] = True
# Read existing data
construct = getattr(classApi, "construct", None)
statics = getattr(classApi, "statics", None)
members = getattr(classApi, "members", None)
if construct is not None:
if hasattr(destApi, "construct"):
warn("Overriding constructor in extension %s by %s", destName, className)
destApi.construct = copy.copy(construct)
if statics is not None:
if not hasattr(destApi, "statics"):
destApi.statics = {}
for staticName in statics:
destApi.statics[staticName] = copy.copy(statics[staticName])
destApi.statics[staticName]["from"] = className
destApi.statics[staticName]["fromLink"] = "static:%s~%s" % (className, staticName)
if members is not None:
if not hasattr(destApi, "members"):
destApi.members = {}
for memberName in members:
destApi.members[memberName] = copy.copy(members[memberName])
destApi.members[memberName]["from"] = className
destApi.members[memberName]["fromLink"] = "member:%s~%s" % (className, memberName)
outdent()
#
# Connecting Uses / UsedBy
#
info("Collecting Use Patterns...")
# This matches all uses with the known classes and only keeps them if matched
allClasses = set(list(apiData))
for className in apiData:
uses = apiData[className].uses
# Rebuild use list
cleanUses = set()
for use in uses:
if use != className and use in allClasses:
cleanUses.add(use)
useEntry = apiData[use]
if not hasattr(useEntry, "usedBy"):
useEntry.usedBy = set()
useEntry.usedBy.add(className)
apiData[className].uses = cleanUses
#
# Collecting errors
#
info("Collecting Errors...")
indent()
for className in sorted(apiData):
classApi = apiData[className]
errors = []
if isErrornous(classApi.main):
errors.append({
"kind": "Main",
"name": None,
"line": 1
})
if hasattr(classApi, "construct"):
if isErrornous(classApi.construct):
errors.append({
"kind": "Constructor",
"name": None,
"line": classApi.construct["line"]
})
for section in ("statics", "members", "properties", "events"):
items = getattr(classApi, section, {})
for itemName in items:
item = items[itemName]
if isErrornous(item):
errors.append({
"kind": itemMap[section],
"name": itemName,
"line": item["line"]
})
if errors:
if printErrors:
warn("Found errors in %s", className)
errorsSorted = sorted(errors, key=lambda entry: entry["line"])
if printErrors:
indent()
for entry in errorsSorted:
if entry["name"]:
warn("%s: %s (line %s)", entry["kind"], entry["name"], entry["line"])
else:
warn("%s (line %s)", entry["kind"], entry["line"])
outdent()
classApi.errors = errorsSorted
outdent()
#
# Building Search Index
#
info("Building Search Index...")
search = {}
def addSearch(classApi, field):
data = getattr(classApi, field, None)
if data:
for name in data:
if not name in search:
search[name] = set()
search[name].add(className)
for className in apiData:
classApi = apiData[className]
addSearch(classApi, "statics")
addSearch(classApi, "members")
addSearch(classApi, "properties")
addSearch(classApi, "events")
#
# Post Process (dict to sorted list)
#
info("Post Processing Data...")
for className in sorted(apiData):
classApi = apiData[className]
convertTags(classApi.main)
construct = getattr(classApi, "construct", None)
if construct:
convertFunction(construct)
convertTags(construct)
for section in ("statics", "members", "properties", "events"):
items = getattr(classApi, section, None)
if items:
sortedList = []
for itemName in sorted(items):
item = items[itemName]
item["name"] = itemName
if "type" in item and item["type"] == "Function":
convertFunction(item)
convertTags(item)
sortedList.append(item)
setattr(classApi, section, sortedList)
#
# Collecting Package Docs
#
info("Collecting Package Docs...")
indent()
# Inject existing package docs into api data
for project in session.getProjects():
docs = project.getDocs()
for packageName in docs:
if self.isIncluded(packageName, classFilter):
debug("Creating package documentation %s", packageName)
apiData[packageName] = docs[packageName].getApi()
# Fill missing package docs
for className in sorted(apiData):
splits = className.split(".")
packageName = splits[0]
for split in splits[1:]:
if not packageName in apiData:
warn("Missing package documentation %s", packageName)
apiData[packageName] = ApiData(packageName)
apiData[packageName].main = {
"type" : "Package",
"name" : packageName
}
packageName = "%s.%s" % (packageName, split)
# Now register all classes in their parent namespace/package
for className in sorted(apiData):
splits = className.split(".")
packageName = ".".join(splits[:-1])
if packageName:
package = apiData[packageName]
# debug("- Registering class %s in parent %s", className, packageName)
entry = {
"name" : splits[-1],
"link" : className,
}
classMain = apiData[className].main
if "doc" in classMain and classMain["doc"]:
summary = extractSummary(classMain["doc"])
if summary:
entry["summary"] = summary
if "type" in classMain and classMain["type"]:
entry["type"] = classMain["type"]
if not hasattr(package, "content"):
package.content = [entry]
else:
package.content.append(entry)
outdent()
#
# Writing API Index
#
debug("Building Index...")
index = {}
for className in sorted(apiData):
classApi = apiData[className]
mainInfo = classApi.main
# Create structure for className
current = index
for split in className.split("."):
if not split in current:
current[split] = {}
current = current[split]
# Store current type
current["$type"] = mainInfo["type"]
# Keep information if
if hasattr(classApi, "content"):
current["$content"] = True
#
# Return
#
return apiData, index, search
def getApi(self, highlight=True):
field = "api[%s]-%s" % (self.id, highlight)
apidata = self.project.getCache().read(field, self.mtime)
if apidata is None:
apidata = ApiData(self.id, highlight)
tree = self.__getTree(context="api")
indent()
apidata.scanTree(tree)
outdent()
metaData = self.getMetaData()
apidata.addAssets(metaData.assets)
for require in metaData.requires:
apidata.addUses(require)
for optional in metaData.optionals:
apidata.removeUses(optional)
apidata.addSize(self.getSize())
apidata.addFields(self.getFields())
self.project.getCache().store(field, apidata, self.mtime)
return apidata
def getApi(self):
field = "api[%s]" % self.__id
apidata = self.__cache.read(field, self.__mtime)
if apidata is None:
apidata = ApiData(self.__name)
apidata.scanTree(self.getTree(cleanup=False))
metaData = self.getMetaData()
apidata.addAssets(metaData.assets)
for require in metaData.requires:
apidata.addUses(require)
for optional in metaData.optionals:
apidata.removeUses(optional)
apidata.addSize(self.getSize())
apidata.addPermutations(self.getPermutationKeys())
self.__cache.store(field, apidata, self.__mtime)
return apidata
