def handleEvents(item, classNode):
for key, value in mapNodeToMap(item).items():
keyvalue = value.parent
value = value.getFirstChild(True, True).get("value")
node = tree.Node("event")
commentAttributes = comment.parseNode(keyvalue)
try:
desc = commentAttributes[0]["text"]
except (IndexError, KeyError):
desc = None
addError(node, "Documentation is missing.", item)
if desc != None:
node.addChild(tree.Node("desc").set("text", desc))
node.set("name", key)
typesNode = tree.Node("types")
node.addChild(typesNode)
itemNode = tree.Node("entry")
typesNode.addChild(itemNode)
itemNode.set("type", value)
handleDeprecated(node, commentAttributes)
handleAccess(node, commentAttributes)
classNode.addListChild("events", node)
def createClassDefine(id):
classDefine, className, classMap = createClassDefineCore(id)
settingsMap = tree.Node("map")
settingsPair = createPair("settings", settingsMap)
variantsMap = tree.Node("map")
variantsPair = createPair("variants", variantsMap)
propertiesMap = tree.Node("map")
propertiesPair = createPair("properties", propertiesMap)
membersMap = tree.Node("map")
membersPair = createPair("members", membersMap)
staticsMap = tree.Node("map")
staticsPair = createPair("statics", staticsMap)
propertiesPair.addChild(createBlockComment("properties"))
membersPair.addChild(createBlockComment("members"))
staticsPair.addChild(createBlockComment("statics"))
settingsPair.addChild(createBlockComment("settings"))
variantsPair.addChild(createBlockComment("variants"))
classMap.addChild(staticsPair)
classMap.addChild(propertiesPair)
classMap.addChild(membersPair)
classMap.addChild(settingsPair)
classMap.addChild(variantsPair)
return classDefine, className, classMap, settingsMap, variantsMap, propertiesMap, membersMap, staticsMap
def handleDeprecated(docNode, commentAttributes):
for docItem in commentAttributes:
if docItem["category"] == "deprecated":
deprecatedNode = tree.Node("deprecated")
if docItem.has_key("text"):
descNode = tree.Node("desc").set("text", docItem["text"])
deprecatedNode.addChild(descNode)
docNode.addChild(deprecatedNode)
def createVariable(l):
var = tree.Node("variable")
for name in l:
iden = tree.Node("identifier")
iden.set("name", name)
var.addChild(iden)
return var
def createPair(key, value, commentParent=None):
par = tree.Node("keyvalue")
sub = tree.Node("value")
par.set("key", key)
par.addChild(sub)
sub.addChild(value)
if commentParent and commentParent.hasChild("commentsBefore"):
par.addChild(commentParent.getChild("commentsBefore"))
return par
def inlineIfStatement(ifNode, conditionValue):
"""
Inline an if statement assuming that the condition of the if
statement evaluates to "conditionValue" (True/False")
"""
if ifNode.type != "loop" or ifNode.get("loopType") != "IF":
raise tree.NodeAccessException("Expected a the LOOP node of an if statement!", mapNode)
replacement = []
newDefinitions = []
reovedDefinitions = []
if ifNode.getChild("elseStatement", False):
if conditionValue:
reovedDefinitions = getDefinitions(ifNode.getChild("elseStatement"))
newDefinitions = getDefinitions(ifNode.getChild("statement"))
replacement = ifNode.getChild("statement").children
else:
reovedDefinitions = getDefinitions(ifNode.getChild("statement"))
newDefinitions = getDefinitions(ifNode.getChild("elseStatement"))
replacement = ifNode.getChild("elseStatement").children
else:
if conditionValue:
newDefinitions = getDefinitions(ifNode.getChild("statement"))
replacement = ifNode.getChild("statement").children
else:
reovedDefinitions = getDefinitions(ifNode.getChild("statement"))
newDefinitions = map(lambda x: x.get("identifier"), newDefinitions)
definitions = []
for definition in reovedDefinitions:
if not definition.get("identifier") in newDefinitions:
definitions.append(definition)
if len(definitions) > 0:
defList = tree.Node("definitionList")
defList.set("line", ifNode.get("line"))
for definition in definitions:
if definition.hasChildren():
del definition.children
defList.addChild(definition)
replacement.append(defList)
if replacement != []:
replaceChildWithNodes(ifNode.parent, ifNode, replacement)
else:
emptyBlock = tree.Node("block");
emptyBlock.set("line", ifNode.get("line"))
ifNode.parent.replaceChild(ifNode, emptyBlock)
def createSyntaxTree (tokenArr):
"""Creates a syntax tree from a token stream.
tokens: the token stream."""
stream = TokenStream(tokenArr)
stream.next()
from pprint import pprint
#pprint([(x['detail'],x['source']) for x in tokenArr])
#pprint([x for x in tokenArr if x['type']=="comment"])
rootBlock = tree.Node("file")
rootBlock.set("file", stream.curr()["id"])
while not stream.finished():
rootBlock.addChild(readStatement(stream))
# collect prob. pending comments
try:
for c in stream.commentsBefore: # stream.commentsBefore might not be defined
rootBlock.addChild(c)
except:
pass
return rootBlock
def handleConstantDefinition(item, classNode):
if (item.type == "assignment"):
# This is a "normal" constant definition
leftItem = item.getFirstListChild("left")
name = leftItem.children[len(leftItem.children) - 1].get("name")
valueNode = item.getChild("right")
elif (item.type == "keyvalue"):
# This is a constant definition of a map-style class (like qx.Const)
name = item.get("key")
valueNode = item.getChild("value")
if not name.isupper():
return
node = tree.Node("constant")
node.set("name", name)
value = None
if valueNode.hasChild("constant"):
node.set("value", valueNode.getChild("constant").get("value"))
node.set(
"type",
valueNode.getChild("constant").get("constantType").capitalize())
commentAttributes = comment.parseNode(item)
description = comment.getAttrib(commentAttributes, "description")
addTypeInfo(node, description, item)
handleDeprecated(node, commentAttributes)
handleAccess(node, commentAttributes)
classNode.addListChild("constants", node)
def handleMixins(item, classNode, docTree, className):
if classNode.get("type", False) == "mixin":
superMixinNames = variableOrArrayNodeToArray(item)
for superMixin in superMixinNames:
superMixinNode = getClassNode(docTree, superMixin)
childMixins = superMixinNode.get("mixins", False)
if childMixins:
childMixins += "," + className
else:
childMixins = className
superMixinNode.set("childClasses", childMixins)
node = tree.Node("interface")
node.set("name", superMixin)
classNode.addListChild("superMixins", node)
else:
mixins = variableOrArrayNodeToArray(item)
for mixin in mixins:
mixinNode = getClassNode(docTree, mixin)
includer = mixinNode.get("includer", False)
if includer:
includer += "," + className
else:
includer = className
mixinNode.set("includer", includer)
classNode.set("mixins", ",".join(mixins))
def createClassDefineCore(id):
call = tree.Node("call")
oper = tree.Node("operand")
para = tree.Node("params")
con = createConstant("string", id)
args = tree.Node("map")
call.addChild(oper)
call.addChild(para)
oper.addChild(createVariable(["qx", "Class", "define"]))
para.addChild(con)
para.addChild(args)
return call, con, args
def createConstant(type, value):
constant = tree.Node("constant")
constant.set("constantType", type)
constant.set("value", value)
if type == "string":
constant.set("detail", "doublequotes")
return constant
def get_section_node(classDoc, section):
section_node = None
for node in api.docTreeIterator(classDoc, node_types[section]):
section_node = node # first should be only
break
if not section_node:
section_node = tree.Node(node_types[section])
classDoc.addChild(section_node)
return section_node
def replace(node, stringList, var="$", verbose=False):
if node.type == "constant" and node.get("constantType") == "string":
if node.get("detail") == "singlequotes":
quote = "'"
elif node.get("detail") == "doublequotes":
quote = '"'
oldvalue = "%s%s%s" % (quote, node.get("value"), quote)
pos = 0
for item in stringList:
if item["value"] == oldvalue:
newvalue = "%s[%s]" % (var, pos)
if verbose:
poldvalue = oldvalue
if isinstance(poldvalue, unicode):
poldvalue = poldvalue.encode("utf-8")
print " - Replace: %s => %s" % (poldvalue, newvalue)
line = node.get("line")
# GENERATE IDENTIFIER
newvariable = tree.Node("variable")
newvariable.set("line", line)
childidentifier = tree.Node("identifier")
childidentifier.set("line", line)
childidentifier.set("name", "SSSS_%s" % pos)
newvariable.addChild(childidentifier)
# REPLACE NODE
node.parent.replaceChild(node, newvariable)
break
pos += 1
if check(node, verbose):
for child in node.children:
replace(child, stringList, var, verbose)
def addTypeInfo(node, commentAttrib=None, item=None):
if commentAttrib == None:
if node.type == "constant" and node.get("value", False):
pass
elif node.type == "param":
addError(
node, "Parameter <code>%s</code> in not documented." %
commentAttrib.get("name"), item)
elif node.type == "return":
addError(node, "Return value is not documented.", item)
else:
addError(node, "Documentation is missing.", item)
return
# add description
if commentAttrib.has_key("text"):
node.addChild(tree.Node("desc").set("text", commentAttrib["text"]))
# add types
if commentAttrib.has_key("type"):
typesNode = tree.Node("types")
node.addChild(typesNode)
for item in commentAttrib["type"]:
itemNode = tree.Node("entry")
typesNode.addChild(itemNode)
itemNode.set("type", item["type"])
if item["dimensions"] != 0:
itemNode.set("dimensions", item["dimensions"])
# add default value
if commentAttrib.has_key("defaultValue"):
defaultValue = commentAttrib["defaultValue"]
if defaultValue != None:
# print "defaultValue: %s" % defaultValue
node.set("defaultValue", defaultValue)
def createBlockComment(txt):
l = "*****************************************************************************"
s = ""
s += "/*\n"
s += "%s\n" % l
s += " %s\n" % txt.upper()
s += "%s\n" % l
s += "*/"
bef = tree.Node("commentsBefore")
com = tree.Node("comment")
bef.addChild(com)
com.set("multiline", True)
com.set("connection", "before")
com.set("text", s)
com.set("detail", Comment.Comment(s).getFormat())
return bef
def createItemNode(type, stream):
# print "CREATE %s" % type
node = tree.Node(type)
node.set("line", stream.currLine())
node.set("column", stream.currColumn())
commentsBefore = stream.clearCommentsBefore()
for comment in commentsBefore:
node.addListChild("commentsBefore", comment)
return node
def createCommentNode(token):
commentNode = tree.Node("comment")
commentNode.set("line", token["line"])
commentNode.set("column", token["column"])
commentNode.set("text", token["source"])
commentNode.set("detail", token["detail"])
commentNode.set("multiline", token["multiline"])
commentNode.set("connection", token["connection"])
commentNode.set("begin", token["begin"])
commentNode.set("end", token["end"])
return commentNode
def addEventNode(classNode, classItem, commentAttrib):
node = tree.Node("event")
node.set("name", commentAttrib["name"])
if commentAttrib.has_key("text"):
node.addChild(tree.Node("desc").set("text", commentAttrib["text"]))
# add types
if commentAttrib.has_key("type"):
typesNode = tree.Node("types")
node.addChild(typesNode)
for item in commentAttrib["type"]:
itemNode = tree.Node("entry")
typesNode.addChild(itemNode)
itemNode.set("type", item["type"])
if item["dimensions"] != 0:
itemNode.set("dimensions", item["dimensions"])
classNode.addListChild("events", node)
def getClassNode(docTree, fullClassName, commentAttributes=None):
if commentAttributes == None:
commentAttributes = {}
dotIndex = fullClassName.rindex(".")
packageName = fullClassName[:dotIndex]
className = fullClassName[dotIndex + 1:]
package = getPackageNode(docTree, packageName)
classNode = package.getListChildByAttribute("classes", "name", className,
False)
if not classNode:
# The class does not exist -> Create it
classNode = tree.Node("class")
classNode.set("name", className)
classNode.set("fullName", fullClassName)
classNode.set("packageName",
fullClassName.replace("." + className, ""))
# Read all description, param and return attributes
for attrib in commentAttributes:
# Add description
if attrib["category"] == "description":
if attrib.has_key("text"):
descNode = tree.Node("desc").set("text", attrib["text"])
classNode.addChild(descNode)
elif attrib["category"] == "see":
if not attrib.has_key("name"):
printDocError(classNode, "Missing target for see.")
return classNode
seeNode = tree.Node("see").set("name", attrib["name"])
classNode.addChild(seeNode)
package.addListChild("classes", classNode)
return classNode
def createDoc(syntaxTree, docTree=None):
if not docTree:
docTree = tree.Node("doctree")
defineNode = findQxDefine(syntaxTree)
if defineNode != None:
variant = selectNode(defineNode, "operand/variable/2/@name").lower()
handleClassDefinition(docTree, defineNode, variant)
global hasDocError
ret = (docTree, hasDocError)
hasDocError = False
return ret
def createPackageDoc(text, packageName, docTree=None):
if not docTree:
docTree = tree.Node("doctree")
package = getPackageNode(docTree, packageName)
commentAttributes = comment.parseText(text)
# Read all description, param and return attributes
for attrib in commentAttributes:
# Add description
if attrib["category"] == "description":
if attrib.has_key("text"):
descNode = tree.Node("desc").set("text", attrib["text"])
package.addChild(descNode)
elif attrib["category"] == "see":
if not attrib.has_key("name"):
printDocError(classNode, "Missing target for see.")
return docTree
seeNode = tree.Node("see").set("name", attrib["name"])
package.addChild(seeNode)
return docTree
def inlineFunction(callNode, funcNode):
params = funcNode.getChild("params")
body = copy.copy(funcNode.getChild("body"))
# Without params is a lot easier
if params.hasChildren():
print " - TODO: With parameters"
else:
print " - Call without params"
# TODO: This is the tricky part doing the transformation from normal to inline
replNode = tree.Node("block")
callNode.parent.replaceChild(callNode, replNode)
def addError(node, msg, syntaxItem):
# print ">>> %s" % msg
errorNode = tree.Node("error")
errorNode.set("msg", msg)
(line, column) = getLineAndColumnFromSyntaxItem(syntaxItem)
if line:
errorNode.set("line", line)
if column:
errorNode.set("column", column)
node.addListChild("errors", errorNode)
node.set("hasError", True)
def handleInterfaceExtend(valueItem, classNode, docTree, className):
superInterfaceNames = variableOrArrayNodeToArray(valueItem)
for superInterface in superInterfaceNames:
superInterfaceNode = getClassNode(docTree, superInterface)
childInterfaces = superInterfaceNode.get("interfaces", False)
if childInterfaces:
childInterfaces += "," + className
else:
childInterfaces = className
superInterfaceNode.set("childClasses", childInterfaces)
node = tree.Node("interface")
node.set("name", superInterface)
classNode.addListChild("superInterfaces", node)
def handlePropertyGroup(propName, propDefinition, classNode):
node = tree.Node("property")
node.set("name", propName)
group = propDefinition["group"].getFirstChild()
groupMembers = [getValue(arrayItem) for arrayItem in group.children]
node.set("group", ",".join(groupMembers))
if propDefinition.has_key("mode"):
node.set("mode",
propDefinition["mode"].getChild("constant").get("value"))
if propDefinition.has_key("themeable"):
node.set("themeable",
propDefinition["themeable"].getChild("constant").get("value"))
return node
def createSyntaxTree(tokenArr, fileId=''):
stream = TokenStream(tokenArr)
stream.next()
#from pprint import pprint
#pprint([(x['detail'],x['source']) for x in tokenArr])
#pprint([x for x in tokenArr if x['type']=="comment"])
rootBlock = tree.Node("file")
rootBlock.set("file", stream.curr()["id"])
while not stream.finished():
rootBlock.addChild(readStatement(stream))
# collect prob. pending comments
for c in stream.commentsBefore:
rootBlock.addChild(c)
return rootBlock
def createSyntaxTree (tokenArr):
"""Creates a syntax tree from a token stream.
tokens: the token stream."""
stream = TokenStream(tokenArr)
stream.next()
rootBlock = tree.Node("file")
rootBlock.set("file", stream.curr()["id"])
while not stream.finished():
rootBlock.addChild(readStatement(stream))
# collect prob. pending comments
try:
for c in stream.commentsBefore: # stream.commentsBefore might not be defined
rootBlock.addChild(c)
except:
pass
return rootBlock
def getPackageNode(docTree, namespace):
currPackage = docTree
childPackageName = ""
for nsPart in namespace.split("."):
childPackage = currPackage.getListChildByAttribute(
"packages", "name", nsPart, False)
childPackageName += nsPart
if not childPackage:
# The package does not exist -> Create it
childPackage = tree.Node("package")
childPackage.set("name", nsPart)
childPackage.set("fullName", childPackageName)
childPackage.set("packageName",
childPackageName.replace("." + nsPart, ""))
currPackage.addListChild("packages", childPackage)
childPackageName += "."
# Update current package
currPackage = childPackage
return currPackage
def fill(node):
if node.type in ["comment", "commentsBefore", "commentsAfter"]:
return
if node.hasParent():
target = node
if node.type == "function":
name = node.get("name", False)
name = None if name == False else name
else:
name = ""
alternative = False
assignType = None
if name != None:
assignType = "function"
# move to hook operation
while target.parent.type in ["first", "second", "third"] and target.parent.parent.type == "operation" and target.parent.parent.get("operator") == "HOOK":
alternative = True
target = target.parent.parent
# move comment to assignment
while target.parent.type == "right" and target.parent.parent.type == "assignment":
target = target.parent.parent
if target.hasChild("left"):
left = target.getChild("left")
if left and left.hasChild("variable"):
var = left.getChild("variable")
last = var.getLastChild(False, True)
if last and last.type == "identifier":
name = last.get("name")
assignType = "object"
for child in var.children:
if child.type == "identifier":
if child.get("name") in ["prototype", "Proto"]:
assignType = "member"
elif child.get("name") in ["class", "base", "Class"]:
assignType = "static"
elif target.parent.type == "definition":
name = target.parent.get("identifier")
assignType = "definition"
# move to definition
if target.parent.type == "assignment" and target.parent.parent.type == "definition" and target.parent.parent.parent.getChildrenLength(True) == 1:
target = target.parent.parent.parent
assignType = "function"
# move comment to keyvalue
if target.parent.type == "value" and target.parent.parent.type == "keyvalue":
target = target.parent.parent
name = target.get("key")
assignType = "map"
if name == "construct":
assignType = "constructor"
if target.parent.type == "map" and target.parent.parent.type == "value" and target.parent.parent.parent.type == "keyvalue":
paname = target.parent.parent.parent.get("key")
if paname == "members":
assignType = "member"
elif paname == "statics":
assignType = "static"
# filter stuff, only add comments to member and static values and to all functions
if assignType in ["member", "static"] and node.type == "function":
if not hasattr(target, "documentationAdded") and target.parent.type != "params":
old = []
commentNode = None
# create commentsBefore
if target.hasChild("commentsBefore"):
commentsBefore = target.getChild("commentsBefore")
if commentsBefore.hasChild("comment"):
for child in commentsBefore.children:
if child.get("detail") in ["javadoc", "qtdoc"]:
old = Comment(child.get("text")).parse(False)
commentNode = child
commentNodeIndex = commentsBefore.children.index(child)
break
else:
commentsBefore = tree.Node("commentsBefore")
target.addChild(commentsBefore)
# create comment node
if commentNode == None:
commentNodeIndex = None
commentNode = tree.Node("comment")
commentNode.set("detail", "javadoc")
#if node.type == "function":
# commentNode.set("text", fromFunction(node, assignType, name, alternative, old))
#else:
# commentNode.set("text", fromNode(node, assignType, name, alternative, old))
commentNode.set("text", fromFunction(node, assignType, name, alternative, old))
commentNode.set("multiline", True)
commentsBefore.addChild(commentNode,commentNodeIndex)
# in case of alternative methods, use the first one, ignore the others
target.documentationAdded = True
if node.hasChildren():
for child in node.children:
fill(child)
def storeApi(self, include, apiPath, variantSet, verify):
self._console.info("Generating API data...")
self._console.indent()
docTree = tree.Node("doctree")
docTree.set("fullName", "")
docTree.set("name", "")
docTree.set("packageName", "")
length = len(include)
self._console.info("Loading class docs...", False)
self._console.indent()
packages = []
AttachMap = {}
hasErrors = False
for pos, fileId in enumerate(include):
self._console.progress(pos+1, length)
fileApi, attachMap = self.getApi(fileId, variantSet)
if fileApi == None:
hasErrors = True
# Only continue merging if there were no errors
if not hasErrors:
# update AttachMap
for cls in attachMap: # 'qx.Class', 'qx.core.Object', 'q', ...
if cls not in AttachMap:
AttachMap[cls] = attachMap[cls]
else:
for section in attachMap[cls]: # 'statics', 'members'
if section not in AttachMap[cls]:
AttachMap[cls][section] = attachMap[cls][section]
else:
for method in attachMap[cls][section]: # 'define', 'showToolTip', ...
if method not in AttachMap[cls][section]:
AttachMap[cls][section][method] = attachMap[cls][section][method]
else:
self._console.warn("Multiple @attach for same target '%s::%s#%s'." % (cls, section, method))
self._mergeApiNodes(docTree, fileApi)
pkgId = self._classesObj[fileId].package
# make sure all parent packages are included
nsparts = pkgId.split('.')
for i in range(len(nsparts)+1):
parentPkg = ".".join(nsparts[0:i])
if not parentPkg in packages:
packages.append(parentPkg)
self._console.outdent()
if hasErrors:
self._console.error("Found erroneous API information. Please see above. Stopping!")
return
self._console.info("Loading package docs...")
self._console.indent()
packages.sort()
for pkgId in packages:
self._mergeApiNodes(docTree, self.getPackageApi(pkgId))
self._console.outdent()
self._console.info("Connecting classes...")
api.connectPackage(docTree, docTree)
self._console.info("Generating search index...")
index = self.docTreeToSearchIndex(docTree, "", "", "")
if verify and "links" in verify:
self.verifyLinks(docTree, index)
self._console.info("Saving data...", False)
self._console.indent()
packageData = api.getPackageData(docTree)
packageJson = json.dumps(packageData)
filetool.save(os.path.join(apiPath, "apidata.json"), packageJson)
# apply the @attach information
for classData in api.classNodeIterator(docTree):
className = classData.get("fullName")
if className in AttachMap:
self._applyAttachInfo(className, classData, AttachMap[className])
# write per-class .json to disk
length = 0
for classData in api.classNodeIterator(docTree):
length += 1
pos = 0
for classData in api.classNodeIterator(docTree):
pos += 1
self._console.progress(pos, length)
nodeData = tree.getNodeData(classData)
nodeJson = json.dumps(nodeData)
fileName = os.path.join(apiPath, classData.get("fullName") + ".json")
filetool.save(fileName, nodeJson)
self._console.outdent()
# writ apiindex.json
self._console.info("Saving index...")
indexContent = json.dumps(index, separators=(',',':'), sort_keys=True) # compact encoding
filetool.save(os.path.join(apiPath, "apiindex.json"), indexContent)
self._console.outdent()
self._console.info("Done")