Exemple #1
0
 def unknown_globals(self, scope):
     # take advantage of Scope() objects
     for id_, scopeVar in scope.globals().items():
         if id_ in self.opts.allowed_globals:
             continue
         elif id_ in lang.GLOBALS: # JS built-ins ('alert' etc.)
             continue
         else:
             # we want to be more specific than just the left-most symbol,
             # like "qx", so let's look at the var uses
             for var_node in scopeVar.uses:
                 var_top = treeutil.findVarRoot(var_node)
                 full_name = (treeutil.assembleVariable(var_top))[0]
                 ok = False
                 if extension_match_in(full_name, self.known_globals_bases, 
                     self.opts.class_namespaces): # known classes (classList + their namespaces)
                     ok = True
                 else:
                     at_hints = get_at_hints(var_node) # check full_name against @ignore hints
                     if at_hints:
                         ok = ( self.is_name_ignore_filtered(full_name, at_hints)
                             or self.is_name_lint_filtered(full_name, at_hints, "ignoreUndefined")) # /**deprecated*/
                 if not ok:
                     issue = warn("Unknown global symbol used: '%s'" % full_name, self.file_name, var_node)
                     self.issues.append(issue)
Exemple #2
0
def optimize(classDefine, classDefNodes):
    patchCount = 0

    # get class map
    try:
        classMap = treeutil.getClassMap(classDefine)
    except tree.NodeAccessException:  # this might happen when the second param is not a map literal
        return 0

    if not "extend" in classMap:
        return 0

    if classMap["extend"].type == "variable":
        superClass = treeutil.assembleVariable(classMap["extend"])[0]
    else:
        return 0  # interfaces can have a list-valued "extend", but we currently don't optimize those

    if "construct" in classMap:
        patchCount = optimizeConstruct(classMap["construct"], superClass, "construct", classDefNodes)

    if not "members" in classMap:
        return patchCount

    members = classMap["members"]
    for methodName, methodNode in members.items():
        patchCount += optimizeConstruct(methodNode, superClass, methodName, classDefNodes)

    return patchCount
Exemple #3
0
        def checkPrivate(allVars):

            privateElement = re.compile(r'\b__')

            def findPrivate(allVars):
                variables = []
                for node in allVars:
                    fullName, isComplete = treeutil.assembleVariable(node)
                    if privateElement.search(fullName):
                        variables.append(node)
                return variables

            def isLocalPrivate(var, fullName):
                allIdentifier = fullName.split('.')
                first = second = None
                if len(allIdentifier) > 0:
                    first  = allIdentifier[0]
                if len(allIdentifier) > 1:
                    second = allIdentifier[1]
                return (first and
                        (first == "this" or first == "that") and 
                        second and
                        privateElement.match(second))
                
            variables = findPrivate(allVars)
            for var in variables:
                fullName = treeutil.assembleVariable(var)[0]
                if isLocalPrivate(var, fullName) and fullName.split('.')[1] not in restricted: # local privates are ok, as long as they are declared
                    self.log(var, "Undeclared private data field '%s'. You should list this field in the members section." % fullName)
            return
Exemple #4
0
 def findPrivate(allVars):
     variables = []
     for node in allVars:
         fullName, isComplete = treeutil.assembleVariable(node)
         if privateElement.search(fullName):
             variables.append(node)
     return variables
Exemple #5
0
    def unknown_globals(self, scope):
        # helper functions
        not_jsignored = inverse(gs.test_ident_is_jsignored)
        not_builtin = inverse(gs.test_ident_is_builtin())
        not_libsymbol = inverse(curry3(gs.test_for_libsymbol,
            self.opts.class_namespaces)(self.known_globals_bases))
        not_confsymbol = lambda node: globals_table[node] not in self.opts.allowed_globals
        def warn_appender(global_nodes):
            for node in global_nodes:
                issue = warn("Unknown global symbol used: '%s'" % globals_table[node], self.file_name, node)
                self.issues.append(issue)

        # ------------------------------
        # collect scope's global use locations
        globals_table = {} # {node: assembled}
        for id_, scopeVar in scope.globals().items():
            for head_node in scopeVar.uses:
                var_top = treeutil.findVarRoot(head_node)
                assembled = (treeutil.assembleVariable(var_top))[0]
                globals_table[head_node] = assembled

        # filter and add remains to warnings

        pipeline(
            globals_table.keys()
            , bind(filter, not_builtin)
            , bind(filter, not_jsignored)
            , bind(filter, not_libsymbol)
            , bind(filter, not_confsymbol)
            , warn_appender
        )
Exemple #6
0
 def unknown_globals(self, scope):
     # collect scope's global use locations
     global_nodes = defaultdict(list)  # {assembled: [node]}
     for id_, scopeVar in scope.globals().items():
         for head_node in scopeVar.uses:
             var_top = treeutil.findVarRoot(head_node)
             full_name = (treeutil.assembleVariable(var_top))[0]
             global_nodes[full_name].append(head_node)
     # filter allowed globals
     # - from config
     global_nodes = dict([(key,nodes) for (key,nodes) in global_nodes.items()
         if key not in self.opts.allowed_globals])
     # - from known classes and namespaces
     global_nodes = dict([(key,nodes) for (key,nodes) in global_nodes.items()
         if not extension_match_in(key, self.known_globals_bases,self.opts.class_namespaces)]) # known classes (classList + their namespaces)
     # - from built-ins
     new_keys = gs.globals_filter_by_builtins(global_nodes.keys())
     global_nodes = dict([(key,nodes) for (key,nodes) in global_nodes.items()
         if key in new_keys])
     # - with jshints
     for key, nodes in global_nodes.items():
         global_nodes[key] = [node for node in nodes 
             if not gs.ident_is_ignored(key, node)]
     # warn remaining
     for key, nodes in global_nodes.items():
         for node in nodes:
             issue = warn("Unknown global symbol used: '%s'" % key, self.file_name, node)
             self.issues.append(issue)
Exemple #7
0
def getFilterMap(callNode, fileId_):
    global fileId, verbose
    verbose = True
    fileId = fileId_

    result = {}
    if callNode.type != "call":
        return result

    operand = callNode.getChild("operand")
    if operand:
        operand_string, isComplete = treeutil.assembleVariable(operand.getChildByPosition(0))
    if not operand or not isComplete or operand_string != "qx.core.Environment.filter":
        log("Warning", "Can only work on qx.core.Environment.filter call. Ignoring this occurrence.", operand)
        return result

    params = callNode.getChild("params")
    if len(params.children) != 1:
        log(
            "Warning",
            "Expecting exactly one argument for qx.core.Environment.filter. Ignoring this occurrence.",
            params,
        )
        return result

    # Get the map from the find call
    firstParam = params.getChildByPosition(0)
    if not firstParam.type == "map":
        log("Warning", "First argument must be a map! Ignoring this occurrence.", firstParam)
        return result
    result = treeutil.mapNodeToMap(firstParam)

    return result
    def depsItem_from_node(self, node):
        scope = node.scope
        # some initializations (might get refined later)
        depsItem = DependencyItem('', '', '')
        depsItem.name           = ''
        depsItem.attribute      = ''
        depsItem.requestor      = self.id
        depsItem.line           = node.get("line", -1)
        depsItem.isLoadDep      = scope.is_load_time
        depsItem.needsRecursion = False
        depsItem.isCall         = False
        depsItem.node           = node
        var_root = treeutil.findVarRoot(node)  # various of the tests need the var (dot) root, rather than the head symbol (like 'qx')

        # .isCall
        if treeutil.isCallOperand(var_root): # it's a function call or new op.
            depsItem.isCall = True  # interesting when following transitive deps

        # .name, .attribute
        assembled = (treeutil.assembleVariable(node))[0]
        #className, classAttribute = self._splitQxClass(assembled)
        className = gs.test_for_libsymbol(assembled, ClassesAll, []) # TODO: no namespaces!?
        if not className: 
            is_lib_class = False
            className = assembled
            classAttribute = ''
        else:
            is_lib_class = True
            if len(assembled) > len(className):
                classAttribute = assembled[len(className)+1:]
            else:
                classAttribute = ''
        # we allow self-references, to be able to track method dependencies within the same class
        assembled_parts = assembled.split('.')
        if assembled_parts[0] == 'this':
            className = self.id
            is_lib_class = True
            if '.' in assembled:
                classAttribute = assembled_parts[1]
        elif scope.is_defer and assembled_parts[0] in DEFER_ARGS:
            className = self.id
            is_lib_class = True
            if '.' in assembled:
                classAttribute = assembled_parts[1]
        if is_lib_class and not classAttribute:  # see if we have to provide 'construct'
            if treeutil.isNEWoperand(var_root):
                classAttribute = 'construct'
        depsItem.name = className
        depsItem.attribute = classAttribute

        # .needsRecursion
        # Mark items that need recursive analysis of their dependencies (bug#1455)
        if (is_lib_class and
            scope.is_load_time and
            (treeutil.isCallOperand(var_root) or 
             treeutil.isNEWoperand(var_root))):
            depsItem.needsRecursion = True

        return depsItem
Exemple #9
0
 def findProtected(allVars):
     variables = []
     for node in allVars:
         # only check protected in lval position
         if (node.hasParent() and node.parent.type == "left" and
             node.parent.hasParent() and node.parent.parent.type == "assignment" and
             protectedElement.search(treeutil.assembleVariable(node)[0])):
             variables.append(node)
     return variables
    def depsItem_from_node(self, node):
        scope = node.scope
        # some initializations (might get refined later)
        depsItem = DependencyItem('', '', '')
        depsItem.name           = ''
        depsItem.attribute      = ''
        depsItem.requestor      = self.id
        depsItem.line           = node.get("line", -1)
        depsItem.isLoadDep      = scope.is_load_time
        depsItem.needsRecursion = False
        depsItem.isCall         = False
        depsItem.node           = node
        is_lib_class             = False
        var_root = treeutil.findVarRoot(node)  # various of the tests need the var (dot) root, rather than the head symbol (like 'qx')

        # .isCall
        if treeutil.isCallOperand(var_root): # it's a function call or new op.
            depsItem.isCall = True  # interesting when following transitive deps

        # .name, .attribute
        assembled = (treeutil.assembleVariable(node))[0]
        className, classAttribute = self._splitQxClass(assembled)
        if not className: 
            if "." in assembled:
                className, classAttribute = assembled.split('.')[:2]
            else:
                className = assembled
        else:
            is_lib_class = True
        # we allow self-references, to be able to track method dependencies within the same class
        if self.is_this(className):
            if className.find('.')>-1:
                classAttribute = className.split('.')[1]
            className = self.id
            is_lib_class = True
        elif scope.is_defer and className in DEFER_ARGS:
            className = self.id
            is_lib_class = True
        if is_lib_class and not classAttribute:  # see if we have to provide 'construct'
            if treeutil.isNEWoperand(var_root):
                classAttribute = 'construct'
        depsItem.name = className
        depsItem.attribute = classAttribute

        # .needsRecursion
        # Mark items that need recursive analysis of their dependencies (bug#1455)
        #if self.followCallDeps(var_root, self.id, className, isLoadTime):
        #if self.id=='qx.bom.element.Style' and depsItem.attribute=='__detectVendorProperties':
        #    import pydb; pydb.debugger()
        if (is_lib_class and
            scope.is_load_time and
            (treeutil.isCallOperand(var_root) or 
             treeutil.isNEWoperand(var_root))):
            depsItem.needsRecursion = True

        return depsItem
Exemple #11
0
 def function_uses_local_privs(self, func_node):
     function_privs = set()
     reg_this_aliases = re.compile(r'\b%s' % "|".join(self.this_aliases))
     scope = func_node.scope
     for id_,scopeVar in scope.vars.items():
         if reg_this_aliases.match(id_):
             for var_use in scopeVar.uses:
                 full_name = treeutil.assembleVariable(var_use)[0]
                 name_parts = full_name.split(".")
                 if len(name_parts) > 1 and self.reg_privs.match(name_parts[1]):
                     function_privs.add((name_parts[1],var_use))
     return function_privs
    def _analyzeClassDepsNode(self, node, depsList, variants, inLoadContext, inDefer=False):

        if node.type == "variable":
            assembled = (treeutil.assembleVariable(node))[0]

            # treat dependencies in defer as requires
            deferNode = self.checkDeferNode(assembled, node)
            if deferNode != None:
                self._analyzeClassDepsNode(deferNode, depsList, variants, inLoadContext=True, inDefer=True)

            (context, className, classAttribute) = self._isInterestingReference(assembled, node, self.id, inDefer)
            # postcond: 
            # - if className != '' it is an interesting reference
            # - might be a known qooxdoo class, or an unknown class (use 'className in self._classes')
            # - if assembled contained ".", classAttribute will contain approx. non-class part

            if className:
                # we allow self-references, to be able to track method dependencies within the same class
                if className == 'this':
                    className = self.id
                elif inDefer and className in DEFER_ARGS:
                    className = self.id
                if not classAttribute:  # see if we have to provide 'construct'
                    if node.hasParentContext("instantiation/*/*/operand"): # 'new ...' position
                        classAttribute = 'construct'
                depsItem = DependencyItem(className, classAttribute, self.id, node.get('line', -1), inLoadContext)
                #print "-- adding: %s (%s:%s)" % (className, treeutil.getFileFromSyntaxItem(node), node.get('line',False))
                if node.hasParentContext("call/operand"): # it's a function call
                    depsItem.isCall = True  # interesting when following transitive deps

                # Adding all items to list; let caller sort things out
                depsList.append(depsItem)

                # Mark items that need recursive analysis of their dependencies (bug#1455)
                if self.followCallDeps(node, self.id, className, inLoadContext):
                    depsItem.needsRecursion = True

        elif node.type == "body" and node.parent.type == "function":
            if (node.parent.hasParentContext("call/operand") 
                or node.parent.hasParentContext("call/operand/group")):
                # if the function is immediately called, it's still load context (if that's what it was before)
                pass
            else:
                inLoadContext = False

        if node.hasChildren():
            for child in node.children:
                self._analyzeClassDepsNode(child, depsList, variants, inLoadContext, inDefer)

        return
Exemple #13
0
 def visit_function(self, node):
     assert hasattr(node, 'scope'), ".scope attribute missing on function; was this tree scope-analyzed?!"
     #print "visiting", node.type
     global_syms = node.scope.globals()
     for id_, scopeVar in global_syms:
         for var_node in scopeVar.uses:
             full_name = (treeutil.assembleVariable(var_node))[0]
             if self.is_ignored_global(full_name, scopeVar): # built-ins, known name spaces, @ignore's
                 continue
             depsItem = self.depsItem_from_node(full_name, var_node)
             self.append(depsItem)
     # recurse
     for cld in node.children:
         self.visit(cld)
    def qualify_deps_item(self, node, isLoadTime, inDefer, depsItem=None):
        if not depsItem:
            depsItem = DependencyItem('', '', '')
        depsItem.name = ''
        depsItem.attribute = ''
        depsItem.requestor = self.id
        depsItem.line = node.get("line", -1)
        depsItem.isLoadDep = isLoadTime
        depsItem.needsRecursion = False
        depsItem.isCall = False
        var_root = treeutil.findVarRoot(node)  # various of the tests need the var (dot) root, rather than the head symbol (like 'qx')

        # .isCall
        if var_root.hasParentContext("call/operand"): # it's a function call
            depsItem.isCall = True  # interesting when following transitive deps

        # .name
        assembled = (treeutil.assembleVariable(node))[0]
        _, className, classAttribute = self._isInterestingReference(assembled, var_root, self.id, inDefer)
        # postcond: 
        # - className != '' must always be true, as we know it is an interesting reference
        # - might be a known qooxdoo class, or an unknown class (use 'className in self._classes')
        # - if assembled contained ".", classAttribute will contain approx. non-class part

        if className:
            # we allow self-references, to be able to track method dependencies within the same class
            if className == 'this':
                className = self.id
            elif inDefer and className in DEFER_ARGS:
                className = self.id
            if not classAttribute:  # see if we have to provide 'construct'
                if treeutil.isNEWoperand(node):
                    classAttribute = 'construct'
            # Can't do the next; it's catching too many occurrences of 'getInstance' that have
            # nothing to do with the singleton 'getInstance' method (just grep in the framework)
            #elif classAttribute == 'getInstance':  # erase 'getInstance' and introduce 'construct' dependency
            #    classAttribute = 'construct'
            depsItem.name = className
            depsItem.attribute = classAttribute

            # .needsRecursion
            # Mark items that need recursive analysis of their dependencies (bug#1455)
            if self.followCallDeps(var_root, self.id, className, isLoadTime):
                depsItem.needsRecursion = True

        if depsItem.name:
            return depsItem
        else:
            return None
Exemple #15
0
 def function_used_deprecated(self, funcnode):
     # take advantage of Scope() objects
     scope = funcnode.scope
     for id_, scopeVar in scope.globals().items():
         # id_ might be an incomplete class id, like "qx"
         # let's look at the var uses
         for var_node in scopeVar.uses:
             full_name = (treeutil.assembleVariable(var_node))[0]
             ok = True
             if (full_name in lang.GLOBALS # JS built-ins ('alert' etc.)
                     and full_name in lang.DEPRECATED):
                 ok = False
                 at_hints = get_at_hints(var_node) # check full_name against @ignore hints
                 if at_hints:
                     ok = self.is_name_lint_filtered(full_name, at_hints, "ignoreDeprecated")
             if not ok:
                 warn("Deprecated global symbol used: '%s'" % full_name, self.file_name, var_node)
    def _findTranslationBlocks(self, node, translation):
        if node.type == "call":
            oper = node.getChild("operand", False)
            if oper:
                var = oper.getChild("variable", False)
                if var:
                    varname = (treeutil.assembleVariable(var))[0]
                    for entry in [ ".tr", ".trn", ".trc", ".marktr" ]:
                        if varname.endswith(entry):
                            self._addTranslationBlock(entry[1:], translation, node, var)
                            break

        if node.hasChildren():
            for child in node.children:
                self._findTranslationBlocks(child, translation)

        return translation
Exemple #17
0
    def _analyzeClassDepsNode(self, node, loadtime, runtime, inFunction, variants):

        fileId = self.id

        if node.type == "variable":
            assembled = (treeutil.assembleVariable(node))[0]

            # treat dependencies in defer as requires
            deferNode = self.checkDeferNode(assembled, node)
            if deferNode != None:
                self._analyzeClassDepsNode(deferNode, loadtime, runtime, False, variants)

            (context, className, classAttribute) = self._isInterestingReference(assembled, node, fileId)
            # postcond: 
            # - if className != '' it is an interesting reference
            # - might be a known qooxdoo class, or an unknown class (use 'className in self._classes')
            # - if assembled contained ".", classAttribute will contain approx. non-class part

            if className:
                # we allow self-references, to be able to track method dependencies within the same class
                if className == 'this':
                    className = fileId
                #print "-- adding: %s (%s:%s)" % (className, treeutil.getFileFromSyntaxItem(node), node.get('line',False))
                depsItem = DependencyItem(className, node.get('line', -1), classAttribute)
                self.addDep(depsItem, inFunction, runtime, loadtime)

                # an attempt to fix static initializers (bug#1455)
                if not inFunction and self.followCallDeps(node, fileId, className):
                    console.debug("Looking for rundeps in call to '%s' of '%s'(%d)" % (assembled, fileId, depsItem.line))
                    console.indent()
                    # getMethodDeps is mutual recursive calling into the current
                    # function, but only does so with inFunction=True, so this
                    # branch is never hit through the recursive call
                    ldeps = self.getMethodDeps(depsItem, variants)
                    loadtime.extend([x for x in ldeps if x not in loadtime]) # add uniquely
                    console.outdent()

        elif node.type == "body" and node.parent.type == "function":
            inFunction = True

        if node.hasChildren():
            for child in node.children:
                self._analyzeClassDepsNode(child, loadtime, runtime, inFunction, variants)

        return
Exemple #18
0
def optimizeConstruct(node, superClass, methodName, classDefNodes):
    patchCount = 0

    # Need to run through all the nodes, to skip embedded qx.*.define(),
    # which will be treated separately

    # Handle Node

    # skip embedded qx.*.define()
    if node in classDefNodes:
        return 0

    elif node.type == "variable" and node.hasParentContext("call/operand"):

        varName, complete = treeutil.assembleVariable(node)
        if not (complete and varName == "this.base"):
            return 0

        call = node.parent.parent

        try:
            firstArgName = treeutil.selectNode(call, "params/1/identifier/@name")
        except tree.NodeAccessException:
            return 0

        if firstArgName != "arguments":
            return 0

        # "construct"
        if methodName == "construct":
            newCall = treeutil.compileString("%s.call()" % superClass)
        # "member"
        else:
            newCall = treeutil.compileString("%s.prototype.%s.call()" % (superClass, methodName))
        newCall.replaceChild(newCall.getChild("params"), call.getChild("params"))  # replace with old arglist
        treeutil.selectNode(newCall, "params/1/identifier").set("name", "this")  # arguments -> this
        call.parent.replaceChild(call, newCall)
        patchCount += 1

    # Handle Children
    if node.hasChildren():
        for child in node.children:
            patchCount += optimizeConstruct(child, superClass, methodName, classDefNodes)

    return patchCount
Exemple #19
0
        def checkProtected(allVars):

            protectedElement = re.compile(r"\b_[^_]")

            def findProtected(allVars):
                variables = []
                for node in allVars:
                    # only check protected in lval position
                    if (
                        node.hasParent()
                        and node.parent.type == "left"
                        and node.parent.hasParent()
                        and node.parent.parent.type == "assignment"
                        and protectedElement.search(treeutil.assembleVariable(node)[0])
                    ):
                        variables.append(node)
                return variables

            def protectedIsLastVarChild(var):
                lastChild = var.getLastChild(ignoreComments=True)  # like "this.a.b" -> b
                if lastChild.type != "identifier":  # rules out this.a._prot[0] which isn't a call anyway
                    return False
                name = treeutil.selectNode(lastChild, "@name")
                if name and protectedElement.match(name):
                    return True
                else:
                    return False

            variables = findProtected(allVars)
            for var in variables:
                # check call with protected "..._protected()..."
                # if (
                #    protectedIsLastVarChild(var) and   # like "this.a.b._protected()", not "this.a._protected.b()"
                #    var.hasParent() and var.parent.type == "operand" and  # parent is "operand"
                #    var.parent.hasParent() and var.parent.parent.type == "call"  # grandparent is "call"
                #    ):   # it's ok as method call
                #    pass
                # else:
                self.log(
                    var,
                    "Protected data field in '%s'. Protected data fields are deprecated. Better use private fields in combination with getter and setter methods."
                    % treeutil.assembleVariable(var)[0],
                )
            return
Exemple #20
0
        def checkImplicit(allVars):

            def hasUndeclaredMember(fullName):
                allIdentifier = fullName.split('.')
                first = second = None
                if len(allIdentifier) > 0:
                    first  = allIdentifier[0]
                if len(allIdentifier) > 1:
                    second = allIdentifier[1]
                return (first and
                        (first == "this" or first == "that") and 
                        second and
                        second not in restricted)     # <- this is bogus, too narrow
                
            for var in allVars:
                fullName = treeutil.assembleVariable(var)[0]
                if hasUndeclaredMember(fullName):
                    self.log(var, "Undeclared local data field in '%s'! You should list this field in the member section." % fullName)

            return
Exemple #21
0
def test_for_libsymbol(symbol, class_names, name_spaces):
    res_name = ""

    # node may be unicode string or Node obj => unify
    if not isinstance(symbol, unicode):
        symbol = treeutil.assembleVariable(symbol)[0]

    # check for a name space match
    if symbol in name_spaces:
        res_name = symbol
    # see if symbol is a (dot-exact) prefix of any of class_names
    else:
        for class_name in class_names:
            if symbol.startswith(class_name) and re.search(r"^%s(?=\.|$)" % re.escape(class_name), symbol):
                # e.g. re.search(r'^mylib.Foo(?=\.|$)', 'mylib.Foo.Bar' is
                # true, but not with 'mylib.FooBar'
                # take the longest match
                if len(class_name) > len(res_name):
                    res_name = class_name
    return res_name
def search_loop(node, stringMap={}, verbose=False):
    if node.type == "call":
        oper = node.getChild("operand", False)

        if oper:
            variable = oper.getChild("variable", False)

            if variable:
                try:
                    variableName = (treeutil.assembleVariable(variable))[0]
                except tree.NodeAccessException:
                    variableName = None

                # Don't extract from locales
                if variableName == "qx.locale.Locale.define" or variableName == "qx.Locale.define":
                    return stringMap

    if node.type == "constant" and node.get("constantType") == "string":
        if verbose:
            pvalue = node.get("value")
            if isinstance(pvalue, unicode):
                pvalue = pvalue.encode("utf-8")
            print "      - Found: '%s'" % pvalue

        if node.get("detail") == "singlequotes":
            quote = "'"
        elif node.get("detail") == "doublequotes":
            quote = '"'

        value = "%s%s%s" % (quote, node.get("value"), quote)

        if value in stringMap:
            stringMap[value] += 1
        else:
            stringMap[value] = 1

    if check(node, verbose):
        for child in node.children:
            search_loop(child, stringMap, verbose)

    return stringMap
    def _analyzeClassDepsNode(self, fileId, node, loadtime, runtime, inFunction):
        if node.type == "variable":
            assembled = (treeutil.assembleVariable(node))[0]

            # treat dependencies in defer as requires
            if assembled == "qx.Class.define" or assembled == "qx.Bootstrap.define":
                if node.parent.type == "operand" and node.parent.parent.type == "call":
                    deferNode = treeutil.selectNode(node, "../../params/2/keyvalue[@key='defer']/value/function/body/block")
                    if deferNode != None:
                        self._analyzeClassDepsNode(fileId, deferNode, loadtime, runtime, False)


            # try to reduce to a class name
            assembledId = None
            if self._classes.has_key(assembled):
                assembledId = assembled

            elif "." in assembled:
                for entryId in self._classes:
                    if assembled.startswith(entryId) and re.match("%s\W" % entryId, assembled):
                        assembledId = entryId
                        break

            if assembledId and assembledId != fileId and self._classes.has_key(assembledId):
                if inFunction:
                    target = runtime
                else:
                    target = loadtime

                if not assembledId in target:
                    target.append(assembledId)

        elif node.type == "body" and node.parent.type == "function":
            inFunction = True

        if node.hasChildren():
            for child in node.children:
                self._analyzeClassDepsNode(fileId, child, loadtime, runtime, inFunction)
        def getReferencesFromSource(fileId, node, runtime):
            # the "variants" param is only to support getMethodDeps()!

            ##
            # currently interesting are 
            # - 'new' operands ("new qx.ui.form.Button(...)"), and 
            # - call operands ("qx.core.Variant.select(...)")

            def isInterestingReference(assembled, node, fileId):
                # check name in 'new ...' position
                if (node.hasParentContext("instantiation/*/*/operand")
                # check name in call position
                or (node.hasParentContext("call/operand"))):
                    # skip built-in classes (Error, document, RegExp, ...)
                    for bi in lang.BUILTIN + ['clazz']:
                        if re.search(r'^%s\b' % bi, assembled):
                            return False
                    # skip scoped vars - expensive, therefore last test
                    if self._isScopedVar(assembled, node, fileId):
                        return False
                    else:
                        return True

                return False
            
            # -----------------------------------------------------------

            if node.type == "variable":
                assembled = (treeutil.assembleVariable(node))[0]

                if isInterestingReference(assembled, node, fileId):
                    runtime.append(assembled)

            if node.hasChildren():
                for child in node.children:
                    getReferencesFromSource(fileId, child, runtime)

            return
Exemple #25
0
    def depsItem_from_node(self, node):
        scope = node.scope
        # some initializations (might get refined later)
        depsItem = DependencyItem('', '', '')
        depsItem.name = ''
        depsItem.attribute = ''
        depsItem.requestor = self.id
        depsItem.line = node.get("line", -1)
        depsItem.isLoadDep = scope.is_load_time
        depsItem.needsRecursion = False
        depsItem.isCall = False
        depsItem.node = node
        is_lib_class = False
        var_root = treeutil.findVarRoot(
            node
        )  # various of the tests need the var (dot) root, rather than the head symbol (like 'qx')

        # .isCall
        if treeutil.isCallOperand(var_root):  # it's a function call or new op.
            depsItem.isCall = True  # interesting when following transitive deps

        # .name, .attribute
        assembled = (treeutil.assembleVariable(node))[0]
        className, classAttribute = self._splitQxClass(assembled)
        assembled_parts = assembled.split('.')
        if not className:
            if "." in assembled:
                className = '.'.join(assembled_parts[:-1])
                classAttribute = assembled_parts[-1]
                #className, classAttribute = assembled.split('.')[:2]
            else:
                className = assembled
        else:
            is_lib_class = True
        # we allow self-references, to be able to track method dependencies within the same class
        if assembled_parts[0] == 'this':
            className = self.id
            is_lib_class = True
            if '.' in assembled:
                classAttribute = assembled_parts[1]
        elif scope.is_defer and assembled_parts[0] in DEFER_ARGS:
            className = self.id
            is_lib_class = True
            if '.' in assembled:
                classAttribute = assembled_parts[1]
        if is_lib_class and not classAttribute:  # see if we have to provide 'construct'
            if treeutil.isNEWoperand(var_root):
                classAttribute = 'construct'
        depsItem.name = className
        depsItem.attribute = classAttribute

        # .needsRecursion
        # Mark items that need recursive analysis of their dependencies (bug#1455)
        #if self.followCallDeps(var_root, self.id, className, isLoadTime):
        #if self.id=='qx.bom.element.Style' and depsItem.attribute=='__detectVendorProperties':
        #    import pydb; pydb.debugger()
        if (is_lib_class and scope.is_load_time
                and (treeutil.isCallOperand(var_root)
                     or treeutil.isNEWoperand(var_root))):
            depsItem.needsRecursion = True

        return depsItem
Exemple #26
0
    def _analyzeClassDepsNode(self, node, depsList, inLoadContext, inDefer=False):

        if node.type == "variable":
            assembled = (treeutil.assembleVariable(node))[0]

            # treat dependencies in defer as requires
            deferNode = self.checkDeferNode(assembled, node)
            if deferNode != None:
                self._analyzeClassDepsNode(deferNode, depsList, inLoadContext=True, inDefer=True)

            (context, className, classAttribute) = self._isInterestingReference(assembled, node, self.id, inDefer)
            # postcond: 
            # - if className != '' it is an interesting reference
            # - might be a known qooxdoo class, or an unknown class (use 'className in self._classes')
            # - if assembled contained ".", classAttribute will contain approx. non-class part

            if className:
                # we allow self-references, to be able to track method dependencies within the same class
                if className == 'this':
                    className = self.id
                elif inDefer and className in DEFER_ARGS:
                    className = self.id
                if not classAttribute:  # see if we have to provide 'construct'
                    if node.hasParentContext("instantiation/*/*/operand"): # 'new ...' position
                        classAttribute = 'construct'
                # Can't do the next; it's catching too many occurrences of 'getInstance' that have
                # nothing to do with the singleton 'getInstance' method (just grep in the framework)
                #elif classAttribute == 'getInstance':  # erase 'getInstance' and introduce 'construct' dependency
                #    classAttribute = 'construct'
                depsItem = DependencyItem(className, classAttribute, self.id, node.get('line', -1), inLoadContext)
                #print "-- adding: %s (%s:%s)" % (className, treeutil.getFileFromSyntaxItem(node), node.get('line',False))
                if node.hasParentContext("call/operand"): # it's a function call
                    depsItem.isCall = True  # interesting when following transitive deps

                # Adding all items to list; let caller sort things out
                depsList.append(depsItem)

                # Mark items that need recursive analysis of their dependencies (bug#1455)
                if self.followCallDeps(node, self.id, className, inLoadContext):
                    depsItem.needsRecursion = True

        # check e.g. qx.core.Environment.get("runtime.name")
        elif node.type == "constant" and node.hasParentContext("call/params"):
            callnode = treeutil.selectNode(node, "../..")
            if variantoptimizer.isEnvironmentCall(callnode):
                className, classAttribute = self.getClassNameFromEnvKey(node.get("value", ""))
                if className:
                    depsItem = DependencyItem(className, classAttribute, self.id, node.get('line', -1), inLoadContext)
                    depsItem.isCall = True  # treat as if actual call, to collect recursive deps
                    depsList.append(depsItem)


        elif node.type == "body" and node.parent.type == "function":
            if (node.parent.hasParentContext("call/operand") 
                or node.parent.hasParentContext("call/operand/group")):
                # if the function is immediately called, it's still load context (if that's what it was before)
                pass
            else:
                inLoadContext = False

        if node.hasChildren():
            for child in node.children:
                self._analyzeClassDepsNode(child, depsList, inLoadContext, inDefer)

        return
Exemple #27
0
    def findClassForFeature(self, featureId, variants, classMaps):

        # get the method name
        clazzId = self.id
        if  featureId == u'':  # corner case: bare class reference outside "new ..."
            return clazzId, featureId
        # TODO: The next doesn't provide much, qx.Class.getInstance has no new dependencies
        # currently (aside from "new this", which I cannot relate back to 'construct'
        # ATM). Leave it in anyway, to not break bug#5660.
        #elif featureId == "getInstance": # corner case: singletons get this from qx.Class
        #    clazzId = "qx.Class"
        elif featureId == "getInstance" and self.type == "singleton":
            featureId = "construct"
        elif featureId in ('call', 'apply'):  # this might get overridden, oh well...
            clazzId = "Function"
        # TODO: getter/setter are also not lexically available!
        # handle .call() ?!
        if clazzId not in self._classesObj: # can't further process non-qooxdoo classes
            # TODO: maybe this should better use something like isInterestingIdentifier()
            # to invoke the same machinery for filtering references like in other places
            return None, None

        # early return if class id is finalized
        if clazzId != self.id:
            classObj = self._classesObj[clazzId]
            featureNode = self.getFeatureNode(featureId, variants)
            if featureNode:
                return clazzId, featureNode
            else:
                return None, None

        # now try this class
        if self.id in classMaps:
            classMap = classMaps[self.id]
        else:
            classMap = classMaps[self.id] = self.getClassMap (variants)
        featureNode = self.getFeatureNode(featureId, variants, classMap)
        if featureNode:
            return self.id, featureNode

        if featureId == 'construct':  # constructor requested, but not supplied in class map
            # supply the default constructor
            featureNode = treeutil.compileString("function(){this.base(arguments);}", self.path)
            return self.id, featureNode

        # inspect inheritance/mixins
        parents = []
        extendVal = classMap.get('extend', None)
        if extendVal:
            extendVal = treeutil.variableOrArrayNodeToArray(extendVal)
            parents.extend(extendVal)
            # this.base calls
            if featureId == "base":
                classId = parents[0]  # first entry must be super-class
                if classId in self._classesObj:
                    return self._classesObj[classId].findClassForFeature('construct', variants, classMaps)
                else:
                    return None, None
        includeVal = classMap.get('include', None)
        if includeVal:
            # 'include' value according to Class spec.
            if includeVal.type in ('variable', 'array'):
                includeVal = treeutil.variableOrArrayNodeToArray(includeVal)
            
            # assume qx.core.Environment.filter() call
            else:
                filterMap = variantoptimizer.getFilterMap(includeVal, self.id)
                includeSymbols = []
                for key, node in filterMap.items():
                    # only consider true or undefined 
                    #if key not in variants or (key in variants and bool(variants[key]):
                    # map value has to be value/variable
                    variable =  node.children[0]
                    assert variable.type == "variable"
                    symbol, isComplete = treeutil.assembleVariable(variable)
                    assert isComplete
                    includeSymbols.append(symbol)
                includeVal = includeSymbols

            parents.extend(includeVal)

        # go through all ancestors
        for parClass in parents:
            if parClass not in self._classesObj:
                continue
            parClassObj = self._classesObj[parClass]
            rclass, keyval = parClassObj.findClassForFeature(featureId, variants, classMaps)
            if rclass:
                return rclass, keyval
        return None, None
Exemple #28
0
 def test(node):
     name = treeutil.assembleVariable(node)[0]
     return test_for_libsymbol(name, class_names, name_spaces)
    def _analyzeClassDepsNode(self, fileId, node, loadtime, runtime, warn, inFunction, variants):
        # analyze a class AST for dependencies (compiler hints not treated here)
        # does not follow dependencies to other classes (ie. it's a "shallow" analysis)!
        # the "variants" param is only to support getMethodDeps()!

        def checkDeferNode(assembled, node):
            deferNode = None
            if assembled == "qx.Class.define" or assembled == "qx.Bootstrap.define" or assembled == "qx.List.define":
                if node.hasParentContext("call/operand"):
                    deferNode = treeutil.selectNode(node, "../../params/2/keyvalue[@key='defer']/value/function/body/block")
            return deferNode

        def reduceAssembled(assembled, node):
            # try to deduce a qooxdoo class from <assembled>
            assembledId = ''
            if assembled in self._classes:
                assembledId = assembled
            elif "." in assembled:
                for entryId in self._classes:
                    if assembled.startswith(entryId) and re.match(r'%s\b' % entryId, assembled):
                        if len(entryId) > len(assembledId): # take the longest match
                            assembledId = entryId
            return assembledId

        def reduceAssembled1(assembled, node):
            def tryKnownClasses(assembled):
                result = ''
                for entryId in self._classes.keys() + ["this"]:
                    if assembled.startswith(entryId) and re.match(r'%s\b' % entryId, assembled):
                        if len(entryId) > len(assembledId): # take the longest match
                            result = entryId
                return result

            def tryReduceClassname(assembled, node):
                result = ''
                # 'new <name>()'
                if (node.hasParentContext("instantiation/*/*/operand")):
                    result = assembled  # whole <name>
                # '"extend" : <name>'
                elif (node.hasParentContext("keyvalue/*") and node.parent.parent.get('key') == 'extend'):
                    result = assembled  # whole <name>
                # 'call' functor
                elif (node.hasParentContext("call/operand")):
                    result = assembled[:assembled.rindex('.')] # drop the method name after last '.'
                return result

            if assembled in self._classes:
                assembledId = assembled
            elif "." in assembled:
                assembledId = tryKnownClasses(assembled)
                if not assembledId:
                    assembledId = tryReduceClassname(assembled, node)
            if not assembledId:
                assembledId = assembled
            return assembledId

        def isUnknownClass(assembled, node, fileId):
            # check name in 'new ...' position
            if (node.hasParentContext("instantiation/*/*/operand")
            # check name in "'extend' : ..." position
            or (node.hasParentContext("keyvalue/*") and node.parent.parent.get('key') == 'extend')):
                # skip built-in classes (Error, document, RegExp, ...)
                if (assembled in lang.BUILTIN + ['clazz'] or re.match(r'this\b', assembled)):
                   return False
                # skip scoped vars - expensive, therefore last test
                elif self._isScopedVar(assembled, node, fileId):
                    return False
                else:
                    return True

            return False
        
        def addId(assembledId, runtime, loadtime, lineno):
            if inFunction:
                target = runtime
            else:
                target = loadtime

            if not assembledId in (x.name for x in target):
                target.append(DependencyItem(assembledId, lineno))

            if (not inFunction and  # only for loadtime items
                self._jobconf.get("dependencies/follow-static-initializers", False) and
                node.hasParentContext("call/operand")  # it's a method call
               ):  
                deps = self.getMethodDeps(assembledId, assembled, variants)
                loadtime.extend([x for x in deps if x not in loadtime]) # add uniquely

            return


        def followCallDeps(assembledId):
            if (assembledId and
                assembledId in self._classes and       # we have a class id
                assembledId != fileId and
                self._jobconf.get("dependencies/follow-static-initializers", False) and
                node.hasParentContext("call/operand")  # it's a method call
               ):
                return True
            return False


        def splitClassAttribute(assembledId, assembled):
            if assembledId == assembled:  # just a class id
                clazzId   = assembledId
                attribute = u''
            else:
                clazzId   = assembledId
                attribute = assembled[ len(assembledId) +1 :] # a.b.c.d = a.b.c + '.' + d
                
            return clazzId, attribute

        # -----------------------------------------------------------

        if node.type == "variable":
            assembled = (treeutil.assembleVariable(node))[0]

            # treat dependencies in defer as requires
            deferNode = checkDeferNode(assembled, node)
            if deferNode != None:
                self._analyzeClassDepsNode(fileId, deferNode, loadtime, runtime, warn, False, variants)

            # try to reduce to a class name
            assembledId = reduceAssembled(assembled, node)

            (context, className, classAttribute) = self._isInterestingReference(assembled, node, fileId)
            # postcond: 
            # - if className != '' it is an interesting reference
            # - might be a known qooxdoo class, or an unknown class (use 'className in self._classes')
            # - if assembled contained ".", classAttribute will contain approx. non-class part

            #if assembledId:
            #    if assembledId in self._classes and assembledId != fileId:
            #        #print "-- adding: %s" % assembledId
            #        #print "-- nameba: %s" % className
            #        #if not className: import pydb; pydb.debugger()
            #        addId(assembledId, runtime, loadtime)
            #else:
            #    if isUnknownClass(assembled, node, fileId):
            #        #print "-- warning: %s" % assembled
            #        #print "-- namebas: %s" % className
            #        warn.append(assembled)

            if className:
                if className != fileId: # not checking for self._classes here!
                    #print "-- adding: %s (%s:%s)" % (className, treeutil.getFileFromSyntaxItem(node), node.get('line',False))
                    addId(className, runtime, loadtime, node.get('line', -1))

            # an attempt to fix static initializers (bug#1455)
            if not inFunction and followCallDeps(assembledId):
                self._console.debug("Looking for rundeps in '%s' of '%s'" % (assembled, assembledId))
                if False: # use old getMethodDeps()
                    ldeps = self.getMethodDeps(assembledId, assembled, variants)
                    # getMethodDeps is mutual recursive calling into the current function, but
                    # only does so with inFunction=True, so this branch is never hit through the
                    # recursive call
                    # make run-time deps of the called method load-deps of the current
                    loadtime.extend([x for x in ldeps if x not in loadtime]) # add uniquely
                else: # new getMethodDeps()
                    self._console.indent()
                    classId, attribId = splitClassAttribute(assembledId, assembled)
                    ldeps = self.getMethodDeps1(classId, attribId, variants)
                    ld = [x[0] for x in ldeps]
                    loadtime.extend([x for x in ld if x not in loadtime]) # add uniquely
                    self._console.outdent()

        elif node.type == "body" and node.parent.type == "function":
            inFunction = True

        if node.hasChildren():
            for child in node.children:
                self._analyzeClassDepsNode(fileId, child, loadtime, runtime, warn, inFunction, variants)

        return
Exemple #30
0
def test_ident_is_jsignored(node):
    var_root = treeutil.findVarRoot(node)
    name = treeutil.assembleVariable(var_root)[0]
    return name_is_jsignored(name, node)
Exemple #31
0
 def test(node):
     var_root = treeutil.findVarRoot(node)
     name = treeutil.assembleVariable(var_root)[0]
     return bool(GlobalSymbolsCombinedPatt.search(name))
Exemple #32
0
def test_ident_is_jsignored(node):
    var_root = treeutil.findVarRoot(node)
    name = treeutil.assembleVariable(var_root)[0]
    return name_is_jsignored(name, node)
Exemple #33
0
 def test(node):
     var_root = treeutil.findVarRoot(node)
     name = treeutil.assembleVariable(var_root)[0]
     return bool(GlobalSymbolsCombinedPatt.search(name))
Exemple #34
0
    def findClassForFeature(self, featureId, variants, classMaps):

        # get the method name
        clazzId = self.id
        if featureId == u'':  # corner case: bare class reference outside "new ..."
            return clazzId, featureId
        # TODO: The next doesn't provide much, qx.Class.getInstance has no new dependencies
        # currently (aside from "new this", which I cannot relate back to 'construct'
        # ATM). Leave it in anyway, to not break bug#5660.
        #elif featureId == "getInstance": # corner case: singletons get this from qx.Class
        #    clazzId = "qx.Class"
        elif featureId == "getInstance" and self.type == "singleton":
            featureId = "construct"
        elif featureId in ('call',
                           'apply'):  # this might get overridden, oh well...
            clazzId = "Function"
        # TODO: getter/setter are also not lexically available!
        # handle .call() ?!
        if clazzId not in ClassesAll:  # can't further process non-qooxdoo classes
            # TODO: maybe this should better use something like isInterestingIdentifier()
            # to invoke the same machinery for filtering references like in other places
            return None, None

        # early return if class id is finalized
        if clazzId != self.id:
            classObj = ClassesAll[clazzId]
            featureNode = self.getFeatureNode(featureId, variants)
            if featureNode:
                return clazzId, featureNode
            else:
                return None, None

        # now try this class
        if self.id in classMaps:
            classMap = classMaps[self.id]
        else:
            classMap = classMaps[self.id] = self.getClassMap(variants)
        featureNode = self.getFeatureNode(featureId, variants, classMap)
        if featureNode:
            return self.id, featureNode

        if featureId == 'construct':  # constructor requested, but not supplied in class map
            # supply the default constructor
            featureNode = treeutil.compileString(
                "function(){this.base(arguments);}", self.path)
            # the next is a hack to provide minimal scope info
            featureNode.set("treegenerator_tag", 1)
            featureNode = scopes.create_scopes(featureNode)
            return self.id, featureNode

        # inspect inheritance/mixins
        parents = []
        extendVal = classMap.get('extend', None)
        if extendVal:
            extendVal = treeutil.variableOrArrayNodeToArray(extendVal)
            parents.extend(extendVal)
            # this.base calls
            if featureId == "base":
                classId = parents[0]  # first entry must be super-class
                if classId in ClassesAll:
                    return ClassesAll[classId].findClassForFeature(
                        'construct', variants, classMaps)
                else:
                    return None, None
        includeVal = classMap.get('include', None)
        if includeVal:
            # 'include' value according to Class spec.
            if includeVal.type in NODE_VARIABLE_TYPES + ('array', ):
                includeVal = treeutil.variableOrArrayNodeToArray(includeVal)

            # assume qx.core.Environment.filter() call
            else:
                filterMap = variantoptimizer.getFilterMap(includeVal, self.id)
                includeSymbols = []
                for key, node in filterMap.items():
                    # only consider true or undefined
                    #if key not in variants or (key in variants and bool(variants[key]):
                    # map value has to be value/variable
                    variable = node.children[0]
                    assert variable.isVar()
                    symbol, isComplete = treeutil.assembleVariable(variable)
                    assert isComplete
                    includeSymbols.append(symbol)
                includeVal = includeSymbols

            parents.extend(includeVal)

        # go through all ancestors
        for parClass in parents:
            if parClass not in ClassesAll:
                continue
            parClassObj = ClassesAll[parClass]
            rclass, keyval = parClassObj.findClassForFeature(
                featureId, variants, classMaps)
            if rclass:
                return rclass, keyval
        return None, None