def lint_check(node, file_name, opts): node = scopes.create_scopes(node) # update scopes if not hasattr(node, 'hint'): node = jshints.create_hints_tree(node) lint = LintChecker(node, file_name, opts) lint.visit(node) return lint.issues
def process(tree, id_): # refresh scopes to get a check-set tree = scopes.create_scopes(tree) check_set = tree.scope.all_var_names() check_set.update(lang.RESERVED.keys()) # assuming a <file> or <block> node statementsNode = tree.getChild("statements") # create a map for strings to var names stringMap = search(statementsNode) if len(stringMap) == 0: return tree # apply the vars #stringList = sort(stringMap) replace(statementsNode, stringMap, check_set) # create a 'var' decl for the string vars stringReplacement = replacement(stringMap) repl_tree = treeutil.compileString(stringReplacement, id_ + "||stringopt") # ensure a wrapping closure closure, closure_block = treeutil.ensureClosureWrapper( statementsNode.children) statementsNode.removeAllChildren() statementsNode.addChild(closure) # add 'var' decl to closure closure_block.addChild(repl_tree, 0) # 'var ...'; decl to front of statement list return tree
def process(tree, id_): # refresh scopes to get a check-set tree = scopes.create_scopes(tree) check_set = tree.scope.all_var_names() check_set.update(lang.RESERVED.keys()) # assuming a <file> or <block> node statementsNode = tree.getChild("statements") # create a map for strings to var names stringMap = search(statementsNode, verbose=False) if len(stringMap) == 0: return tree # apply the vars #stringList = sort(stringMap) replace(statementsNode, stringMap, check_set) # create a 'var' decl for the string vars stringReplacement = replacement(stringMap) repl_tree = treeutil.compileString(stringReplacement, id_ + "||stringopt") # ensure a wrapping closure closure, closure_block = treeutil.ensureClosureWrapper(statementsNode.children) statementsNode.removeAllChildren() statementsNode.addChild(closure) # add 'var' decl to closure closure_block.addChild(repl_tree, 0) # 'var ...'; decl to front of statement list return tree
def search(node): # we have to scope-analyze again, as other optimizations might have # changed the original tree (variants, strings, ... optimizations) node = scopes.create_scopes(node) # protect certain scopes from optimization protect_visitor = ProtectionVisitor() protect_visitor.visit(node.scope) # optimize scopes var_optimizer = OptimizerVisitor(node) var_optimizer.visit(node.scope)
def globals_check(node, file_name, opts): node = scopes.create_scopes(node) # update scopes lint = GlobalsChecker(node, file_name, opts) #print "Globals", file_name lint.visit(node.scope) #import cProfile #cProfile.runctx("lint.visit(node)",globals(),locals(), # "/home/thron7/tmp/prof/deps.prof"+str(cnt)) #global cnt #cnt += 1 return lint.issues
def process(node, globals_map): # make sure we have current scope tree node = scopes.create_scopes(node) # replace globals in tree globals_optimizer = GlobalsOptimizer(globals_map) globals_optimizer.visit(node.scope) # get new globals from tree new_symbols = list(treeutil.findQxDefineR(node)) new_names = operands_of_calls(new_symbols) # add to globals_map for new_name in new_names: globals_map.add(new_name) # this will create replacements if they're new # add defining code lines to closure propagate_new_globals(node, new_names, globals_map)
def process(node, globals_map_=None): #print "globals optimization:", str(node) #globals_map = globals_map_ or seed_globals_map() #import pydb; pydb.debugger() globals_map = globals_map_ or gmap # make sure we have a current scope tree node = scopes.create_scopes(node) # replace globals in tree globals_optimizer = GlobalsOptimizer(globals_map) globals_optimizer.visit(node.scope) # get new globals from tree new_names = globals_optimizer.new_qx_classes # add to globals_map for new_name in new_names: globals_map.add(new_name) # this will create replacements if they're new # add defining code lines to closure node = propagate_new_globals(node, new_names, globals_map) return node
def do_lint(file_, popup): if popup: logger = util.PopupLogger() else: logger = util.TextMateLogger() logger.printHeader("qooxdoo JavaScript lint", "qooxdoo JavaScript lint") try: opts = lint.defaultOptions() opts.allowed_globals = ['qx', 'qxWeb', 'q'] tree_ = treegenerator.createFileTree_from_string( codecs.open(file_, "r", "utf-8").read()) tree_ = scopes.create_scopes(tree_) if not getattr(context, 'console', None): context.console = Log() if not getattr(context, 'jobconf', None): context.jobconf = ExtMap() context.jobconf.set("lint-check/warn-unknown-jsdoc-keys", True) lint.lint_check(tree_, "", opts) except treegenerator.SyntaxException, e: logger.log(file_, 0, 0, str(e))
def lint_check(node, file_name, opts): node = scopes.create_scopes(node) # update scopes lint = LintChecker(node, file_name, opts) lint.visit(node) return lint.issues
def scope_globals(node): node = scopes.create_scopes(node) # update scopes globals_getter = GlobalsExtractor(node) globals_getter.visit(node.scope) return globals_getter.global_nodes
def main(argv=None): # init Context module Context.console = Log() Context.jobconf = {} if argv is None: argv = sys.argv parser = OptionParser(description="Checks ECMAScript/JavaScript files for common errors.") parser.add_option( "--action", "-a", dest="actions", metavar="ACTION", choices=["ALL", "undefined_variables", "unused_variables", "multidefined_variables", "maps", "blocks", "fields"], action="append", default=[], help="""Performs the given checks on the input files. This parameter may be supplied multiple times. Valid arguments are: "ALL" (default): Perform all checks "undefined_variables": Look for identifier, which are referenced in the global scope. This action can find misspelled identifier and missing 'var' statements. You can use the '-g' flag to add valid global identifiers. "unused_variables": Look for identifier, which are defined but never used. "multidefined_variables": Look for identifier, which are defined multiple times. "blocks" : Look for single statments in bodies of if's and loops that are not enclosed by braces. "fields" : Look for class attributes, checking definedness, privates and protected fields. "maps": Look for duplicate keys in map declarations. """ ) parser.add_option( "-g", dest="globals", help="Add an allowed global identifier GLOBAL", metavar="GLOBAL", action="append", default=[] ) (options, args) = parser.parse_args(argv) if len(args) == 1: parser.print_help() sys.exit(1) if options.globals: globals_ = options.globals else: globals_ = [] checkAll = "ALL" in options.actions or len(options.actions) == 0 # construct opts argument for lint_check keys_map ={ # map cli 'action' keys to lint.options "undefined_variables" : ["ignore_undefined_globals"], "unused_variables" : ["ignore_unused_variables", "ignore_unused_parameter"], "multidefined_variables" : ["ignore_multiple_vardecls"], "maps" : ["ignore_multiple_mapkeys"], "blocks" : ["ignore_no_loop_block"], "fields" : ["ignore_reference_fields", "ignore_undeclared_privates"], } opts = lint.defaultOptions() opts.allowed_globals = options.globals for opt in (o for o in vars(opts) if o.startswith("ignore_")): if checkAll: setattr(opts, opt, False) else: for argopt in keys_map: if argopt in options.actions and opt in keys_map[argopt]: setattr(opts, opt, False) break else: setattr(opts, opt, True) for filename in args[1:]: tree_ = treegenerator.createFileTree_from_string(codecs.open(filename, "r", "utf-8").read()) tree_ = scopes.create_scopes(tree_) lint.lint_check(tree_, filename, opts) return ## TODO: rc
def main(argv=None): # init Context module Context.console = Log() Context.jobconf = {} if argv is None: argv = sys.argv parser = OptionParser( description="Checks ECMAScript/JavaScript files for common errors.") parser.add_option( "--action", "-a", dest="actions", metavar="ACTION", choices=[ "ALL", "undefined_variables", "unused_variables", "multidefined_variables", "maps", "blocks", "fields" ], action="append", default=[], help= """Performs the given checks on the input files. This parameter may be supplied multiple times. Valid arguments are: "ALL" (default): Perform all checks "undefined_variables": Look for identifier, which are referenced in the global scope. This action can find misspelled identifier and missing 'var' statements. You can use the '-g' flag to add valid global identifiers. "unused_variables": Look for identifier, which are defined but never used. "multidefined_variables": Look for identifier, which are defined multiple times. "blocks" : Look for single statments in bodies of if's and loops that are not enclosed by braces. "fields" : Look for class attributes, checking definedness, privates and protected fields. "maps": Look for duplicate keys in map declarations. """) parser.add_option("-g", dest="globals", help="Add an allowed global identifier GLOBAL", metavar="GLOBAL", action="append", default=[]) (options, args) = parser.parse_args(argv) if len(args) == 1: parser.print_help() sys.exit(1) if options.globals: globals_ = options.globals else: globals_ = [] checkAll = "ALL" in options.actions or len(options.actions) == 0 # construct opts argument for lint_check keys_map ={ # map cli 'action' keys to lint.options "undefined_variables" : ["ignore_undefined_globals"], "unused_variables" : ["ignore_unused_variables", "ignore_unused_parameter"], "multidefined_variables" : ["ignore_multiple_vardecls"], "maps" : ["ignore_multiple_mapkeys"], "blocks" : ["ignore_no_loop_block"], "fields" : ["ignore_reference_fields", "ignore_undeclared_privates"], } opts = lint.defaultOptions() opts.allowed_globals = options.globals for opt in (o for o in vars(opts) if o.startswith("ignore_")): if checkAll: setattr(opts, opt, False) else: for argopt in keys_map: if argopt in options.actions and opt in keys_map[argopt]: setattr(opts, opt, False) break else: setattr(opts, opt, True) for filename in args[1:]: tree_ = treegenerator.createFileTree_from_string( codecs.open(filename, "r", "utf-8").read()) tree_ = scopes.create_scopes(tree_) lint.lint_check(tree_, filename, opts) return ## TODO: rc
class MClassCode(object): # -------------------------------------------------------------------------- # Tree Interface # -------------------------------------------------------------------------- ## # Interface method. Delivers class syntax tree: # - handles cache # - can be called with alternative parser (treegenerator) # def tree(self, treegen=treegenerator, force=False): cache = self.context['cache'] console = self.context['console'] tradeSpaceForSpeed = False # Caution: setting this to True seems to make builds slower, at least on some platforms!? cacheId = "tree%s-%s-%s" % (treegen.tag, self.path, util.toString({})) self.treeId = cacheId # Lookup for unoptimized tree tree, _ = cache.read(cacheId, self.path, memory=tradeSpaceForSpeed) # Tree still undefined?, create it! if tree == None or force: console.debug("Parsing file: %s..." % self.id) console.indent() # tokenize fileContent = filetool.read(self.path, self.encoding) fileId = self.path if self.path else self.id try: tokens = tokenizer.parseStream(fileContent, self.id) except SyntaxException, e: # add file info e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:] raise e # parse console.outdent() console.debug("Generating tree: %s..." % self.id) console.indent() try: tree = treegen.createFileTree(tokens, fileId) except SyntaxException, e: # add file info e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:] raise e # annotate with scopes if True: console.outdent() console.debug("Calculating scopes: %s..." % self.id) console.indent() tree = scopes.create_scopes(tree) #if self.id == "gui.Application": # import pydb; pydb.debugger() #tree.scope.prrnt() # lint check if False: console.outdent() console.debug("Checking JavaScript source code: %s..." % self.id) console.indent() # construct parse-level check options opts = lint.defaultOptions() lint.lint_check(tree, self.id, opts) # store unoptimized tree cache.write(cacheId, tree, memory=tradeSpaceForSpeed) console.outdent()
except SyntaxException, e: # add file info e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:] raise # Check class id against file id try: self.checkClassId(tree) except ValueError, e: # add file info e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:] raise # Annotate with scopes if True: tree = scopes.create_scopes(tree) # checks for suitable treegenerator_tag #tree.scope.prrnt() #print self.id, " (globals):", [c for s in tree.scope.scope_iterator() for c in s.globals()] # Annotate scopes with load time information if True: if hasattr(tree,'scope') and tree.scope: load_time.load_time_check(tree.scope) # Annotate with jsdoc hints if True: tree = jshints.create_hints_tree(tree) # Store unoptimized tree cache.write(cacheId, tree, memory=tradeSpaceForSpeed)
def findClassForFeature(self, featureId, variants, classMaps): # get the method name clazzId = self.id if featureId == u'': # corner case: bare class reference outside "new ..." return clazzId, featureId # TODO: The next doesn't provide much, qx.Class.getInstance has no new dependencies # currently (aside from "new this", which I cannot relate back to 'construct' # ATM). Leave it in anyway, to not break bug#5660. #elif featureId == "getInstance": # corner case: singletons get this from qx.Class # clazzId = "qx.Class" elif featureId == "getInstance" and self.type == "singleton": featureId = "construct" elif featureId in ('call', 'apply'): # this might get overridden, oh well... clazzId = "Function" # TODO: getter/setter are also not lexically available! # handle .call() ?! if clazzId not in ClassesAll: # can't further process non-qooxdoo classes # TODO: maybe this should better use something like isInterestingIdentifier() # to invoke the same machinery for filtering references like in other places return None, None # early return if class id is finalized if clazzId != self.id: classObj = ClassesAll[clazzId] featureNode = self.getFeatureNode(featureId, variants) if featureNode: return clazzId, featureNode else: return None, None # now try this class if self.id in classMaps: classMap = classMaps[self.id] else: classMap = classMaps[self.id] = self.getClassMap (variants) featureNode = self.getFeatureNode(featureId, variants, classMap) if featureNode: return self.id, featureNode if featureId == 'construct': # constructor requested, but not supplied in class map # supply the default constructor featureNode = treeutil.compileString("function(){this.base(arguments);}", self.path) # the next is a hack to provide minimal scope info featureNode.set("treegenerator_tag", 1) featureNode = scopes.create_scopes(featureNode) return self.id, featureNode # inspect inheritance/mixins parents = [] extendVal = classMap.get('extend', None) if extendVal: extendVal = treeutil.variableOrArrayNodeToArray(extendVal) parents.extend(extendVal) # this.base calls if featureId == "base": classId = parents[0] # first entry must be super-class if classId in ClassesAll: return ClassesAll[classId].findClassForFeature('construct', variants, classMaps) else: return None, None includeVal = classMap.get('include', None) if includeVal: # 'include' value according to Class spec. if includeVal.type in NODE_VARIABLE_TYPES + ('array',): includeVal = treeutil.variableOrArrayNodeToArray(includeVal) # assume qx.core.Environment.filter() call else: filterMap = variantoptimizer.getFilterMap(includeVal, self.id) includeSymbols = [] for key, node in filterMap.items(): # only consider true or undefined #if key not in variants or (key in variants and bool(variants[key]): # map value has to be value/variable variable = node.children[0] assert variable.isVar() symbol, isComplete = treeutil.assembleVariable(variable) assert isComplete includeSymbols.append(symbol) includeVal = includeSymbols parents.extend(includeVal) # go through all ancestors for parClass in parents: if parClass not in ClassesAll: continue parClassObj = ClassesAll[parClass] rclass, keyval = parClassObj.findClassForFeature(featureId, variants, classMaps) if rclass: return rclass, keyval return None, None
def run_compile(fileName, fileContent, options, args): fileId = fileName tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName) if not options.quiet: print(">>> Creating tree...") tree = treegenerator.createFileTree(tokens) tree = scopes.create_scopes(tree) # optimizing tree if len(options.variants) > 0: if not options.quiet: print(">>> Selecting variants...") varmap = {} for entry in options.variants: pos = entry.index(":") varmap[entry[0:pos]] = entry[pos+1:] variantoptimizer.search(tree, varmap, fileId) if options.all or options.basecalls: if not options.quiet: print(">>> Optimizing basecalls...") basecalloptimizer.patch(tree) #if options.all or options.inline: # if not options.quiet: # print(">>> Optimizing inline...") # inlineoptimizer.patch(tree) if options.all or options.strings: if not options.quiet: print(">>> Optimizing strings...") _optimizeStrings(tree, fileId) if options.all or options.variables: if not options.quiet: print(">>> Optimizing variables...") variableoptimizer.search(tree) if options.all or options.globals: if not options.quiet: print(">>> Optimizing globals...") tree = globalsoptimizer.process(tree) if options.all or options.privates: if not options.quiet: print(">>> Optimizing privates...") privates = {} if options.cache: cache = Cache(options.cache, interruptRegistry=interruptRegistry ) privates, _ = cache.read(options.privateskey) if privates == None: privates = {} privateoptimizer.patch(tree, fileId, privates) if options.cache: cache.write(options.privateskey, privates) if not options.quiet: print(">>> Compiling...") result = [u''] result = Packer().serializeNode(tree, None, result, True) result = u''.join(result) print(result.encode('utf-8')) return
def run_compile(fileName, fileContent, options, args): fileId = fileName tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName) if not options.quiet: print(">>> Creating tree...") tree = treegenerator.createFileTree(tokens) tree = scopes.create_scopes(tree) # optimizing tree if len(options.variants) > 0: if not options.quiet: print(">>> Selecting variants...") varmap = {} for entry in options.variants: pos = entry.index(":") varmap[entry[0:pos]] = entry[pos + 1:] variantoptimizer.search(tree, varmap, fileId) if options.all or options.basecalls: if not options.quiet: print(">>> Optimizing basecalls...") basecalloptimizer.patch(tree) #if options.all or options.inline: # if not options.quiet: # print(">>> Optimizing inline...") # inlineoptimizer.patch(tree) if options.all or options.strings: if not options.quiet: print(">>> Optimizing strings...") _optimizeStrings(tree, fileId) if options.all or options.variables: if not options.quiet: print(">>> Optimizing variables...") variableoptimizer.search(tree) if options.all or options.globals: if not options.quiet: print(">>> Optimizing globals...") tree = globalsoptimizer.process(tree) if options.all or options.privates: if not options.quiet: print(">>> Optimizing privates...") privates = {} if options.cache: cache = Cache(options.cache, interruptRegistry=interruptRegistry) privates, _ = cache.read(options.privateskey) if privates == None: privates = {} privateoptimizer.patch(tree, fileId, privates) if options.cache: cache.write(options.privateskey, privates) if not options.quiet: print(">>> Compiling...") result = [u''] result = Packer().serializeNode(tree, None, result, True) result = u''.join(result) print(result.encode('utf-8')) return