def getTree(fileDb, fileId, options): if not fileDb[fileId].has_key("tree"): if options.verbose: print " - Generating tree for %s..." % fileId useCache = False loadCache = False fileEntry = fileDb[fileId] filePath = fileEntry["path"] if options.cacheDirectory != None: cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl") useCache = True if not filetool.checkCache(filePath, cachePath): loadCache = True if loadCache: tree = filetool.readCache(cachePath) else: tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options)) if useCache: if options.verbose: print " - Caching tree for %s..." % fileId filetool.storeCache(cachePath, tree) fileDb[fileId]["tree"] = tree return fileDb[fileId]["tree"]
def migrateFile( filePath, compiledPatches, compiledInfos, hasPatchModule=False, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = extractFileContentId(fileContent); # Apply patches patchedContent = fileContent if hasPatchModule and fileId is not None: import patch tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): options.prettyPrint = True # make sure it's set result = [u''] result = pretty.prettyNode(tree, options, result) patchedContent = u''.join(result) # apply RE patches patchedContent = regtool(patchedContent, compiledPatches, True, filePath) patchedContent = regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def compileString(jsString, uniqueId=""): """ Compile a string containing a JavaScript fragment into a syntax tree. """ return treegenerator.createSyntaxTree( tokenizer.parseStream(jsString, uniqueId)).getFirstChild( ).getFirstChild() # strip (file (statements ...) nodes
def migrateFile(self, filePath, compiledPatches, compiledInfos, hasPatchModule=False, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = loader.extractFileContentId(fileContent); # Apply patches patchedContent = fileContent if hasPatchModule and fileId is not None: import patch tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): options.prettyPrint = True # make sure it's set patchedContent = compiler.compile(tree, options) # apply RE patches patchedContent = self.regtool(patchedContent, compiledPatches, True, filePath) patchedContent = self.regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def _getSourceTree(self, cacheId, tradeSpaceForSpeed): cache = self.context['cache'] console = self.context['console'] # Lookup for unoptimized tree tree, _ = cache.read(cacheId, self.path, memory=tradeSpaceForSpeed) # Tree still undefined?, create it! if tree == None: console.debug("Parsing file: %s..." % self.id) console.indent() fileContent = filetool.read(self.path, self.encoding) tokens = tokenizer.parseStream(fileContent, self.id) console.outdent() console.debug("Generating tree: %s..." % self.id) console.indent() tree = treegenerator.createSyntaxTree(tokens) # allow exceptions to propagate # store unoptimized tree #print "Caching %s" % cacheId cache.write(cacheId, tree, memory=tradeSpaceForSpeed, writeToFile=True) console.outdent() return tree
def tokenize(file): try: data = "".join(file.readlines()) restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data)) print '<?xml version="1.0" encoding="UTF-8"?>' print restree.toXml() except treegenerator.SyntaxException, e: print e
def __init__(self, filename, logger=None): self.filename = filename content = filetool.read(filename) self.tree = treegenerator.createSyntaxTree(tokenizer.parseStream(content)) self.script = Script(self.tree, self.filename) if not logger: self.logger = ConsoleLogger() else: self.logger = logger
def __init__(self, filename, logger=None): self.filename = filename content = filetool.read(filename) self.tree = treegenerator.createSyntaxTree( tokenizer.parseStream(content)) self.script = Script(self.tree, self.filename) if not logger: self.logger = ConsoleLogger() else: self.logger = logger
def validate(file): logger = util.TextMateLogger() logger.printHeader("qooxdoo JavaScript Syntax check", "qooxdoo JavaScript Syntax check") has_errors = False try: data = "".join(file.readlines()) restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data)) except treegenerator.SyntaxException, e: has_errors = True errorRe = re.compile("(.*file:, line:(\d+), column:(\d+))") match = errorRe.match(str(e)).groups() print "<a href='txmt://open?line=%s&column=%s'>%s</a>" % (match[1], match[2], match[0])
def validate(file): logger = util.TextMateLogger() logger.printHeader("qooxdoo JavaScript Syntax check", "qooxdoo JavaScript Syntax check") has_errors = False try: data = "".join(file.readlines()) restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data)) except treegenerator.SyntaxException, e: has_errors = True errorRe = re.compile("(.*file:, line:(\d+), column:(\d+))") match = errorRe.match(str(e)).groups() print "<a href='txmt://open?line=%s&column=%s'>%s</a>" % ( match[1], match[2], match[0])
def getTree(self, fileId, variants=None): fileEntry = self._classes[fileId] filePath = fileEntry["path"] if variants: cacheId = "tree-%s-%s" % (filePath, util.toString(variants)) else: cacheId = "tree-%s" % filePath tradeSpaceForSpeed = False # Caution: setting this to True seems to make builds slower, at least on some platforms!? tree = self._cache.read(cacheId, filePath, memory=tradeSpaceForSpeed) if tree != None: return tree # Lookup for unoptimized tree if variants != None: tree = self._cache.read("tree-%s" % fileId, filePath, memory=tradeSpaceForSpeed) # Tree still undefined?, create it! if tree == None: self._console.debug("Parsing file: %s..." % fileId) self._console.indent() fileContent = filetool.read(fileEntry["path"], fileEntry["encoding"]) tokens = tokenizer.parseStream(fileContent, fileId) self._console.outdent() self._console.debug("Generating tree: %s..." % fileId) self._console.indent() tree = treegenerator.createSyntaxTree(tokens) # allow exceptions to propagate # store unoptimized tree self._cache.write("tree-%s" % fileId, tree, memory=tradeSpaceForSpeed, writeToFile=True) self._console.outdent() # Call variant optimizer if variants != None: self._console.debug("Selecting variants: %s..." % fileId) self._console.indent() variantoptimizer.search(tree, variants, fileId) self._console.outdent() # store optimized tree self._cache.write(cacheId, tree, memory=tradeSpaceForSpeed, writeToFile=True) return tree
def getTree(self, fileId, variants=None): fileEntry = self._classes[fileId] filePath = fileEntry["path"] if variants: cacheId = "tree-%s-%s" % (fileId, idlist.toString(variants)) else: cacheId = "tree-%s" % fileId tree = self._cache.read(cacheId, filePath) if tree != None: return tree # Lookup for unoptimized tree if variants != None: tree = self._cache.read("tree-%s" % fileId, filePath) # Tree still undefined?, create it! if tree == None: self._console.debug("Parsing file: %s..." % fileId) self._console.indent() fileEntry = self._classes[fileId] fileContent = filetool.read(fileEntry["path"], fileEntry["encoding"]) tokens = tokenizer.parseStream(fileContent, fileId) self._console.outdent() self._console.debug("Generating tree: %s..." % fileId) self._console.indent() try: tree = treegenerator.createSyntaxTree(tokens) except treegenerator.SyntaxException, detail: self._console.error("%s" % detail) sys.exit(1) self._console.outdent() self._console.debug("Selecting variants: %s..." % fileId) self._console.indent()
def main(): parser = optparse.OptionParser(option_class=ExtendAction) usage_str = '''%prog [options] file.js,...''' parser.set_usage(usage_str) # General flags parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose output mode (extra verbose)") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="quiet output") # Optimization flags parser.add_option("-n", "--variables", action="store_true", dest="variables", default=False, help="optimize variables") parser.add_option("-s", "--strings", action="store_true", dest="strings", default=False, help="optimize strings") parser.add_option("-p", "--privates", action="store_true", dest="privates", default=False, help="optimize privates") parser.add_option("-b", "--basecalls", action="store_true", dest="basecalls", default=False, help="optimize basecalls") parser.add_option("-i", "--inline", action="store_true", dest="inline", default=False, help="optimize inline") parser.add_option("--all", action="store_true", dest="all", default=False, help="optimize all") # Variant support parser.add_option("--variant", action="extend", dest="variants", metavar="KEY:VALUE", type="string", default=[], help="Selected variants") # Action modifier parser.add_option("--pretty", action="store_true", dest="pretty", default=False, help="print out pretty printed") parser.add_option("--tree", action="store_true", dest="tree", default=False, help="print out tree") parser.add_option("--lint", action="store_true", dest="lint", default=False, help="ecmalint the file") # Cache support parser.add_option("-c", "--cache", dest="cache", metavar="CACHEPATH", type="string", default="", help="path to cache directory") parser.add_option("--privateskey", dest="privateskey", metavar="CACHEKEY", type="string", default="", help="cache key for privates") # # Process arguments # (options, args) = parser.parse_args(sys.argv[1:]) if len(args) == 0: print ">>> Missing filename!" return if not options.quiet: print ">>> Parsing file..." fileName = args[0] fileContent = filetool.read(fileName, "utf-8") fileId = "xxx" tokens = tokenizer.parseStream(fileContent, fileName) if not options.quiet: print ">>> Creating tree..." tree = treegenerator.createSyntaxTree(tokens) # # Optimizing tree # if len(options.variants) > 0: if not options.quiet: print ">>> Selecting variants..." varmap = {} for entry in options.variants: pos = entry.index(":") varmap[entry[0:pos]] = entry[pos + 1:] variantoptimizer.search(tree, varmap, fileId) if options.all or options.basecalls: if not options.quiet: print ">>> Optimizing basecalls..." basecalloptimizer.patch(tree) if options.all or options.inline: if not options.quiet: print ">>> Optimizing inline..." inlineoptimizer.patch(tree) if options.all or options.strings: if not options.quiet: print ">>> Optimizing strings..." _optimizeStrings(tree, fileId) if options.all or options.variables: if not options.quiet: print ">>> Optimizing variables..." variableoptimizer.search(tree) if options.all or options.privates: if not options.quiet: print ">>> Optimizing privates..." if options.cache: cache = Cache(options.cache, Log()) privates, _ = cache.read(options.privateskey) if privates != None: privateoptimizer.load(privates) privateoptimizer.patch(tree, fileId) if options.cache: cache.write(options.privateskey, privateoptimizer.get()) # # Output the result # if options.lint: if not options.quiet: print ">>> Executing ecmalint..." print "Needs implementation" elif options.tree: if not options.quiet: print ">>> Printing out tree..." print tree.toXml().encode('utf-8') else: if not options.quiet: print ">>> Compiling..." compiled = _compileTree(tree, options.pretty) print compiled.encode('utf-8')
def main(): parser = optparse.OptionParser(option_class=ExtendAction) usage_str = '''%prog [options] file.js,...''' parser.set_usage(usage_str) # General flags parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose output mode (extra verbose)") # Optimization flags parser.add_option("-n", "--variables", action="store_true", dest="variables", default=False, help="optimize variables") parser.add_option("-s", "--strings", action="store_true", dest="strings", default=False, help="optimize strings") parser.add_option("-p", "--privates", action="store_true", dest="privates", default=False, help="optimize privates") parser.add_option("-b", "--basecalls", action="store_true", dest="basecalls", default=False, help="optimize basecalls") parser.add_option("-i", "--inline", action="store_true", dest="inline", default=False, help="optimize inline") parser.add_option("--all", action="store_true", dest="all", default=False, help="optimize all") # Variant support parser.add_option("--variant", action="extend", dest="variants", metavar="KEY:VALUE", type="string", default=[], help="Selected variants") # Action modifier parser.add_option("--pretty", action="store_true", dest="pretty", default=False, help="print out pretty printed") parser.add_option("--tree", action="store_true", dest="tree", default=False, help="print out tree") parser.add_option("--apiXml", action="store_true", dest="apiXml", default=False, help="print out api data as XML") parser.add_option("--apiJson", action="store_true", dest="apiJson", default=False, help="print out api data as JSON") parser.add_option("--lint", action="store_true", dest="lint", default=False, help="ecmalint the file") # # Process arguments # (options, args) = parser.parse_args(sys.argv[1:]) if len(args) == 0: print ">>> Missing filename!" return print ">>> Parsing file..." fileName = args[0] fileContent = filetool.read(fileName, "utf-8") fileId = "xxx" tokens = tokenizer.parseStream(fileContent, fileName) print ">>> Creating tree..." tree = treegenerator.createSyntaxTree(tokens) # # Optimizing tree # if len(options.variants) > 0: print ">>> Selecting variants..." variantoptimizer.search(tree, options.variants, fileId) if options.all or options.basecalls: print ">>> Optimizing basecalls..." basecalloptimizer.patch(tree) if options.all or options.inline: print ">>> Optimizing inline..." inlineoptimizer.patch(tree) if options.all or options.strings: print ">>> Optimizing strings..." _optimizeStrings(tree, fileId) if options.all or options.variables: print ">>> Optimizing variables..." variableoptimizer.search(tree) if options.all or options.privates: print ">>> Optimizing privates..." privateoptimizer.patch(tree, fileId) # # Output the result # if options.apiXml or options.apiJson: (data, hasError) = api.createDoc(tree) if hasError: print "Error in API docs!" elif options.apiXml: print ">>> API data as XML..." print data.toXml() else: print ">>> API data as JSON..." print data.toJson() elif options.lint: print ">>> Executing ecmalint..." print "Needs implementation" elif options.tree: print ">>> Printing out tree..." print tree.toXml() else: print ">>> Compiling..." compiled = _compileTree(tree, options.pretty) print compiled
def main(): parser = optparse.OptionParser(option_class=ExtendAction) usage_str = '''%prog [options] file.js,...''' parser.set_usage(usage_str) # General flags parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose output mode (extra verbose)") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="quiet output") # Optimization flags parser.add_option("-n", "--variables", action="store_true", dest="variables", default=False, help="optimize variables") parser.add_option("-s", "--strings", action="store_true", dest="strings", default=False, help="optimize strings") parser.add_option("-p", "--privates", action="store_true", dest="privates", default=False, help="optimize privates") parser.add_option("-b", "--basecalls", action="store_true", dest="basecalls", default=False, help="optimize basecalls") parser.add_option("-i", "--inline", action="store_true", dest="inline", default=False, help="optimize inline") parser.add_option("-r", "--variants", action="store_true", dest="variantsopt", default=False, help="optimize variants") parser.add_option("-m", "--comments", action="store_true", dest="comments", default=False, help="optimize comments") parser.add_option("--all", action="store_true", dest="all", default=False, help="optimize all") # Variant support parser.add_option("--variant", action="extend", dest="variants", metavar="KEY:VALUE", type="string", default=[], help="Selected variants") # Action modifier parser.add_option("--pretty", action="store_true", dest="pretty", default=False, help="print out pretty printed") parser.add_option("--tree", action="store_true", dest="tree", default=False, help="print out tree") parser.add_option("--lint", action="store_true", dest="lint", default=False, help="ecmalint the file") # Cache support parser.add_option("-c", "--cache", dest="cache", metavar="CACHEPATH", type="string", default="", help="path to cache directory") parser.add_option("--privateskey", dest="privateskey", metavar="CACHEKEY", type="string", default="", help="cache key for privates") # # Process arguments # (options, args) = parser.parse_args(sys.argv[1:]) if len(args) == 0: print ">>> Missing filename!" return if not options.quiet: print ">>> Parsing file..." fileName = args[0] fileContent = filetool.read(fileName, "utf-8") fileId = "xxx" tokens = tokenizer.parseStream(fileContent, fileName) if not options.quiet: print ">>> Creating tree..." tree = treegenerator.createSyntaxTree(tokens) # # Optimizing tree # if len(options.variants) > 0: if not options.quiet: print ">>> Selecting variants..." varmap = {} for entry in options.variants: pos = entry.index(":") varmap[entry[0:pos]] = entry[pos+1:] variantoptimizer.search(tree, varmap, fileId) if options.all or options.basecalls: if not options.quiet: print ">>> Optimizing basecalls..." basecalloptimizer.patch(tree) if options.all or options.inline: if not options.quiet: print ">>> Optimizing inline..." inlineoptimizer.patch(tree) if options.all or options.strings: if not options.quiet: print ">>> Optimizing strings..." _optimizeStrings(tree, fileId) if options.all or options.variables: if not options.quiet: print ">>> Optimizing variables..." variableoptimizer.search(tree) if options.all or options.privates: if not options.quiet: print ">>> Optimizing privates..." privates = {} if options.cache: cache = Cache(options.cache, interruptRegistry=interruptRegistry ) privates, _ = cache.read(options.privateskey) if privates == None: privates = {} privateoptimizer.patch(tree, fileId, privates) if options.cache: cache.write(options.privateskey, privates) # # Output the result # if options.lint: if not options.quiet: print ">>> Executing ecmalint..." print "Needs implementation" elif options.tree: if not options.quiet: print ">>> Printing out tree..." print tree.toXml().encode('utf-8') else: if not options.quiet: print ">>> Compiling..." compiled = _compileTree(tree, options.pretty) print compiled.encode('utf-8')
def compileString(jsString, uniqueId=""): """ Compile a string containing a JavaScript fragment into a syntax tree. """ return treegenerator.createSyntaxTree(tokenizer.parseStream(jsString, uniqueId)).getFirstChild().getFirstChild() # strip (file (statements ...) nodes