def tree(self, treegen=treegenerator, force=False): cache = self.context['cache'] console = self.context['console'] tradeSpaceForSpeed = False # Caution: setting this to True seems to make builds slower, at least on some platforms!? cacheId = "tree%s-%s-%s" % (treegen.tag, self.path, util.toString({})) self.treeId = cacheId # Lookup for unoptimized tree tree, _ = cache.read(cacheId, self.path, memory=tradeSpaceForSpeed) # Tree still undefined?, create it! if tree == None or force: console.debug("Parsing file: %s..." % self.id) console.indent() # Tokenize fileContent = filetool.read(self.path, self.encoding) fileId = self.path if self.path else self.id try: tokens = tokenizer.Tokenizer().parseStream(fileContent, self.id) except SyntaxException, e: # add file info e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:] raise e # Parse try: tree = treegen.createFileTree(tokens, fileId) except SyntaxException, e: # add file info e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:] raise
def compileString(jsString, uniqueId=""): """ Compile a string containing a JavaScript fragment into a syntax tree. """ return treegenerator.createFileTree(tokenizer.Tokenizer().parseStream( jsString, uniqueId)).getFirstChild().getFirstChild( ) # strip (file (statements ...) nodes
def run_tree(fileName, fileContent, options, args): tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName) if not options.quiet: print ">>> Creating tree..." tree = treegenerator.createFileTree(tokens) if not options.quiet: print ">>> Printing out tree..." print tree.toXml().encode('utf-8') return
def run_pretty(fileName, fileContent, options, args): #elif options.pretty: # for testing formatter_2 # options = formatter.FormatterOptions() # options = formatter.defaultOptions(options) # print formatter.formatStream(tokens, options) tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName) tree = treegenerator_3.createFileTree(tokens) # use special tree optns = formatter.defaultOptions() optns.prettypCommentsBlockAdd = False result = [u''] result = formatter.formatNode(tree, optns, result) result = u''.join(result) print(result) return
def migrateFile(filePath, compiledPatches, compiledInfos, patchFile, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = extractFileContentId(fileContent) # Apply patches patchedContent = fileContent if patchFile and fileId is not None: #import patch patch = {} execfile(patchFile, patch) tree = treegenerator.createFileTree( tokenizer.Tokenizer().parseStream(fileContent)) # If there were any changes, compile the result if patch['patch'](fileId, tree): options.prettyPrint = True # make sure it's set result = [u''] #result = pretty.prettyNode(tree, options, result) result = formatter_.formatNode(tree, options, result) patchedContent = u''.join(result) # apply RE patches patchedContent = regtool(patchedContent, compiledPatches, True, filePath) patchedContent = regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def run_compile(fileName, fileContent, options, args): fileId = fileName tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName) if not options.quiet: print(">>> Creating tree...") tree = treegenerator.createFileTree(tokens) tree = scopes.create_scopes(tree) # optimizing tree if len(options.variants) > 0: if not options.quiet: print(">>> Selecting variants...") varmap = {} for entry in options.variants: pos = entry.index(":") varmap[entry[0:pos]] = entry[pos + 1:] variantoptimizer.search(tree, varmap, fileId) if options.all or options.basecalls: if not options.quiet: print(">>> Optimizing basecalls...") basecalloptimizer.patch(tree) #if options.all or options.inline: # if not options.quiet: # print(">>> Optimizing inline...") # inlineoptimizer.patch(tree) if options.all or options.strings: if not options.quiet: print(">>> Optimizing strings...") _optimizeStrings(tree, fileId) if options.all or options.variables: if not options.quiet: print(">>> Optimizing variables...") variableoptimizer.search(tree) if options.all or options.globals: if not options.quiet: print(">>> Optimizing globals...") tree = globalsoptimizer.process(tree) if options.all or options.privates: if not options.quiet: print(">>> Optimizing privates...") privates = {} if options.cache: cache = Cache(options.cache, interruptRegistry=interruptRegistry) privates, _ = cache.read(options.privateskey) if privates == None: privates = {} privateoptimizer.patch(tree, fileId, privates) if options.cache: cache.write(options.privateskey, privates) if not options.quiet: print(">>> Compiling...") result = [u''] result = Packer().serializeNode(tree, None, result, True) result = u''.join(result) print(result.encode('utf-8')) return
def main(): parser = optparse.OptionParser(option_class=ExtendAction) usage_str = '''%prog [options] file.js,...''' parser.set_usage(usage_str) # General flags parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose output mode (extra verbose)") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="quiet output") # Optimization flags parser.add_option("-n", "--variables", action="store_true", dest="variables", default=False, help="optimize variables") parser.add_option("-s", "--strings", action="store_true", dest="strings", default=False, help="optimize strings") parser.add_option("-p", "--privates", action="store_true", dest="privates", default=False, help="optimize privates") parser.add_option("-b", "--basecalls", action="store_true", dest="basecalls", default=False, help="optimize basecalls") parser.add_option("-i", "--inline", action="store_true", dest="inline", default=False, help="optimize inline") parser.add_option("-r", "--variants", action="store_true", dest="variantsopt", default=False, help="optimize variants") parser.add_option("-m", "--comments", action="store_true", dest="comments", default=False, help="optimize comments") parser.add_option("--all", action="store_true", dest="all", default=False, help="optimize all") # Variant support parser.add_option("--variant", action="extend", dest="variants", metavar="KEY:VALUE", type="string", default=[], help="Selected variants") # Action modifier parser.add_option("--pretty", action="store_true", dest="pretty", default=False, help="print out pretty printed") parser.add_option("--tree", action="store_true", dest="tree", default=False, help="print out tree") parser.add_option("--lint", action="store_true", dest="lint", default=False, help="ecmalint the file") # Cache support parser.add_option("-c", "--cache", dest="cache", metavar="CACHEPATH", type="string", default="", help="path to cache directory") parser.add_option("--privateskey", dest="privateskey", metavar="CACHEKEY", type="string", default="", help="cache key for privates") # # Process arguments # (options, args) = parser.parse_args(sys.argv[1:]) if len(args) == 0: print ">>> Missing filename!" return if not options.quiet: print ">>> Parsing file..." fileName = args[0] fileContent = filetool.read(fileName, "utf-8") fileId = "xxx" tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName) if not options.quiet: print ">>> Creating tree..." tree = treegenerator.createFileTree(tokens) # treegenerator_3 #print repr(tree) #return # - treegenerator_3 # # Optimizing tree # if len(options.variants) > 0: if not options.quiet: print ">>> Selecting variants..." varmap = {} for entry in options.variants: pos = entry.index(":") varmap[entry[0:pos]] = entry[pos + 1:] variantoptimizer.search(tree, varmap, fileId) if options.all or options.basecalls: if not options.quiet: print ">>> Optimizing basecalls..." basecalloptimizer.patch(tree) if options.all or options.inline: if not options.quiet: print ">>> Optimizing inline..." inlineoptimizer.patch(tree) if options.all or options.strings: if not options.quiet: print ">>> Optimizing strings..." _optimizeStrings(tree, fileId) if options.all or options.variables: if not options.quiet: print ">>> Optimizing variables..." variableoptimizer.search(tree) if options.all or options.privates: if not options.quiet: print ">>> Optimizing privates..." privates = {} if options.cache: cache = Cache(options.cache, interruptRegistry=interruptRegistry) privates, _ = cache.read(options.privateskey) if privates == None: privates = {} privateoptimizer.patch(tree, fileId, privates) if options.cache: cache.write(options.privateskey, privates) # # Output the result # if options.lint: if not options.quiet: print ">>> Executing ecmalint..." print "Needs implementation" elif options.tree: if not options.quiet: print ">>> Printing out tree..." print tree.toXml().encode('utf-8') #elif options.pretty: # for testing formatter_2 # options = formatter.FormatterOptions() # options = formatter.defaultOptions(options) # print formatter.formatStream(tokens, options) else: if not options.quiet: print ">>> Compiling..." if options.pretty: tree = treegenerator_3.createFileTree(tokens) # use special tree compiled = _compileTree(tree, options.pretty) print compiled.encode('utf-8')
elif x == s: res = readStatement(tokenStream) print res.toXml() elif x == b: res = readBlock(tokenStream) print res.toXml() else: raise RuntimeError("Wrong test parameter: %s" % x) if __name__ == "__main__": import sys, os from ecmascript.frontend import tokenizer if len(sys.argv)>1: arg1 = sys.argv[1] p = TreeGenerator() if os.path.isfile(arg1): text = filetool.read(sys.argv[1]) else: text = arg1 tokenArr = tokenizer.Tokenizer().parseStream(text) print p.parse(tokenArr).toXml() else: execfile (os.path.normpath(os.path.join(__file__, "../../../../test/compiler/treegenerator.py"))) for t in tests: try: test(*t) except SyntaxException: print "PARSE FAILED:", repr(t)
def formatString(string_, options): ts = tokenizer.Tokenizer().parseStream(string_) return formatStream(ts, options)