def tree(self, treegen=treegenerator, force=False):
cache = self.context['cache']
console = self.context['console']
tradeSpaceForSpeed = False # Caution: setting this to True seems to make builds slower, at least on some platforms!?
cacheId = "tree%s-%s-%s" % (treegen.tag, self.path, util.toString({}))
self.treeId = cacheId
# Lookup for unoptimized tree
tree, _ = cache.read(cacheId, self.path, memory=tradeSpaceForSpeed)
# Tree still undefined?, create it!
if tree == None or force:
console.debug("Parsing file: %s..." % self.id)
console.indent()
# Tokenize
fileContent = filetool.read(self.path, self.encoding)
fileId = self.path if self.path else self.id
try:
tokens = tokenizer.Tokenizer().parseStream(fileContent, self.id)
except SyntaxException, e:
# add file info
e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:]
raise e
# Parse
try:
tree = treegen.createFileTree(tokens, fileId)
except SyntaxException, e:
# add file info
e.args = (e.args[0] + "\nFile: %s" % fileId,) + e.args[1:]
raise
def compileString(jsString, uniqueId=""):
"""
Compile a string containing a JavaScript fragment into a syntax tree.
"""
return treegenerator.createFileTree(tokenizer.Tokenizer().parseStream(
jsString, uniqueId)).getFirstChild().getFirstChild(
) # strip (file (statements ...) nodes
def run_tree(fileName, fileContent, options, args):
tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName)
if not options.quiet: print ">>> Creating tree..."
tree = treegenerator.createFileTree(tokens)
if not options.quiet: print ">>> Printing out tree..."
print tree.toXml().encode('utf-8')
return
def run_pretty(fileName, fileContent, options, args):
#elif options.pretty: # for testing formatter_2
# options = formatter.FormatterOptions()
# options = formatter.defaultOptions(options)
# print formatter.formatStream(tokens, options)
tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName)
tree = treegenerator_3.createFileTree(tokens) # use special tree
optns = formatter.defaultOptions()
optns.prettypCommentsBlockAdd = False
result = [u'']
result = formatter.formatNode(tree, optns, result)
result = u''.join(result)
print(result)
return
def migrateFile(filePath,
compiledPatches,
compiledInfos,
patchFile,
options=None,
encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = extractFileContentId(fileContent)
# Apply patches
patchedContent = fileContent
if patchFile and fileId is not None:
#import patch
patch = {}
execfile(patchFile, patch)
tree = treegenerator.createFileTree(
tokenizer.Tokenizer().parseStream(fileContent))
# If there were any changes, compile the result
if patch['patch'](fileId, tree):
options.prettyPrint = True # make sure it's set
result = [u'']
#result = pretty.prettyNode(tree, options, result)
result = formatter_.formatNode(tree, options, result)
patchedContent = u''.join(result)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." %
filePath)
filetool.save(filePath, patchedContent, encoding)
def run_compile(fileName, fileContent, options, args):
fileId = fileName
tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName)
if not options.quiet:
print(">>> Creating tree...")
tree = treegenerator.createFileTree(tokens)
tree = scopes.create_scopes(tree)
# optimizing tree
if len(options.variants) > 0:
if not options.quiet:
print(">>> Selecting variants...")
varmap = {}
for entry in options.variants:
pos = entry.index(":")
varmap[entry[0:pos]] = entry[pos + 1:]
variantoptimizer.search(tree, varmap, fileId)
if options.all or options.basecalls:
if not options.quiet:
print(">>> Optimizing basecalls...")
basecalloptimizer.patch(tree)
#if options.all or options.inline:
# if not options.quiet:
# print(">>> Optimizing inline...")
# inlineoptimizer.patch(tree)
if options.all or options.strings:
if not options.quiet:
print(">>> Optimizing strings...")
_optimizeStrings(tree, fileId)
if options.all or options.variables:
if not options.quiet:
print(">>> Optimizing variables...")
variableoptimizer.search(tree)
if options.all or options.globals:
if not options.quiet:
print(">>> Optimizing globals...")
tree = globalsoptimizer.process(tree)
if options.all or options.privates:
if not options.quiet:
print(">>> Optimizing privates...")
privates = {}
if options.cache:
cache = Cache(options.cache, interruptRegistry=interruptRegistry)
privates, _ = cache.read(options.privateskey)
if privates == None:
privates = {}
privateoptimizer.patch(tree, fileId, privates)
if options.cache:
cache.write(options.privateskey, privates)
if not options.quiet:
print(">>> Compiling...")
result = [u'']
result = Packer().serializeNode(tree, None, result, True)
result = u''.join(result)
print(result.encode('utf-8'))
return
def main():
parser = optparse.OptionParser(option_class=ExtendAction)
usage_str = '''%prog [options] file.js,...'''
parser.set_usage(usage_str)
# General flags
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="verbose output mode (extra verbose)")
parser.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="quiet output")
# Optimization flags
parser.add_option("-n",
"--variables",
action="store_true",
dest="variables",
default=False,
help="optimize variables")
parser.add_option("-s",
"--strings",
action="store_true",
dest="strings",
default=False,
help="optimize strings")
parser.add_option("-p",
"--privates",
action="store_true",
dest="privates",
default=False,
help="optimize privates")
parser.add_option("-b",
"--basecalls",
action="store_true",
dest="basecalls",
default=False,
help="optimize basecalls")
parser.add_option("-i",
"--inline",
action="store_true",
dest="inline",
default=False,
help="optimize inline")
parser.add_option("-r",
"--variants",
action="store_true",
dest="variantsopt",
default=False,
help="optimize variants")
parser.add_option("-m",
"--comments",
action="store_true",
dest="comments",
default=False,
help="optimize comments")
parser.add_option("--all",
action="store_true",
dest="all",
default=False,
help="optimize all")
# Variant support
parser.add_option("--variant",
action="extend",
dest="variants",
metavar="KEY:VALUE",
type="string",
default=[],
help="Selected variants")
# Action modifier
parser.add_option("--pretty",
action="store_true",
dest="pretty",
default=False,
help="print out pretty printed")
parser.add_option("--tree",
action="store_true",
dest="tree",
default=False,
help="print out tree")
parser.add_option("--lint",
action="store_true",
dest="lint",
default=False,
help="ecmalint the file")
# Cache support
parser.add_option("-c",
"--cache",
dest="cache",
metavar="CACHEPATH",
type="string",
default="",
help="path to cache directory")
parser.add_option("--privateskey",
dest="privateskey",
metavar="CACHEKEY",
type="string",
default="",
help="cache key for privates")
#
# Process arguments
#
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) == 0:
print ">>> Missing filename!"
return
if not options.quiet:
print ">>> Parsing file..."
fileName = args[0]
fileContent = filetool.read(fileName, "utf-8")
fileId = "xxx"
tokens = tokenizer.Tokenizer().parseStream(fileContent, fileName)
if not options.quiet:
print ">>> Creating tree..."
tree = treegenerator.createFileTree(tokens)
# treegenerator_3
#print repr(tree)
#return
# - treegenerator_3
#
# Optimizing tree
#
if len(options.variants) > 0:
if not options.quiet:
print ">>> Selecting variants..."
varmap = {}
for entry in options.variants:
pos = entry.index(":")
varmap[entry[0:pos]] = entry[pos + 1:]
variantoptimizer.search(tree, varmap, fileId)
if options.all or options.basecalls:
if not options.quiet:
print ">>> Optimizing basecalls..."
basecalloptimizer.patch(tree)
if options.all or options.inline:
if not options.quiet:
print ">>> Optimizing inline..."
inlineoptimizer.patch(tree)
if options.all or options.strings:
if not options.quiet:
print ">>> Optimizing strings..."
_optimizeStrings(tree, fileId)
if options.all or options.variables:
if not options.quiet:
print ">>> Optimizing variables..."
variableoptimizer.search(tree)
if options.all or options.privates:
if not options.quiet:
print ">>> Optimizing privates..."
privates = {}
if options.cache:
cache = Cache(options.cache, interruptRegistry=interruptRegistry)
privates, _ = cache.read(options.privateskey)
if privates == None:
privates = {}
privateoptimizer.patch(tree, fileId, privates)
if options.cache:
cache.write(options.privateskey, privates)
#
# Output the result
#
if options.lint:
if not options.quiet:
print ">>> Executing ecmalint..."
print "Needs implementation"
elif options.tree:
if not options.quiet:
print ">>> Printing out tree..."
print tree.toXml().encode('utf-8')
#elif options.pretty: # for testing formatter_2
# options = formatter.FormatterOptions()
# options = formatter.defaultOptions(options)
# print formatter.formatStream(tokens, options)
else:
if not options.quiet:
print ">>> Compiling..."
if options.pretty:
tree = treegenerator_3.createFileTree(tokens) # use special tree
compiled = _compileTree(tree, options.pretty)
print compiled.encode('utf-8')
elif x == s:
res = readStatement(tokenStream)
print res.toXml()
elif x == b:
res = readBlock(tokenStream)
print res.toXml()
else:
raise RuntimeError("Wrong test parameter: %s" % x)
if __name__ == "__main__":
import sys, os
from ecmascript.frontend import tokenizer
if len(sys.argv)>1:
arg1 = sys.argv[1]
p = TreeGenerator()
if os.path.isfile(arg1):
text = filetool.read(sys.argv[1])
else:
text = arg1
tokenArr = tokenizer.Tokenizer().parseStream(text)
print p.parse(tokenArr).toXml()
else:
execfile (os.path.normpath(os.path.join(__file__, "../../../../test/compiler/treegenerator.py")))
for t in tests:
try:
test(*t)
except SyntaxException:
print "PARSE FAILED:", repr(t)
def formatString(string_, options):
ts = tokenizer.Tokenizer().parseStream(string_)
return formatStream(ts, options)
