def getTree(fileDb, fileId, options):
if not fileDb[fileId].has_key("tree"):
if options.verbose:
print " - Generating tree for %s..." % fileId
useCache = False
loadCache = False
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
if options.cacheDirectory != None:
cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl")
useCache = True
if not filetool.checkCache(filePath, cachePath):
loadCache = True
if loadCache:
tree = filetool.readCache(cachePath)
else:
tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options))
if useCache:
if options.verbose:
print " - Caching tree for %s..." % fileId
filetool.storeCache(cachePath, tree)
fileDb[fileId]["tree"] = tree
return fileDb[fileId]["tree"]
def migrateFile(
filePath, compiledPatches, compiledInfos,
hasPatchModule=False, options=None, encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = extractFileContentId(fileContent);
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
result = [u'']
result = pretty.prettyNode(tree, options, result)
patchedContent = u''.join(result)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." % filePath)
filetool.save(filePath, patchedContent, encoding)
def compileString(jsString, uniqueId=""):
"""
Compile a string containing a JavaScript fragment into a syntax tree.
"""
return treegenerator.createSyntaxTree(
tokenizer.parseStream(jsString, uniqueId)).getFirstChild(
).getFirstChild() # strip (file (statements ...) nodes
def migrateFile(self, filePath, compiledPatches, compiledInfos,
hasPatchModule=False, options=None, encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = loader.extractFileContentId(fileContent);
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
patchedContent = compiler.compile(tree, options)
# apply RE patches
patchedContent = self.regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = self.regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." % filePath)
filetool.save(filePath, patchedContent, encoding)
def _getSourceTree(self, cacheId, tradeSpaceForSpeed):
cache = self.context['cache']
console = self.context['console']
# Lookup for unoptimized tree
tree, _ = cache.read(cacheId, self.path, memory=tradeSpaceForSpeed)
# Tree still undefined?, create it!
if tree == None:
console.debug("Parsing file: %s..." % self.id)
console.indent()
fileContent = filetool.read(self.path, self.encoding)
tokens = tokenizer.parseStream(fileContent, self.id)
console.outdent()
console.debug("Generating tree: %s..." % self.id)
console.indent()
tree = treegenerator.createSyntaxTree(tokens) # allow exceptions to propagate
# store unoptimized tree
#print "Caching %s" % cacheId
cache.write(cacheId, tree, memory=tradeSpaceForSpeed, writeToFile=True)
console.outdent()
return tree
def tokenize(file):
try:
data = "".join(file.readlines())
restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data))
print '<?xml version="1.0" encoding="UTF-8"?>'
print restree.toXml()
except treegenerator.SyntaxException, e:
print e
def tokenize(file):
try:
data = "".join(file.readlines())
restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data))
print '<?xml version="1.0" encoding="UTF-8"?>'
print restree.toXml()
except treegenerator.SyntaxException, e:
print e
def __init__(self, filename, logger=None):
self.filename = filename
content = filetool.read(filename)
self.tree = treegenerator.createSyntaxTree(tokenizer.parseStream(content))
self.script = Script(self.tree, self.filename)
if not logger:
self.logger = ConsoleLogger()
else:
self.logger = logger
def __init__(self, filename, logger=None):
self.filename = filename
content = filetool.read(filename)
self.tree = treegenerator.createSyntaxTree(
tokenizer.parseStream(content))
self.script = Script(self.tree, self.filename)
if not logger:
self.logger = ConsoleLogger()
else:
self.logger = logger
def validate(file):
logger = util.TextMateLogger()
logger.printHeader("qooxdoo JavaScript Syntax check", "qooxdoo JavaScript Syntax check")
has_errors = False
try:
data = "".join(file.readlines())
restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data))
except treegenerator.SyntaxException, e:
has_errors = True
errorRe = re.compile("(.*file:, line:(\d+), column:(\d+))")
match = errorRe.match(str(e)).groups()
print "<a href='txmt://open?line=%s&column=%s'>%s</a>" % (match[1], match[2], match[0])
def validate(file):
logger = util.TextMateLogger()
logger.printHeader("qooxdoo JavaScript Syntax check",
"qooxdoo JavaScript Syntax check")
has_errors = False
try:
data = "".join(file.readlines())
restree = treegenerator.createSyntaxTree(tokenizer.parseStream(data))
except treegenerator.SyntaxException, e:
has_errors = True
errorRe = re.compile("(.*file:, line:(\d+), column:(\d+))")
match = errorRe.match(str(e)).groups()
print "<a href='txmt://open?line=%s&column=%s'>%s</a>" % (
match[1], match[2], match[0])
def getTree(self, fileId, variants=None):
fileEntry = self._classes[fileId]
filePath = fileEntry["path"]
if variants:
cacheId = "tree-%s-%s" % (filePath, util.toString(variants))
else:
cacheId = "tree-%s" % filePath
tradeSpaceForSpeed = False # Caution: setting this to True seems to make builds slower, at least on some platforms!?
tree = self._cache.read(cacheId, filePath, memory=tradeSpaceForSpeed)
if tree != None:
return tree
# Lookup for unoptimized tree
if variants != None:
tree = self._cache.read("tree-%s" % fileId, filePath, memory=tradeSpaceForSpeed)
# Tree still undefined?, create it!
if tree == None:
self._console.debug("Parsing file: %s..." % fileId)
self._console.indent()
fileContent = filetool.read(fileEntry["path"], fileEntry["encoding"])
tokens = tokenizer.parseStream(fileContent, fileId)
self._console.outdent()
self._console.debug("Generating tree: %s..." % fileId)
self._console.indent()
tree = treegenerator.createSyntaxTree(tokens) # allow exceptions to propagate
# store unoptimized tree
self._cache.write("tree-%s" % fileId, tree, memory=tradeSpaceForSpeed, writeToFile=True)
self._console.outdent()
# Call variant optimizer
if variants != None:
self._console.debug("Selecting variants: %s..." % fileId)
self._console.indent()
variantoptimizer.search(tree, variants, fileId)
self._console.outdent()
# store optimized tree
self._cache.write(cacheId, tree, memory=tradeSpaceForSpeed, writeToFile=True)
return tree
def getTree(self, fileId, variants=None):
fileEntry = self._classes[fileId]
filePath = fileEntry["path"]
if variants:
cacheId = "tree-%s-%s" % (fileId, idlist.toString(variants))
else:
cacheId = "tree-%s" % fileId
tree = self._cache.read(cacheId, filePath)
if tree != None:
return tree
# Lookup for unoptimized tree
if variants != None:
tree = self._cache.read("tree-%s" % fileId, filePath)
# Tree still undefined?, create it!
if tree == None:
self._console.debug("Parsing file: %s..." % fileId)
self._console.indent()
fileEntry = self._classes[fileId]
fileContent = filetool.read(fileEntry["path"], fileEntry["encoding"])
tokens = tokenizer.parseStream(fileContent, fileId)
self._console.outdent()
self._console.debug("Generating tree: %s..." % fileId)
self._console.indent()
try:
tree = treegenerator.createSyntaxTree(tokens)
except treegenerator.SyntaxException, detail:
self._console.error("%s" % detail)
sys.exit(1)
self._console.outdent()
self._console.debug("Selecting variants: %s..." % fileId)
self._console.indent()
def main():
parser = optparse.OptionParser(option_class=ExtendAction)
usage_str = '''%prog [options] file.js,...'''
parser.set_usage(usage_str)
# General flags
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="verbose output mode (extra verbose)")
parser.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="quiet output")
# Optimization flags
parser.add_option("-n",
"--variables",
action="store_true",
dest="variables",
default=False,
help="optimize variables")
parser.add_option("-s",
"--strings",
action="store_true",
dest="strings",
default=False,
help="optimize strings")
parser.add_option("-p",
"--privates",
action="store_true",
dest="privates",
default=False,
help="optimize privates")
parser.add_option("-b",
"--basecalls",
action="store_true",
dest="basecalls",
default=False,
help="optimize basecalls")
parser.add_option("-i",
"--inline",
action="store_true",
dest="inline",
default=False,
help="optimize inline")
parser.add_option("--all",
action="store_true",
dest="all",
default=False,
help="optimize all")
# Variant support
parser.add_option("--variant",
action="extend",
dest="variants",
metavar="KEY:VALUE",
type="string",
default=[],
help="Selected variants")
# Action modifier
parser.add_option("--pretty",
action="store_true",
dest="pretty",
default=False,
help="print out pretty printed")
parser.add_option("--tree",
action="store_true",
dest="tree",
default=False,
help="print out tree")
parser.add_option("--lint",
action="store_true",
dest="lint",
default=False,
help="ecmalint the file")
# Cache support
parser.add_option("-c",
"--cache",
dest="cache",
metavar="CACHEPATH",
type="string",
default="",
help="path to cache directory")
parser.add_option("--privateskey",
dest="privateskey",
metavar="CACHEKEY",
type="string",
default="",
help="cache key for privates")
#
# Process arguments
#
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) == 0:
print ">>> Missing filename!"
return
if not options.quiet:
print ">>> Parsing file..."
fileName = args[0]
fileContent = filetool.read(fileName, "utf-8")
fileId = "xxx"
tokens = tokenizer.parseStream(fileContent, fileName)
if not options.quiet:
print ">>> Creating tree..."
tree = treegenerator.createSyntaxTree(tokens)
#
# Optimizing tree
#
if len(options.variants) > 0:
if not options.quiet:
print ">>> Selecting variants..."
varmap = {}
for entry in options.variants:
pos = entry.index(":")
varmap[entry[0:pos]] = entry[pos + 1:]
variantoptimizer.search(tree, varmap, fileId)
if options.all or options.basecalls:
if not options.quiet:
print ">>> Optimizing basecalls..."
basecalloptimizer.patch(tree)
if options.all or options.inline:
if not options.quiet:
print ">>> Optimizing inline..."
inlineoptimizer.patch(tree)
if options.all or options.strings:
if not options.quiet:
print ">>> Optimizing strings..."
_optimizeStrings(tree, fileId)
if options.all or options.variables:
if not options.quiet:
print ">>> Optimizing variables..."
variableoptimizer.search(tree)
if options.all or options.privates:
if not options.quiet:
print ">>> Optimizing privates..."
if options.cache:
cache = Cache(options.cache, Log())
privates, _ = cache.read(options.privateskey)
if privates != None:
privateoptimizer.load(privates)
privateoptimizer.patch(tree, fileId)
if options.cache:
cache.write(options.privateskey, privateoptimizer.get())
#
# Output the result
#
if options.lint:
if not options.quiet:
print ">>> Executing ecmalint..."
print "Needs implementation"
elif options.tree:
if not options.quiet:
print ">>> Printing out tree..."
print tree.toXml().encode('utf-8')
else:
if not options.quiet:
print ">>> Compiling..."
compiled = _compileTree(tree, options.pretty)
print compiled.encode('utf-8')
def main():
parser = optparse.OptionParser(option_class=ExtendAction)
usage_str = '''%prog [options] file.js,...'''
parser.set_usage(usage_str)
# General flags
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose output mode (extra verbose)")
# Optimization flags
parser.add_option("-n", "--variables", action="store_true", dest="variables", default=False, help="optimize variables")
parser.add_option("-s", "--strings", action="store_true", dest="strings", default=False, help="optimize strings")
parser.add_option("-p", "--privates", action="store_true", dest="privates", default=False, help="optimize privates")
parser.add_option("-b", "--basecalls", action="store_true", dest="basecalls", default=False, help="optimize basecalls")
parser.add_option("-i", "--inline", action="store_true", dest="inline", default=False, help="optimize inline")
parser.add_option("--all", action="store_true", dest="all", default=False, help="optimize all")
# Variant support
parser.add_option("--variant", action="extend", dest="variants", metavar="KEY:VALUE", type="string", default=[], help="Selected variants")
# Action modifier
parser.add_option("--pretty", action="store_true", dest="pretty", default=False, help="print out pretty printed")
parser.add_option("--tree", action="store_true", dest="tree", default=False, help="print out tree")
parser.add_option("--apiXml", action="store_true", dest="apiXml", default=False, help="print out api data as XML")
parser.add_option("--apiJson", action="store_true", dest="apiJson", default=False, help="print out api data as JSON")
parser.add_option("--lint", action="store_true", dest="lint", default=False, help="ecmalint the file")
#
# Process arguments
#
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) == 0:
print ">>> Missing filename!"
return
print ">>> Parsing file..."
fileName = args[0]
fileContent = filetool.read(fileName, "utf-8")
fileId = "xxx"
tokens = tokenizer.parseStream(fileContent, fileName)
print ">>> Creating tree..."
tree = treegenerator.createSyntaxTree(tokens)
#
# Optimizing tree
#
if len(options.variants) > 0:
print ">>> Selecting variants..."
variantoptimizer.search(tree, options.variants, fileId)
if options.all or options.basecalls:
print ">>> Optimizing basecalls..."
basecalloptimizer.patch(tree)
if options.all or options.inline:
print ">>> Optimizing inline..."
inlineoptimizer.patch(tree)
if options.all or options.strings:
print ">>> Optimizing strings..."
_optimizeStrings(tree, fileId)
if options.all or options.variables:
print ">>> Optimizing variables..."
variableoptimizer.search(tree)
if options.all or options.privates:
print ">>> Optimizing privates..."
privateoptimizer.patch(tree, fileId)
#
# Output the result
#
if options.apiXml or options.apiJson:
(data, hasError) = api.createDoc(tree)
if hasError:
print "Error in API docs!"
elif options.apiXml:
print ">>> API data as XML..."
print data.toXml()
else:
print ">>> API data as JSON..."
print data.toJson()
elif options.lint:
print ">>> Executing ecmalint..."
print "Needs implementation"
elif options.tree:
print ">>> Printing out tree..."
print tree.toXml()
else:
print ">>> Compiling..."
compiled = _compileTree(tree, options.pretty)
print compiled
def main():
parser = optparse.OptionParser(option_class=ExtendAction)
usage_str = '''%prog [options] file.js,...'''
parser.set_usage(usage_str)
# General flags
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose output mode (extra verbose)")
parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="quiet output")
# Optimization flags
parser.add_option("-n", "--variables", action="store_true", dest="variables", default=False, help="optimize variables")
parser.add_option("-s", "--strings", action="store_true", dest="strings", default=False, help="optimize strings")
parser.add_option("-p", "--privates", action="store_true", dest="privates", default=False, help="optimize privates")
parser.add_option("-b", "--basecalls", action="store_true", dest="basecalls", default=False, help="optimize basecalls")
parser.add_option("-i", "--inline", action="store_true", dest="inline", default=False, help="optimize inline")
parser.add_option("-r", "--variants", action="store_true", dest="variantsopt", default=False, help="optimize variants")
parser.add_option("-m", "--comments", action="store_true", dest="comments", default=False, help="optimize comments")
parser.add_option("--all", action="store_true", dest="all", default=False, help="optimize all")
# Variant support
parser.add_option("--variant", action="extend", dest="variants", metavar="KEY:VALUE", type="string", default=[], help="Selected variants")
# Action modifier
parser.add_option("--pretty", action="store_true", dest="pretty", default=False, help="print out pretty printed")
parser.add_option("--tree", action="store_true", dest="tree", default=False, help="print out tree")
parser.add_option("--lint", action="store_true", dest="lint", default=False, help="ecmalint the file")
# Cache support
parser.add_option("-c", "--cache", dest="cache", metavar="CACHEPATH", type="string", default="", help="path to cache directory")
parser.add_option("--privateskey", dest="privateskey", metavar="CACHEKEY", type="string", default="", help="cache key for privates")
#
# Process arguments
#
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) == 0:
print ">>> Missing filename!"
return
if not options.quiet:
print ">>> Parsing file..."
fileName = args[0]
fileContent = filetool.read(fileName, "utf-8")
fileId = "xxx"
tokens = tokenizer.parseStream(fileContent, fileName)
if not options.quiet:
print ">>> Creating tree..."
tree = treegenerator.createSyntaxTree(tokens)
#
# Optimizing tree
#
if len(options.variants) > 0:
if not options.quiet:
print ">>> Selecting variants..."
varmap = {}
for entry in options.variants:
pos = entry.index(":")
varmap[entry[0:pos]] = entry[pos+1:]
variantoptimizer.search(tree, varmap, fileId)
if options.all or options.basecalls:
if not options.quiet:
print ">>> Optimizing basecalls..."
basecalloptimizer.patch(tree)
if options.all or options.inline:
if not options.quiet:
print ">>> Optimizing inline..."
inlineoptimizer.patch(tree)
if options.all or options.strings:
if not options.quiet:
print ">>> Optimizing strings..."
_optimizeStrings(tree, fileId)
if options.all or options.variables:
if not options.quiet:
print ">>> Optimizing variables..."
variableoptimizer.search(tree)
if options.all or options.privates:
if not options.quiet:
print ">>> Optimizing privates..."
privates = {}
if options.cache:
cache = Cache(options.cache,
interruptRegistry=interruptRegistry
)
privates, _ = cache.read(options.privateskey)
if privates == None:
privates = {}
privateoptimizer.patch(tree, fileId, privates)
if options.cache:
cache.write(options.privateskey, privates)
#
# Output the result
#
if options.lint:
if not options.quiet:
print ">>> Executing ecmalint..."
print "Needs implementation"
elif options.tree:
if not options.quiet:
print ">>> Printing out tree..."
print tree.toXml().encode('utf-8')
else:
if not options.quiet:
print ">>> Compiling..."
compiled = _compileTree(tree, options.pretty)
print compiled.encode('utf-8')
def compileString(jsString, uniqueId=""):
"""
Compile a string containing a JavaScript fragment into a syntax tree.
"""
return treegenerator.createSyntaxTree(tokenizer.parseStream(jsString, uniqueId)).getFirstChild().getFirstChild() # strip (file (statements ...) nodes