def getTree(fileDb, fileId, options):
if not fileDb[fileId].has_key("tree"):
if options.verbose:
print " - Generating tree for %s..." % fileId
useCache = False
loadCache = False
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
if options.cacheDirectory != None:
cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl")
useCache = True
if not filetool.checkCache(filePath, cachePath):
loadCache = True
if loadCache:
tree = filetool.readCache(cachePath)
else:
tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options))
if useCache:
if options.verbose:
print " - Caching tree for %s..." % fileId
filetool.storeCache(cachePath, tree)
fileDb[fileId]["tree"] = tree
return fileDb[fileId]["tree"]
def getTree(fileDb, fileId, options):
if not fileDb[fileId].has_key("tree"):
if options.verbose:
print " - Generating tree for %s..." % fileId
useCache = False
loadCache = False
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
if options.cacheDirectory != None:
cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl")
useCache = True
if not filetool.checkCache(filePath, cachePath):
loadCache = True
if loadCache:
tree = filetool.readCache(cachePath)
else:
tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options))
if useCache:
if options.verbose:
print " - Caching tree for %s..." % fileId
filetool.storeCache(cachePath, tree)
fileDb[fileId]["tree"] = tree
return fileDb[fileId]["tree"]
def migrateFile(
filePath, compiledPatches, compiledInfos,
hasPatchModule=False, options=None, encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = loader.extractFileContentId(fileContent);
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
patchedContent = compiler.compile(tree, options)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." % filePath)
filetool.save(filePath, patchedContent, encoding)
def main():
global options
parser = optparse.OptionParser()
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
print "Processing %s" % (fileName)
restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding))
track(restree)
def main():
parser = optparse.OptionParser()
parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="")
parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False)
parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
else:
print "Compiling %s => stdout" % fileName
restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, "", options.encoding))
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, "$")
compiledString = compile(restree, not options.compress)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def migrateFile(filePath,
compiledPatches,
compiledInfos,
hasPatchModule=False,
options=None,
encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = loader.extractFileContentId(fileContent)
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(
tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
patchedContent = compiler.compile(tree, options)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." %
filePath)
filetool.save(filePath, patchedContent, encoding)
def main():
global options
parser = optparse.OptionParser()
parser.add_option("--encoding",
dest="encoding",
default="utf-8",
metavar="ENCODING",
help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
print "Processing %s" % (fileName)
restree = treegenerator.createSyntaxTree(
tokenizer.parseFile(fileName, fileName, options.encoding))
track(restree)
def main():
global options
parser = optparse.OptionParser(option_class=optparseext.ExtendAction)
parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="")
parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False)
parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.")
parser.add_option("--optimize-privates", action="store_true", dest="optimizePrivates", default=False, help="Optimize privates. Protected them and reducing size.")
parser.add_option("--obfuscate-accessors", action="store_true", dest="obfuscateAccessors", default=False, help="Enable accessor obfuscation")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
parser.add_option("--use-variant", action="extend", dest="useVariant", type="string", metavar="NAMESPACE.KEY:VALUE", default=[], help="Optimize for the given variant.")
# Options for pretty printing
addCommandLineOptions(parser)
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding))
if len(options.useVariant) > 0:
variantMap = {}
for variant in options.useVariant:
keyValue = variant.split(":")
if len(keyValue) != 2:
print " * Error: Variants must be specified as key value pair separated by ':'!"
sys.exit(1)
variantMap[keyValue[0]] = keyValue[1]
variantoptimizer.search(restree, variantMap, fileName)
if options.obfuscateAccessors:
accessorobfuscator.process(restree)
if options.optimizePrivates:
privateoptimizer.patch("A", restree, {})
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, 0, "$")
if options.compress:
options.prettyPrint = False # make sure it's set
else:
options.prettyPrint = True
compiledString = compile(restree, options)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def main():
global options
parser = optparse.OptionParser(option_class=optparseext.ExtendAction)
parser.add_option("-w",
"--write",
action="store_true",
dest="write",
default=False,
help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e",
"--extension",
dest="extension",
metavar="EXTENSION",
help="The EXTENSION to use",
default="")
parser.add_option("-c",
"--compress",
action="store_true",
dest="compress",
help="Enable compression",
default=False)
parser.add_option("--optimize-variables",
action="store_true",
dest="optimizeVariables",
default=False,
help="Optimize variables. Reducing size.")
parser.add_option(
"--optimize-privates",
action="store_true",
dest="optimizePrivates",
default=False,
help="Optimize privates. Protected them and reducing size.")
parser.add_option("--obfuscate-accessors",
action="store_true",
dest="obfuscateAccessors",
default=False,
help="Enable accessor obfuscation")
parser.add_option("--encoding",
dest="encoding",
default="utf-8",
metavar="ENCODING",
help="Defines the encoding expected for input files.")
parser.add_option("--use-variant",
action="extend",
dest="useVariant",
type="string",
metavar="NAMESPACE.KEY:VALUE",
default=[],
help="Optimize for the given variant.")
# Options for pretty printing
addCommandLineOptions(parser)
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName,
options.extension)
restree = treegenerator.createSyntaxTree(
tokenizer.parseFile(fileName, fileName, options.encoding))
if len(options.useVariant) > 0:
variantMap = {}
for variant in options.useVariant:
keyValue = variant.split(":")
if len(keyValue) != 2:
print " * Error: Variants must be specified as key value pair separated by ':'!"
sys.exit(1)
variantMap[keyValue[0]] = keyValue[1]
variantoptimizer.search(restree, variantMap, fileName)
if options.obfuscateAccessors:
accessorobfuscator.process(restree)
if options.optimizePrivates:
privateoptimizer.patch("A", restree, {})
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, 0, "$")
if options.compress:
options.prettyPrint = False # make sure it's set
else:
options.prettyPrint = True
compiledString = compile(restree, options)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def main():
parser = optparse.OptionParser()
parser.add_option("-w",
"--write",
action="store_true",
dest="write",
default=False,
help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e",
"--extension",
dest="extension",
metavar="EXTENSION",
help="The EXTENSION to use",
default="")
parser.add_option("-c",
"--compress",
action="store_true",
dest="compress",
help="Enable compression",
default=False)
parser.add_option("--optimize-variables",
action="store_true",
dest="optimizeVariables",
default=False,
help="Optimize variables. Reducing size.")
parser.add_option("--encoding",
dest="encoding",
default="utf-8",
metavar="ENCODING",
help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName,
options.extension)
else:
print "Compiling %s => stdout" % fileName
restree = treegenerator.createSyntaxTree(
tokenizer.parseFile(fileName, "", options.encoding))
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, "$")
compiledString = compile(restree, not options.compress)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def compileString(jsString, uniqueId=""):
"""
Compile a string containing a JavaScript fragment into a syntax tree.
"""
return treegenerator.createSyntaxTree(tokenizer.parseStream(jsString, uniqueId)).getFirstChild()
def handle(fileList, fileDb, options):
confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget)
infoPath = os.path.join(confPath, "info")
patchPath = os.path.join(confPath, "patches")
importedModule = False
infoList = []
patchList = []
htmlList = getHtmlList(options)
print " * Number of script input files: %s" % len(fileList)
print " * Number of HTML input files: %s" % len(htmlList)
print " * Update to version: %s" % options.migrationTarget
print " * Searching for patch module..."
for root, dirs, files in os.walk(confPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
if os.path.splitext(fileName)[1] != config.PYEXT:
continue
if fileName == "patch.py":
print " - Importing..."
if not root in sys.path:
sys.path.insert(0, root)
import patch
importedModule = True
emptyLine = re.compile("^\s*$")
print " * Searching for info expression data..."
for root, dirs, files in os.walk(infoPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
infoList.append({"path":filePath, "content":fileContent.split("\n")})
if options.verbose:
print " - %s" % filePath
print " - Number of info files: %s" % len(infoList)
print " - Compiling expressions..."
compiledInfos = []
for infoFile in infoList:
print " - %s" % os.path.basename(infoFile["path"])
for line in infoFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledInfos.append(compiled)
print " - Number of infos: %s" % len(compiledInfos)
print " * Searching for patch expression data..."
for root, dirs, files in os.walk(patchPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
patchList.append({"path":filePath, "content":fileContent.split("\n")})
if options.verbose:
print " - %s" % filePath
print " - Number of patch files: %s" % len(patchList)
print " - Compiling expressions..."
compiledPatches = []
for patchFile in patchList:
print " - %s" % os.path.basename(patchFile["path"])
for line in patchFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledPatches.append(compiled)
print " - Number of patches: %s" % len(compiledPatches)
print
print " FILE PROCESSING:"
print "----------------------------------------------------------------------------"
if len(fileList) > 0:
print " * Processing script files:"
for fileId in fileList:
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
print " - %s" % fileId
# Read in original content
fileContent = filetool.read(filePath, fileEncoding)
patchedContent = fileContent
# Apply patches
if importedModule:
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
patchedContent = compiler.compile(tree, True)
patchedContent = regtool(patchedContent, compiledPatches, True, options)
patchedContent = regtool(patchedContent, compiledInfos, False, options)
# Write file
if patchedContent != fileContent:
print " - Store modifications..."
filetool.save(filePath, patchedContent, fileEncoding)
print " * Done"
if len(htmlList) > 0:
print " * Processing HTML files:"
for filePath in htmlList:
print " - %s" % filePath
# Read in original content
fileContent = filetool.read(filePath)
patchedContent = fileContent
patchedContent = regtool(patchedContent, compiledPatches, True, options)
patchedContent = regtool(patchedContent, compiledInfos, False, options)
# Write file
if patchedContent != fileContent:
print " - Store modifications..."
filetool.save(filePath, patchedContent)
print " * Done"