def getTree(fileDb, fileId, options): if not fileDb[fileId].has_key("tree"): if options.verbose: print " - Generating tree for %s..." % fileId useCache = False loadCache = False fileEntry = fileDb[fileId] filePath = fileEntry["path"] if options.cacheDirectory != None: cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl") useCache = True if not filetool.checkCache(filePath, cachePath): loadCache = True if loadCache: tree = filetool.readCache(cachePath) else: tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options)) if useCache: if options.verbose: print " - Caching tree for %s..." % fileId filetool.storeCache(cachePath, tree) fileDb[fileId]["tree"] = tree return fileDb[fileId]["tree"]
def migrateFile( filePath, compiledPatches, compiledInfos, hasPatchModule=False, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = loader.extractFileContentId(fileContent); # Apply patches patchedContent = fileContent if hasPatchModule and fileId is not None: import patch tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): options.prettyPrint = True # make sure it's set patchedContent = compiler.compile(tree, options) # apply RE patches patchedContent = regtool(patchedContent, compiledPatches, True, filePath) patchedContent = regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def main(): global options parser = optparse.OptionParser() parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.") (options, args) = parser.parse_args() if len(args) == 0: print "Needs one or more arguments (files) to compile!" sys.exit(1) for fileName in args: print "Processing %s" % (fileName) restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding)) track(restree)
def main(): parser = optparse.OptionParser() parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.") parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="") parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False) parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.") parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.") (options, args) = parser.parse_args() if len(args) == 0: print "Needs one or more arguments (files) to compile!" sys.exit(1) for fileName in args: if options.write: print "Compiling %s => %s%s" % (fileName, fileName, options.extension) else: print "Compiling %s => stdout" % fileName restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, "", options.encoding)) if options.optimizeVariables: variableoptimizer.search(restree, [], 0, "$") compiledString = compile(restree, not options.compress) if options.write: if compiledString != "" and not compiledString.endswith("\n"): compiledString += "\n" filetool.save(fileName + options.extension, compiledString) else: try: print compiledString except UnicodeEncodeError: print " * Could not encode result to ascii. Use '-w' instead." sys.exit(1)
def migrateFile(filePath, compiledPatches, compiledInfos, hasPatchModule=False, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = loader.extractFileContentId(fileContent) # Apply patches patchedContent = fileContent if hasPatchModule and fileId is not None: import patch tree = treegenerator.createSyntaxTree( tokenizer.parseStream(fileContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): options.prettyPrint = True # make sure it's set patchedContent = compiler.compile(tree, options) # apply RE patches patchedContent = regtool(patchedContent, compiledPatches, True, filePath) patchedContent = regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def main(): global options parser = optparse.OptionParser() parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.") (options, args) = parser.parse_args() if len(args) == 0: print "Needs one or more arguments (files) to compile!" sys.exit(1) for fileName in args: print "Processing %s" % (fileName) restree = treegenerator.createSyntaxTree( tokenizer.parseFile(fileName, fileName, options.encoding)) track(restree)
def main(): global options parser = optparse.OptionParser(option_class=optparseext.ExtendAction) parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.") parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="") parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False) parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.") parser.add_option("--optimize-privates", action="store_true", dest="optimizePrivates", default=False, help="Optimize privates. Protected them and reducing size.") parser.add_option("--obfuscate-accessors", action="store_true", dest="obfuscateAccessors", default=False, help="Enable accessor obfuscation") parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.") parser.add_option("--use-variant", action="extend", dest="useVariant", type="string", metavar="NAMESPACE.KEY:VALUE", default=[], help="Optimize for the given variant.") # Options for pretty printing addCommandLineOptions(parser) (options, args) = parser.parse_args() if len(args) == 0: print "Needs one or more arguments (files) to compile!" sys.exit(1) for fileName in args: if options.write: print "Compiling %s => %s%s" % (fileName, fileName, options.extension) restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding)) if len(options.useVariant) > 0: variantMap = {} for variant in options.useVariant: keyValue = variant.split(":") if len(keyValue) != 2: print " * Error: Variants must be specified as key value pair separated by ':'!" sys.exit(1) variantMap[keyValue[0]] = keyValue[1] variantoptimizer.search(restree, variantMap, fileName) if options.obfuscateAccessors: accessorobfuscator.process(restree) if options.optimizePrivates: privateoptimizer.patch("A", restree, {}) if options.optimizeVariables: variableoptimizer.search(restree, [], 0, 0, "$") if options.compress: options.prettyPrint = False # make sure it's set else: options.prettyPrint = True compiledString = compile(restree, options) if options.write: if compiledString != "" and not compiledString.endswith("\n"): compiledString += "\n" filetool.save(fileName + options.extension, compiledString) else: try: print compiledString except UnicodeEncodeError: print " * Could not encode result to ascii. Use '-w' instead." sys.exit(1)
def main(): global options parser = optparse.OptionParser(option_class=optparseext.ExtendAction) parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.") parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="") parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False) parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.") parser.add_option( "--optimize-privates", action="store_true", dest="optimizePrivates", default=False, help="Optimize privates. Protected them and reducing size.") parser.add_option("--obfuscate-accessors", action="store_true", dest="obfuscateAccessors", default=False, help="Enable accessor obfuscation") parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.") parser.add_option("--use-variant", action="extend", dest="useVariant", type="string", metavar="NAMESPACE.KEY:VALUE", default=[], help="Optimize for the given variant.") # Options for pretty printing addCommandLineOptions(parser) (options, args) = parser.parse_args() if len(args) == 0: print "Needs one or more arguments (files) to compile!" sys.exit(1) for fileName in args: if options.write: print "Compiling %s => %s%s" % (fileName, fileName, options.extension) restree = treegenerator.createSyntaxTree( tokenizer.parseFile(fileName, fileName, options.encoding)) if len(options.useVariant) > 0: variantMap = {} for variant in options.useVariant: keyValue = variant.split(":") if len(keyValue) != 2: print " * Error: Variants must be specified as key value pair separated by ':'!" sys.exit(1) variantMap[keyValue[0]] = keyValue[1] variantoptimizer.search(restree, variantMap, fileName) if options.obfuscateAccessors: accessorobfuscator.process(restree) if options.optimizePrivates: privateoptimizer.patch("A", restree, {}) if options.optimizeVariables: variableoptimizer.search(restree, [], 0, 0, "$") if options.compress: options.prettyPrint = False # make sure it's set else: options.prettyPrint = True compiledString = compile(restree, options) if options.write: if compiledString != "" and not compiledString.endswith("\n"): compiledString += "\n" filetool.save(fileName + options.extension, compiledString) else: try: print compiledString except UnicodeEncodeError: print " * Could not encode result to ascii. Use '-w' instead." sys.exit(1)
def main(): parser = optparse.OptionParser() parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.") parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="") parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False) parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.") parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.") (options, args) = parser.parse_args() if len(args) == 0: print "Needs one or more arguments (files) to compile!" sys.exit(1) for fileName in args: if options.write: print "Compiling %s => %s%s" % (fileName, fileName, options.extension) else: print "Compiling %s => stdout" % fileName restree = treegenerator.createSyntaxTree( tokenizer.parseFile(fileName, "", options.encoding)) if options.optimizeVariables: variableoptimizer.search(restree, [], 0, "$") compiledString = compile(restree, not options.compress) if options.write: if compiledString != "" and not compiledString.endswith("\n"): compiledString += "\n" filetool.save(fileName + options.extension, compiledString) else: try: print compiledString except UnicodeEncodeError: print " * Could not encode result to ascii. Use '-w' instead." sys.exit(1)
def compileString(jsString, uniqueId=""): """ Compile a string containing a JavaScript fragment into a syntax tree. """ return treegenerator.createSyntaxTree(tokenizer.parseStream(jsString, uniqueId)).getFirstChild()
def handle(fileList, fileDb, options): confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget) infoPath = os.path.join(confPath, "info") patchPath = os.path.join(confPath, "patches") importedModule = False infoList = [] patchList = [] htmlList = getHtmlList(options) print " * Number of script input files: %s" % len(fileList) print " * Number of HTML input files: %s" % len(htmlList) print " * Update to version: %s" % options.migrationTarget print " * Searching for patch module..." for root, dirs, files in os.walk(confPath): # Filter ignored directories for ignoredDir in config.DIRIGNORE: if ignoredDir in dirs: dirs.remove(ignoredDir) # Searching for files for fileName in files: filePath = os.path.join(root, fileName) if os.path.splitext(fileName)[1] != config.PYEXT: continue if fileName == "patch.py": print " - Importing..." if not root in sys.path: sys.path.insert(0, root) import patch importedModule = True emptyLine = re.compile("^\s*$") print " * Searching for info expression data..." for root, dirs, files in os.walk(infoPath): # Filter ignored directories for ignoredDir in config.DIRIGNORE: if ignoredDir in dirs: dirs.remove(ignoredDir) # Searching for files for fileName in files: filePath = os.path.join(root, fileName) fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8")) infoList.append({"path":filePath, "content":fileContent.split("\n")}) if options.verbose: print " - %s" % filePath print " - Number of info files: %s" % len(infoList) print " - Compiling expressions..." compiledInfos = [] for infoFile in infoList: print " - %s" % os.path.basename(infoFile["path"]) for line in infoFile["content"]: if emptyLine.match(line) or line.startswith("#") or line.startswith("//"): continue compiled = entryCompiler(line) if compiled != None: compiledInfos.append(compiled) print " - Number of infos: %s" % len(compiledInfos) print " * Searching for patch expression data..." for root, dirs, files in os.walk(patchPath): # Filter ignored directories for ignoredDir in config.DIRIGNORE: if ignoredDir in dirs: dirs.remove(ignoredDir) # Searching for files for fileName in files: filePath = os.path.join(root, fileName) fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8")) patchList.append({"path":filePath, "content":fileContent.split("\n")}) if options.verbose: print " - %s" % filePath print " - Number of patch files: %s" % len(patchList) print " - Compiling expressions..." compiledPatches = [] for patchFile in patchList: print " - %s" % os.path.basename(patchFile["path"]) for line in patchFile["content"]: if emptyLine.match(line) or line.startswith("#") or line.startswith("//"): continue compiled = entryCompiler(line) if compiled != None: compiledPatches.append(compiled) print " - Number of patches: %s" % len(compiledPatches) print print " FILE PROCESSING:" print "----------------------------------------------------------------------------" if len(fileList) > 0: print " * Processing script files:" for fileId in fileList: fileEntry = fileDb[fileId] filePath = fileEntry["path"] fileEncoding = fileEntry["encoding"] print " - %s" % fileId # Read in original content fileContent = filetool.read(filePath, fileEncoding) patchedContent = fileContent # Apply patches if importedModule: tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): patchedContent = compiler.compile(tree, True) patchedContent = regtool(patchedContent, compiledPatches, True, options) patchedContent = regtool(patchedContent, compiledInfos, False, options) # Write file if patchedContent != fileContent: print " - Store modifications..." filetool.save(filePath, patchedContent, fileEncoding) print " * Done" if len(htmlList) > 0: print " * Processing HTML files:" for filePath in htmlList: print " - %s" % filePath # Read in original content fileContent = filetool.read(filePath) patchedContent = fileContent patchedContent = regtool(patchedContent, compiledPatches, True, options) patchedContent = regtool(patchedContent, compiledInfos, False, options) # Write file if patchedContent != fileContent: print " - Store modifications..." filetool.save(filePath, patchedContent) print " * Done"