def migrateFile( filePath, compiledPatches, compiledInfos, hasPatchModule=False, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = loader.extractFileContentId(fileContent); # Apply patches patchedContent = fileContent if hasPatchModule and fileId is not None: import patch tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): options.prettyPrint = True # make sure it's set patchedContent = compiler.compile(tree, options) # apply RE patches patchedContent = regtool(patchedContent, compiledPatches, True, filePath) patchedContent = regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def getTokens(fileDb, fileId, options): if not fileDb[fileId].has_key("tokens"): if options.verbose: print " - Generating tokens for %s..." % fileId useCache = False loadCache = False fileEntry = fileDb[fileId] filePath = fileEntry["path"] fileEncoding = fileEntry["encoding"] if options.cacheDirectory != None: cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tokens.pcl") useCache = True if not filetool.checkCache(filePath, cachePath, getInternalModTime(options)): loadCache = True if loadCache: tokens = filetool.readCache(cachePath) else: fileContent = filetool.read(filePath, fileEncoding) tokens = tokenizer.parseStream(fileContent, fileId) if useCache: if options.verbose: print " - Caching tokens for %s..." % fileId filetool.storeCache(cachePath, tokens) fileDb[fileId]["tokens"] = tokens return fileDb[fileId]["tokens"]
def getTokens(fileDb, fileId, options): if not fileDb[fileId].has_key("tokens"): if options.verbose: print " - Generating tokens for %s..." % fileId useCache = False loadCache = False fileEntry = fileDb[fileId] filePath = fileEntry["path"] fileEncoding = fileEntry["encoding"] if options.cacheDirectory != None: cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tokens.pcl") useCache = True if not filetool.checkCache(filePath, cachePath): loadCache = True if loadCache: tokens = filetool.readCache(cachePath) else: fileContent = filetool.read(filePath, fileEncoding) # TODO: This hack is neccesary because the current parser cannot handle comments # without a context. if fileDb[fileId]["meta"]: fileContent += "\n(function() {})()" tokens = tokenizer.parseStream(fileContent, fileId) if useCache: if options.verbose: print " - Caching tokens for %s..." % fileId filetool.storeCache(cachePath, tokens) fileDb[fileId]["tokens"] = tokens return fileDb[fileId]["tokens"]
def getTokens(fileDb, fileId, options): if not fileDb[fileId].has_key("tokens"): if options.verbose: print " - Generating tokens for %s..." % fileId useCache = False loadCache = False fileEntry = fileDb[fileId] filePath = fileEntry["path"] fileEncoding = fileEntry["encoding"] if options.cacheDirectory != None: cachePath = os.path.join( filetool.normalize(options.cacheDirectory), fileId + "-tokens.pcl") useCache = True if not filetool.checkCache(filePath, cachePath, getInternalModTime(options)): loadCache = True if loadCache: tokens = filetool.readCache(cachePath) else: fileContent = filetool.read(filePath, fileEncoding) tokens = tokenizer.parseStream(fileContent, fileId) if useCache: if options.verbose: print " - Caching tokens for %s..." % fileId filetool.storeCache(cachePath, tokens) fileDb[fileId]["tokens"] = tokens return fileDb[fileId]["tokens"]
def migrateFile(filePath, compiledPatches, compiledInfos, hasPatchModule=False, options=None, encoding="UTF-8"): logging.info(" - File: %s" % filePath) # Read in original content fileContent = filetool.read(filePath, encoding) fileId = loader.extractFileContentId(fileContent) # Apply patches patchedContent = fileContent if hasPatchModule and fileId is not None: import patch tree = treegenerator.createSyntaxTree( tokenizer.parseStream(fileContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): options.prettyPrint = True # make sure it's set patchedContent = compiler.compile(tree, options) # apply RE patches patchedContent = regtool(patchedContent, compiledPatches, True, filePath) patchedContent = regtool(patchedContent, compiledInfos, False, filePath) # Write file if patchedContent != fileContent: logging.info(" - %s has been modified. Storing modifications ..." % filePath) filetool.save(filePath, patchedContent, encoding)
def compileString(jsString, uniqueId=""): """ Compile a string containing a JavaScript fragment into a syntax tree. """ return treegenerator.createSyntaxTree(tokenizer.parseStream(jsString, uniqueId)).getFirstChild()
def handle(fileList, fileDb, options): confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget) infoPath = os.path.join(confPath, "info") patchPath = os.path.join(confPath, "patches") importedModule = False infoList = [] patchList = [] htmlList = getHtmlList(options) print " * Number of script input files: %s" % len(fileList) print " * Number of HTML input files: %s" % len(htmlList) print " * Update to version: %s" % options.migrationTarget print " * Searching for patch module..." for root, dirs, files in os.walk(confPath): # Filter ignored directories for ignoredDir in config.DIRIGNORE: if ignoredDir in dirs: dirs.remove(ignoredDir) # Searching for files for fileName in files: filePath = os.path.join(root, fileName) if os.path.splitext(fileName)[1] != config.PYEXT: continue if fileName == "patch.py": print " - Importing..." if not root in sys.path: sys.path.insert(0, root) import patch importedModule = True emptyLine = re.compile("^\s*$") print " * Searching for info expression data..." for root, dirs, files in os.walk(infoPath): # Filter ignored directories for ignoredDir in config.DIRIGNORE: if ignoredDir in dirs: dirs.remove(ignoredDir) # Searching for files for fileName in files: filePath = os.path.join(root, fileName) fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8")) infoList.append({"path":filePath, "content":fileContent.split("\n")}) if options.verbose: print " - %s" % filePath print " - Number of info files: %s" % len(infoList) print " - Compiling expressions..." compiledInfos = [] for infoFile in infoList: print " - %s" % os.path.basename(infoFile["path"]) for line in infoFile["content"]: if emptyLine.match(line) or line.startswith("#") or line.startswith("//"): continue compiled = entryCompiler(line) if compiled != None: compiledInfos.append(compiled) print " - Number of infos: %s" % len(compiledInfos) print " * Searching for patch expression data..." for root, dirs, files in os.walk(patchPath): # Filter ignored directories for ignoredDir in config.DIRIGNORE: if ignoredDir in dirs: dirs.remove(ignoredDir) # Searching for files for fileName in files: filePath = os.path.join(root, fileName) fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8")) patchList.append({"path":filePath, "content":fileContent.split("\n")}) if options.verbose: print " - %s" % filePath print " - Number of patch files: %s" % len(patchList) print " - Compiling expressions..." compiledPatches = [] for patchFile in patchList: print " - %s" % os.path.basename(patchFile["path"]) for line in patchFile["content"]: if emptyLine.match(line) or line.startswith("#") or line.startswith("//"): continue compiled = entryCompiler(line) if compiled != None: compiledPatches.append(compiled) print " - Number of patches: %s" % len(compiledPatches) print print " FILE PROCESSING:" print "----------------------------------------------------------------------------" if len(fileList) > 0: print " * Processing script files:" for fileId in fileList: fileEntry = fileDb[fileId] filePath = fileEntry["path"] fileEncoding = fileEntry["encoding"] print " - %s" % fileId # Read in original content fileContent = filetool.read(filePath, fileEncoding) patchedContent = fileContent # Apply patches if importedModule: tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent)) # If there were any changes, compile the result if patch.patch(fileId, tree): patchedContent = compiler.compile(tree, True) patchedContent = regtool(patchedContent, compiledPatches, True, options) patchedContent = regtool(patchedContent, compiledInfos, False, options) # Write file if patchedContent != fileContent: print " - Store modifications..." filetool.save(filePath, patchedContent, fileEncoding) print " * Done" if len(htmlList) > 0: print " * Processing HTML files:" for filePath in htmlList: print " - %s" % filePath # Read in original content fileContent = filetool.read(filePath) patchedContent = fileContent patchedContent = regtool(patchedContent, compiledPatches, True, options) patchedContent = regtool(patchedContent, compiledInfos, False, options) # Write file if patchedContent != fileContent: print " - Store modifications..." filetool.save(filePath, patchedContent) print " * Done"