Beispiel #1
0
def getTree(fileDb, fileId, options):
    if not fileDb[fileId].has_key("tree"):
        if options.verbose:
            print "    - Generating tree for %s..." % fileId

        useCache = False
        loadCache = False

        fileEntry = fileDb[fileId]
        filePath = fileEntry["path"]

        if options.cacheDirectory != None:
            cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl")
            useCache = True

            if not filetool.checkCache(filePath, cachePath):
                loadCache = True

        if loadCache:
            tree = filetool.readCache(cachePath)
        else:
            tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options))

            if useCache:
                if options.verbose:
                    print "    - Caching tree for %s..." % fileId

                filetool.storeCache(cachePath, tree)

        fileDb[fileId]["tree"] = tree

    return fileDb[fileId]["tree"]
Beispiel #2
0
def getTree(fileDb, fileId, options):
    if not fileDb[fileId].has_key("tree"):
        if options.verbose:
            print "    - Generating tree for %s..." % fileId

        useCache = False
        loadCache = False

        fileEntry = fileDb[fileId]
        filePath = fileEntry["path"]

        if options.cacheDirectory != None:
            cachePath = os.path.join(filetool.normalize(options.cacheDirectory), fileId + "-tree.pcl")
            useCache = True

            if not filetool.checkCache(filePath, cachePath):
                loadCache = True

        if loadCache:
            tree = filetool.readCache(cachePath)
        else:
            tree = treegenerator.createSyntaxTree(getTokens(fileDb, fileId, options))

            if useCache:
                if options.verbose:
                    print "    - Caching tree for %s..." % fileId

                filetool.storeCache(cachePath, tree)

        fileDb[fileId]["tree"] = tree

    return fileDb[fileId]["tree"]
def migrateFile(
                filePath, compiledPatches, compiledInfos,
                hasPatchModule=False, options=None, encoding="UTF-8"):

    logging.info("  - File: %s" % filePath)

    # Read in original content
    fileContent = filetool.read(filePath, encoding)

    fileId = loader.extractFileContentId(fileContent);

    # Apply patches
    patchedContent = fileContent

    if hasPatchModule and fileId is not None:

        import patch
        tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent))

        # If there were any changes, compile the result
        if patch.patch(fileId, tree):
            options.prettyPrint = True  # make sure it's set
            patchedContent = compiler.compile(tree, options)

    # apply RE patches
    patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
    patchedContent = regtool(patchedContent, compiledInfos, False, filePath)

    # Write file
    if patchedContent != fileContent:
        logging.info("    - %s has been modified. Storing modifications ..." % filePath)
        filetool.save(filePath, patchedContent, encoding)
def main():
    global options

    parser = optparse.OptionParser()

    parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Needs one or more arguments (files) to compile!"
        sys.exit(1)

    for fileName in args:
        print "Processing %s" % (fileName)

        restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding))    
        
        track(restree)
Beispiel #5
0
def main():
  parser = optparse.OptionParser()

  parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
  parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="")
  parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False)
  parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.")
  parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")

  (options, args) = parser.parse_args()

  if len(args) == 0:
    print "Needs one or more arguments (files) to compile!"
    sys.exit(1)

  for fileName in args:
    if options.write:
      print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
    else:
      print "Compiling %s => stdout" % fileName

    restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, "", options.encoding))

    if options.optimizeVariables:
      variableoptimizer.search(restree, [], 0, "$")

    compiledString = compile(restree, not options.compress)
    if options.write:
      if compiledString != "" and not compiledString.endswith("\n"):
        compiledString += "\n"
        
      filetool.save(fileName + options.extension, compiledString)

    else:
      try:
        print compiledString

      except UnicodeEncodeError:
        print "  * Could not encode result to ascii. Use '-w' instead."
        sys.exit(1)
Beispiel #6
0
def migrateFile(filePath,
                compiledPatches,
                compiledInfos,
                hasPatchModule=False,
                options=None,
                encoding="UTF-8"):

    logging.info("  - File: %s" % filePath)

    # Read in original content
    fileContent = filetool.read(filePath, encoding)

    fileId = loader.extractFileContentId(fileContent)

    # Apply patches
    patchedContent = fileContent

    if hasPatchModule and fileId is not None:

        import patch
        tree = treegenerator.createSyntaxTree(
            tokenizer.parseStream(fileContent))

        # If there were any changes, compile the result
        if patch.patch(fileId, tree):
            options.prettyPrint = True  # make sure it's set
            patchedContent = compiler.compile(tree, options)

    # apply RE patches
    patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
    patchedContent = regtool(patchedContent, compiledInfos, False, filePath)

    # Write file
    if patchedContent != fileContent:
        logging.info("    - %s has been modified. Storing modifications ..." %
                     filePath)
        filetool.save(filePath, patchedContent, encoding)
Beispiel #7
0
def main():
    global options

    parser = optparse.OptionParser()

    parser.add_option("--encoding",
                      dest="encoding",
                      default="utf-8",
                      metavar="ENCODING",
                      help="Defines the encoding expected for input files.")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Needs one or more arguments (files) to compile!"
        sys.exit(1)

    for fileName in args:
        print "Processing %s" % (fileName)

        restree = treegenerator.createSyntaxTree(
            tokenizer.parseFile(fileName, fileName, options.encoding))

        track(restree)
def main():
    global options

    parser = optparse.OptionParser(option_class=optparseext.ExtendAction)

    parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
    parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="")
    parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False)
    parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.")
    parser.add_option("--optimize-privates", action="store_true", dest="optimizePrivates", default=False, help="Optimize privates. Protected them and reducing size.")
    parser.add_option("--obfuscate-accessors", action="store_true", dest="obfuscateAccessors", default=False, help="Enable accessor obfuscation")
    parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
    parser.add_option("--use-variant", action="extend", dest="useVariant", type="string", metavar="NAMESPACE.KEY:VALUE", default=[], help="Optimize for the given variant.")
    
    # Options for pretty printing
    addCommandLineOptions(parser)

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Needs one or more arguments (files) to compile!"
        sys.exit(1)

    for fileName in args:
        if options.write:
            print "Compiling %s => %s%s" % (fileName, fileName, options.extension)

        restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding))

        if len(options.useVariant) > 0:
            variantMap = {}
            for variant in options.useVariant:
                keyValue = variant.split(":")
                if len(keyValue) != 2:
                    print "  * Error: Variants must be specified as key value pair separated by ':'!"
                    sys.exit(1)
    
                variantMap[keyValue[0]] = keyValue[1]     
                
            variantoptimizer.search(restree, variantMap, fileName)  
            
        if options.obfuscateAccessors:
            accessorobfuscator.process(restree)
            
        if options.optimizePrivates:
            privateoptimizer.patch("A", restree, {})

        if options.optimizeVariables:
            variableoptimizer.search(restree, [], 0, 0, "$")

        if options.compress:
            options.prettyPrint = False  # make sure it's set
        else:
            options.prettyPrint = True
            
        compiledString = compile(restree, options)
        if options.write:
            if compiledString != "" and not compiledString.endswith("\n"):
                compiledString += "\n"

            filetool.save(fileName + options.extension, compiledString)

        else:
            try:
                print compiledString

            except UnicodeEncodeError:
                print "  * Could not encode result to ascii. Use '-w' instead."
                sys.exit(1)
Beispiel #9
0
def main():
    global options

    parser = optparse.OptionParser(option_class=optparseext.ExtendAction)

    parser.add_option("-w",
                      "--write",
                      action="store_true",
                      dest="write",
                      default=False,
                      help="Writes file to incoming fileName + EXTENSION.")
    parser.add_option("-e",
                      "--extension",
                      dest="extension",
                      metavar="EXTENSION",
                      help="The EXTENSION to use",
                      default="")
    parser.add_option("-c",
                      "--compress",
                      action="store_true",
                      dest="compress",
                      help="Enable compression",
                      default=False)
    parser.add_option("--optimize-variables",
                      action="store_true",
                      dest="optimizeVariables",
                      default=False,
                      help="Optimize variables. Reducing size.")
    parser.add_option(
        "--optimize-privates",
        action="store_true",
        dest="optimizePrivates",
        default=False,
        help="Optimize privates. Protected them and reducing size.")
    parser.add_option("--obfuscate-accessors",
                      action="store_true",
                      dest="obfuscateAccessors",
                      default=False,
                      help="Enable accessor obfuscation")
    parser.add_option("--encoding",
                      dest="encoding",
                      default="utf-8",
                      metavar="ENCODING",
                      help="Defines the encoding expected for input files.")
    parser.add_option("--use-variant",
                      action="extend",
                      dest="useVariant",
                      type="string",
                      metavar="NAMESPACE.KEY:VALUE",
                      default=[],
                      help="Optimize for the given variant.")

    # Options for pretty printing
    addCommandLineOptions(parser)

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Needs one or more arguments (files) to compile!"
        sys.exit(1)

    for fileName in args:
        if options.write:
            print "Compiling %s => %s%s" % (fileName, fileName,
                                            options.extension)

        restree = treegenerator.createSyntaxTree(
            tokenizer.parseFile(fileName, fileName, options.encoding))

        if len(options.useVariant) > 0:
            variantMap = {}
            for variant in options.useVariant:
                keyValue = variant.split(":")
                if len(keyValue) != 2:
                    print "  * Error: Variants must be specified as key value pair separated by ':'!"
                    sys.exit(1)

                variantMap[keyValue[0]] = keyValue[1]

            variantoptimizer.search(restree, variantMap, fileName)

        if options.obfuscateAccessors:
            accessorobfuscator.process(restree)

        if options.optimizePrivates:
            privateoptimizer.patch("A", restree, {})

        if options.optimizeVariables:
            variableoptimizer.search(restree, [], 0, 0, "$")

        if options.compress:
            options.prettyPrint = False  # make sure it's set
        else:
            options.prettyPrint = True

        compiledString = compile(restree, options)
        if options.write:
            if compiledString != "" and not compiledString.endswith("\n"):
                compiledString += "\n"

            filetool.save(fileName + options.extension, compiledString)

        else:
            try:
                print compiledString

            except UnicodeEncodeError:
                print "  * Could not encode result to ascii. Use '-w' instead."
                sys.exit(1)
Beispiel #10
0
def main():
    parser = optparse.OptionParser()

    parser.add_option("-w",
                      "--write",
                      action="store_true",
                      dest="write",
                      default=False,
                      help="Writes file to incoming fileName + EXTENSION.")
    parser.add_option("-e",
                      "--extension",
                      dest="extension",
                      metavar="EXTENSION",
                      help="The EXTENSION to use",
                      default="")
    parser.add_option("-c",
                      "--compress",
                      action="store_true",
                      dest="compress",
                      help="Enable compression",
                      default=False)
    parser.add_option("--optimize-variables",
                      action="store_true",
                      dest="optimizeVariables",
                      default=False,
                      help="Optimize variables. Reducing size.")
    parser.add_option("--encoding",
                      dest="encoding",
                      default="utf-8",
                      metavar="ENCODING",
                      help="Defines the encoding expected for input files.")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Needs one or more arguments (files) to compile!"
        sys.exit(1)

    for fileName in args:
        if options.write:
            print "Compiling %s => %s%s" % (fileName, fileName,
                                            options.extension)
        else:
            print "Compiling %s => stdout" % fileName

        restree = treegenerator.createSyntaxTree(
            tokenizer.parseFile(fileName, "", options.encoding))

        if options.optimizeVariables:
            variableoptimizer.search(restree, [], 0, "$")

        compiledString = compile(restree, not options.compress)
        if options.write:
            if compiledString != "" and not compiledString.endswith("\n"):
                compiledString += "\n"

            filetool.save(fileName + options.extension, compiledString)

        else:
            try:
                print compiledString

            except UnicodeEncodeError:
                print "  * Could not encode result to ascii. Use '-w' instead."
                sys.exit(1)
Beispiel #11
0
def compileString(jsString, uniqueId=""):
    """
    Compile a string containing a JavaScript fragment into a syntax tree.
    """
    return treegenerator.createSyntaxTree(tokenizer.parseStream(jsString, uniqueId)).getFirstChild()
Beispiel #12
0
def handle(fileList, fileDb, options):
  confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget)

  infoPath = os.path.join(confPath, "info")
  patchPath = os.path.join(confPath, "patches")

  importedModule = False
  infoList = []
  patchList = []
  htmlList = getHtmlList(options)




  print "  * Number of script input files: %s" % len(fileList)
  print "  * Number of HTML input files: %s" % len(htmlList)
  print "  * Update to version: %s" % options.migrationTarget



  print "  * Searching for patch module..."

  for root, dirs, files in os.walk(confPath):

    # Filter ignored directories
    for ignoredDir in config.DIRIGNORE:
      if ignoredDir in dirs:
        dirs.remove(ignoredDir)

    # Searching for files
    for fileName in files:
      filePath = os.path.join(root, fileName)

      if os.path.splitext(fileName)[1] != config.PYEXT:
        continue

      if fileName == "patch.py":
        print "    - Importing..."

        if not root in sys.path:
          sys.path.insert(0, root)

        import patch
        importedModule = True







  emptyLine = re.compile("^\s*$")



  print "  * Searching for info expression data..."

  for root, dirs, files in os.walk(infoPath):

    # Filter ignored directories
    for ignoredDir in config.DIRIGNORE:
      if ignoredDir in dirs:
        dirs.remove(ignoredDir)

    # Searching for files
    for fileName in files:
      filePath = os.path.join(root, fileName)

      fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
      infoList.append({"path":filePath, "content":fileContent.split("\n")})

      if options.verbose:
        print "    - %s" % filePath

  print "    - Number of info files: %s" % len(infoList)

  print "    - Compiling expressions..."

  compiledInfos = []

  for infoFile in infoList:
    print "      - %s" % os.path.basename(infoFile["path"])
    for line in infoFile["content"]:
      if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
        continue

      compiled = entryCompiler(line)
      if compiled != None:
        compiledInfos.append(compiled)

  print "    - Number of infos: %s" % len(compiledInfos)




  print "  * Searching for patch expression data..."

  for root, dirs, files in os.walk(patchPath):

    # Filter ignored directories
    for ignoredDir in config.DIRIGNORE:
      if ignoredDir in dirs:
        dirs.remove(ignoredDir)

    # Searching for files
    for fileName in files:
      filePath = os.path.join(root, fileName)

      fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
      patchList.append({"path":filePath, "content":fileContent.split("\n")})

      if options.verbose:
        print "    - %s" % filePath

  print "    - Number of patch files: %s" % len(patchList)

  print "    - Compiling expressions..."

  compiledPatches = []

  for patchFile in patchList:
    print "      - %s" % os.path.basename(patchFile["path"])
    for line in patchFile["content"]:
      if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
        continue

      compiled = entryCompiler(line)
      if compiled != None:
        compiledPatches.append(compiled)

  print "    - Number of patches: %s" % len(compiledPatches)








  print
  print "  FILE PROCESSING:"
  print "----------------------------------------------------------------------------"

  if len(fileList) > 0:
    print "  * Processing script files:"

    for fileId in fileList:
      fileEntry = fileDb[fileId]

      filePath = fileEntry["path"]
      fileEncoding = fileEntry["encoding"]

      print "    - %s" % fileId

      # Read in original content
      fileContent = filetool.read(filePath, fileEncoding)
      patchedContent = fileContent

      # Apply patches
      if importedModule:
        tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent))

        # If there were any changes, compile the result
        if patch.patch(fileId, tree):
          patchedContent = compiler.compile(tree, True)

      patchedContent = regtool(patchedContent, compiledPatches, True, options)
      patchedContent = regtool(patchedContent, compiledInfos, False, options)

      # Write file
      if patchedContent != fileContent:
        print "      - Store modifications..."
        filetool.save(filePath, patchedContent, fileEncoding)

    print "  * Done"



  if len(htmlList) > 0:
    print "  * Processing HTML files:"

    for filePath in htmlList:
      print "    - %s" % filePath

      # Read in original content
      fileContent = filetool.read(filePath)

      patchedContent = fileContent
      patchedContent = regtool(patchedContent, compiledPatches, True, options)
      patchedContent = regtool(patchedContent, compiledInfos, False, options)

      # Write file
      if patchedContent != fileContent:
        print "      - Store modifications..."
        filetool.save(filePath, patchedContent)

    print "  * Done"