Пример #1
0
def gunzip(filePath, encoding="utf-8"):
    if not filePath.endswith(".gz"):
        filePath = filePath + ".gz"

    inputFile = sys_gzip.open(filePath, "rb")
    content = inputFile.read()

    return textutil.any2Unix(unicode(content))
Пример #2
0
def gunzip(filePath, encoding="utf-8"):
    if not filePath.endswith(".gz"):
        filePath = filePath + ".gz"

    inputFile = sys_gzip.open(filePath, "rb")
    content = inputFile.read()
    
    return textutil.any2Unix(unicode(content))
Пример #3
0
def read(filePath, encoding="utf_8"):
    try:
        ref = codecs.open(filePath, encoding=encoding, mode="r")
        content = ref.read()
        ref.close()

        return textutil.any2Unix(unicode(content))

    except IOError, (errno, strerror):
        print "  * I/O error(%s): %s" % (errno, strerror)
        sys.exit(1)
Пример #4
0
def read(filePath, encoding="utf_8"):
    try:
        ref = codecs.open(filePath, encoding=encoding, mode="r")
        content = ref.read()
        ref.close()

        return textutil.any2Unix(unicode(content))

    except IOError, (errno, strerror):
        print "  * I/O error(%s): %s (%s)" % (errno, strerror, filePath)
        sys.exit(1)
Пример #5
0
def readPatchInfoFiles(baseDir):
    """
    Reads all patch/info files from a directory and compiles the containing
    regular expressions.
    Retuns a list comiled RE (the output of entryCompiler)
    """
    patchList = []
    emptyLine = re.compile("^\s*$")

    for root, dirs, files in os.walk(baseDir):

        # Filter ignored directories
        for ignoredDir in config.DIRIGNORE:
            if ignoredDir in dirs:
                dirs.remove(ignoredDir)

        # Searching for files
        for fileName in files:
            filePath = os.path.join(root, fileName)

            fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
            patchList.append({
                "path": filePath,
                "content": fileContent.split("\n")
            })

            logging.debug("    - %s" % filePath)

    logging.debug("    - Compiling expressions...")

    compiledPatches = []

    for patchFile in patchList:
        logging.debug("      - %s" % os.path.basename(patchFile["path"]))
        for line in patchFile["content"]:
            if emptyLine.match(line) or line.startswith(
                    "#") or line.startswith("//"):
                continue

            compiled = entryCompiler(line)
            if compiled != None:
                compiledPatches.append(compiled)

    return compiledPatches
Пример #6
0
def read(filePath, encoding="utf_8"):
    def getRowCol(text, pos):
        row = len(text[:pos].split("\n"))
        col = pos - text[:pos].rfind("\n")
        if col < 0:
            col = pos
        return (row, col)

    try:
        ref = codecs.open(filePath, encoding=encoding, mode="r")
        try:
            content = ref.read()
        except UnicodeDecodeError, e:
            rowCol = getRowCol(e.object, e.start)
            e.reason = e.reason + "\nFile %s Line %s Column %s" % (str(filePath), str(rowCol[0]), str(rowCol[1]))
            raise e
        finally:
            ref.close()

        return textutil.any2Unix(unicode(content))
Пример #7
0
def read(filePath, encoding="utf_8"):
    def getRowCol(text, pos):
        row = len(text[:pos].split("\n"))
        col = pos - text[:pos].rfind("\n")
        if col < 0:
            col = pos
        return (row, col)

    try:
        ref = codecs.open(filePath, encoding=encoding, mode="r")
        try:
            content = ref.read()
        except UnicodeDecodeError, e:
            rowCol = getRowCol(e.object, e.start)
            e.reason = e.reason + "\nFile %s Line %s Column %s" % (
                str(filePath), str(rowCol[0]), str(rowCol[1]))
            raise e
        finally:
            ref.close()

        return textutil.any2Unix(unicode(content))
Пример #8
0
def readPatchInfoFiles(baseDir):
    """
    Reads all patch/info files from a directory and compiles the containing
    regular expressions.
    Retuns a list comiled RE (the output of entryCompiler)
    """
    patchList = []
    emptyLine = re.compile("^\s*$")

    for root, dirs, files in os.walk(baseDir):

        # Filter ignored directories
        for ignoredDir in config.DIRIGNORE:
            if ignoredDir in dirs:
                dirs.remove(ignoredDir)

        # Searching for files
        for fileName in files:
            filePath = os.path.join(root, fileName)

            fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
            patchList.append({"path":filePath, "content":fileContent.split("\n")})

            logging.debug("    - %s" % filePath)

    logging.debug("    - Compiling expressions...")

    compiledPatches = []

    for patchFile in patchList:
        logging.debug("      - %s" % os.path.basename(patchFile["path"]))
        for line in patchFile["content"]:
            if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
                continue

            compiled = entryCompiler(line)
            if compiled != None:
                compiledPatches.append(compiled)

    return compiledPatches
Пример #9
0
def execute(fileDb, moduleDb, options, pkgid="", names=[]):

    additionalOutput = []

    ######################################################################
    #  SORT OF INCLUDE LIST
    ######################################################################

    print
    print "  SORT OF INCLUDE LIST:"
    print "----------------------------------------------------------------------------"

    if options.verbose:
        print "  * Include (with dependencies): %s" % options.includeWithDeps
        print "  * Include (without dependencies): %s" % options.includeWithoutDeps
        print "  * Exclude (with dependencies): %s" % options.excludeWithDeps
        print "  * Exclude (without dependencies): %s" % options.excludeWithoutDeps

    print "  * Sorting %s classes..." % len(fileDb)

    sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)

    if len(sortedIncludeList) == 0:
        print "    - No class files to include. Exciting!"
        sys.exit(1)
    else:
        print "    - Including %s classes" % len(sortedIncludeList)

    if options.printIncludes:
        print
        print "  PRINT OF INCLUDE ORDER:"
        print "----------------------------------------------------------------------------"
        print "  * The files will be included in this order:"
        for fileId in sortedIncludeList:
            print "    - %s" % fileId

    if options.printDeps:
        print
        print "  OUTPUT OF DEPENDENCIES:"
        print "----------------------------------------------------------------------------"
        print "  * These are all included files with their dependencies:"
        for fileId in sortedIncludeList:
            print "    - %s" % fileId
            if len(fileDb[fileId]["loadtimeDeps"]) > 0:
                print "      - Loadtime: "
                for depEntry in fileDb[fileId]["loadtimeDeps"]:
                    print "        - %s" % depEntry

            if len(fileDb[fileId]["afterDeps"]) > 0:
                print "      - After: "
                for depEntry in fileDb[fileId]["afterDeps"]:
                    print "        - %s" % depEntry

            if len(fileDb[fileId]["runtimeDeps"]) > 0:
                print "      - Runtime: "
                for depEntry in fileDb[fileId]["runtimeDeps"]:
                    print "        - %s" % depEntry

            if len(fileDb[fileId]["loadDeps"]) > 0:
                print "      - Load: "
                for depEntry in fileDb[fileId]["loadDeps"]:
                    print "        - %s" % depEntry

            if len(fileDb[fileId]["optionalDeps"]) > 0:
                print "      - Optional: "
                for depEntry in fileDb[fileId]["optionalDeps"]:
                    print "        - %s" % depEntry

    ######################################################################
    #  GRAPHVIZ OUTPUT
    ######################################################################

    if options.depDotFile:
        graph.store(fileDb, sortedIncludeList, options)

    ######################################################################
    #  SOURCE MIGRATION
    ######################################################################

    if options.migrateSource:
        print
        print "  SOURCE MIGRATION:"
        print "----------------------------------------------------------------------------"

        print "  * Migrate Source Code..."

        migrator.handle(sortedIncludeList, fileDb, options)

        # Return after migration: Ignore other jobs
        return

    ######################################################################
    #  GENERATION OF PRETTY PRINTED CODE
    ######################################################################

    if options.fixSource:
        print
        print "  FIX SOURCE CODE:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Fixing code..."
        else:
            print "  * Fixing code: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - Reading %s" % fileId

            fileEntry = fileDb[fileId]

            filePath = fileEntry["path"]
            fileEncoding = fileEntry["encoding"]

            fileContent = filetool.read(filePath, fileEncoding)
            fixedContent = textutil.removeTrailingSpaces(
                textutil.tab2Space(textutil.any2Unix(fileContent), 2))

            if fixedContent != fileContent:
                if options.verbose:
                    print "      - Storing modifications..."
                else:
                    sys.stdout.write("!")
                    sys.stdout.flush()

                filetool.save(filePath, fixedContent, fileEncoding)

            elif not options.verbose:
                sys.stdout.write(".")
                sys.stdout.flush()

        if not options.verbose:
            print

        # Return after fixing: Ignore other jobs
        return

    ######################################################################
    #  GENERATION OF PRETTY PRINTED CODE
    ######################################################################

    if options.prettyPrint:
        print
        print "  GENERATION OF PRETTY PRINTED CODE:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Pretty printing..."
        else:
            print "  * Pretty printing: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - Compiling %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            prettyFileContent = compiler.compile(
                loader.getTree(fileDb, fileId, options), True)

            if not prettyFileContent.endswith("\n"):
                prettyFileContent += "\n"

            filetool.save(fileDb[fileId]["path"], prettyFileContent)

        if not options.verbose:
            print

        # Return after pretty print: Ignore other jobs
        return

    ######################################################################
    #  STRING OPTIMIZATION
    ######################################################################

    if options.optimizeStrings:
        print
        print "  STRING OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Searching strings..."
        else:
            print "  * Searching strings: ",

        stringMap = {}

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            localMap = loader.getStrings(fileDb, fileId, options)

            for value in localMap:
                if value in stringMap:
                    stringMap[value] += localMap[value]
                else:
                    stringMap[value] = localMap[value]

        if not options.verbose:
            print

        counter = 0
        for value in stringMap:
            counter += stringMap[value]

        stringList = stringoptimizer.sort(stringMap)

        print "  * Found %s strings (used %s times)" % (len(stringMap),
                                                        counter)

        if options.verbose:
            print "  * Replacing strings..."
        else:
            print "  * Replacing strings: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            stringoptimizer.replace(loader.getTree(fileDb, fileId, options),
                                    stringList, "$" + pkgid, options.verbose)

        if not options.verbose:
            print

        print "  * Generating replacement..."
        additionalOutput.append(
            stringoptimizer.replacement(stringList, "$" + pkgid))

    ######################################################################
    #  LOCAL VARIABLE OPTIMIZATION
    ######################################################################

    if options.optimizeVariables:
        print
        print "  LOCAL VARIABLE OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Optimizing variables..."
        else:
            print "  * Optimizing variables: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            variableoptimizer.search(
                loader.getTree(fileDb, fileId, options), [],
                0,
                "$",
                skipPrefix=options.optimizeVariablesSkipPrefix,
                debug=options.enableDebug)

        if not options.verbose:
            print

    ######################################################################
    #  NAME OBFUSCATION
    ######################################################################

    if options.obfuscateIdentifiers:
        print
        print "  OBFUSCATE IDENTIFIERS:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Obfuscating identifiers..."
        else:
            print "  * Obfuscating identifiers: ",

        counter = 0

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            counter += obfuscator.update(
                loader.getTree(fileDb, fileId, options), names, "$$")

        if not options.verbose:
            print

        print "  * Updated %s names" % counter

    ######################################################################
    #  TOKEN STORAGE
    ######################################################################

    if options.storeTokens:
        print
        print "  TOKEN STORAGE:"
        print "----------------------------------------------------------------------------"

        if options.tokenOutputDirectory == None:
            print "  * You must define the token output directory!"
            sys.exit(1)

        if options.verbose:
            print "  * Storing tokens..."
        else:
            print "  * Storing tokens: ",

        for fileId in sortedIncludeList:
            tokenString = tokenizer.convertTokensToString(
                loader.getTokens(fileDb, fileId, options))

            if options.verbose:
                print "    * writing tokens for %s (%s KB)..." % (
                    fileIdm, len(tokenString) / 1000.0)
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            filetool.save(
                os.path.join(filetool.normalize(options.tokenOutputDirectory),
                             fileId + config.TOKENEXT), tokenString)

        if not options.verbose:
            print

    ######################################################################
    #  TREE STORAGE
    ######################################################################

    if options.storeTree:
        print
        print "  TREE STORAGE:"
        print "----------------------------------------------------------------------------"

        if options.treeOutputDirectory == None:
            print "  * You must define the tree output directory!"
            sys.exit(1)

        if options.verbose:
            print "  * Storing tree..."
        else:
            print "  * Storing tree: ",

        for fileId in sortedIncludeList:
            treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(
                loader.getTree(fileDb, fileId, options))

            if options.verbose:
                print "    * writing tree for %s (%s KB)..." % (
                    fileId, len(treeString) / 1000.0)
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            filetool.save(
                os.path.join(filetool.normalize(options.treeOutputDirectory),
                             fileId + config.XMLEXT), treeString)

        if not options.verbose:
            print

    ######################################################################
    #  GENERATION OF API
    ######################################################################

    if options.generateApiDocumentation:
        print
        print "  GENERATION OF API:"
        print "----------------------------------------------------------------------------"

        if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
            print "  * You must define one of JSON or XML API documentation file!"

        docTree = None

        if options.verbose:
            print "  * Generating API tree..."
        else:
            print "  * Generating API tree: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            docTree = api.createDoc(loader.getTree(fileDb, fileId, options),
                                    docTree)

        if not options.verbose:
            print

        if docTree:
            print "  * Finalizing tree..."
            api.postWorkPackage(docTree, docTree)

        if options.apiDocumentationXmlFile != None:
            print "  * Writing XML API file to %s" % options.apiDocumentationXmlFile

            xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"

            if options.addNewLines:
                xmlContent += "\n" + tree.nodeToXmlString(docTree)
            else:
                xmlContent += tree.nodeToXmlString(docTree, "", "", "")

            filetool.save(options.apiDocumentationXmlFile, xmlContent,
                          options.xmlOutputEncoding)

        if options.apiDocumentationJsonFile != None:
            print "  * Writing JSON API file to %s" % options.apiDocumentationJsonFile

            if options.addNewLines:
                jsonContent = tree.nodeToJsonString(docTree)
            else:
                jsonContent = tree.nodeToJsonString(docTree, "", "", "")

            filetool.save(options.apiDocumentationJsonFile, jsonContent,
                          options.scriptOutputEncoding)

    ######################################################################
    #  CREATE COPY OF RESOURCES
    ######################################################################

    if options.copyResources:

        print
        print "  CREATE COPY OF RESOURCES:"
        print "----------------------------------------------------------------------------"

        resources.copy(options, sortedIncludeList, fileDb)

    ######################################################################
    #  GENERATION OF SETTINGS
    ######################################################################

    if options.generateSourceScript or options.generateCompiledScript:
        settingsStr = ""

        if len(options.defineRuntimeSetting) != 0:
            print
            print "  GENERATION OF SETTINGS:"
            print "----------------------------------------------------------------------------"

            print "  * Processing input data..."
            settingsStr = settings.generate(options)

            if options.settingsScriptFile:
                print "   * Storing result to %s" % options.settingsScriptFile
                filetool.save(options.settingsScriptFile, settingsStr)

                # clear settings for build and source
                settingsStr = ""

    ######################################################################
    #  GENERATION OF SOURCE VERSION
    ######################################################################

    if options.generateSourceScript:
        print
        print "  GENERATION OF SOURCE SCRIPT:"
        print "----------------------------------------------------------------------------"

        if options.sourceScriptFile == None and (
                options.sourceTemplateInputFile == None
                or options.sourceTemplateOutputFile == None):
            print "  * You must define at least one source script file or template input/output."
            sys.exit(1)

        if options.sourceScriptFile:
            options.sourceScriptFile = os.path.normpath(
                options.sourceScriptFile)

        if options.sourceTemplateInputFile:
            options.sourceTemplateInputFile = os.path.normpath(
                options.sourceTemplateInputFile)

        if options.sourceTemplateOutputFile:
            options.sourceTemplateOutputFile = os.path.normpath(
                options.sourceTemplateOutputFile)

        print "  * Generating script block..."

        # Handling line feed setting
        sourceLineFeed = ""
        if options.addNewLines:
            sourceLineFeed = "\n"

        # Generating inline code...
        inlineCode = ""
        inlineCode += settingsStr + sourceLineFeed
        inlineCode += "qx.IS_SOURCE=true;%s" % sourceLineFeed
        inlineCode += "qx.VERSION=\"%s\";%s" % (options.version,
                                                sourceLineFeed)
        inlineCode += "".join(additionalOutput)

        # Generating script block
        scriptBlocks = ""
        scriptBlocks += '<script type="text/javascript">%s</script>' % inlineCode
        for fileId in sortedIncludeList:
            if fileDb[fileId]["classUri"] == None:
                print "  * Missing class URI definition for class path %s." % fileDb[
                    fileId]["classPath"]
                sys.exit(1)

            scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (
                os.path.join(fileDb[fileId]["classUri"],
                             fileDb[fileId]["pathId"].replace(
                                 ".", os.sep)), config.JSEXT)
            scriptBlocks += sourceLineFeed

        if options.sourceScriptFile != None:
            print "  * Storing includer as %s..." % options.sourceScriptFile
            sourceScript = "document.write('%s');" % scriptBlocks.replace(
                "'", "\\'")
            if options.addNewLines:
                sourceScript = sourceScript.replace("\n", "\\\n")
            filetool.save(options.sourceScriptFile, sourceScript,
                          options.scriptOutputEncoding)

        if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
            print "  * Patching template: %s => %s" % (
                options.sourceTemplateInputFile,
                options.sourceTemplateOutputFile)
            tmpl = filetool.read(options.sourceTemplateInputFile)
            res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
            filetool.save(options.sourceTemplateOutputFile, res,
                          options.scriptOutputEncoding)

    ######################################################################
    #  GENERATION OF COMPILED VERSION
    ######################################################################

    if options.generateCompiledScript:
        print
        print "  GENERATION OF COMPILED SCRIPT:"
        print "----------------------------------------------------------------------------"

        buildLineFeed = ""
        if options.addNewLines:
            buildLineFeed = "\n"

        inlineCode = ""
        inlineCode += settingsStr + buildLineFeed
        inlineCode += "qx.IS_SOURCE=false;%s" % buildLineFeed
        inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, buildLineFeed)
        inlineCode += "".join(additionalOutput)

        compiledOutput = inlineCode

        if options.compiledScriptFile == None:
            print "  * You must define the compiled script file!"
            sys.exit(1)

        if options.verbose:
            print "  * Compiling..."
        else:
            print "  * Compiling: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - Compiling %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            compiledFileContent = compiler.compile(
                loader.getTree(fileDb, fileId, options), False,
                options.addNewLines, options.enableDebug)

            if options.addFileIds:
                compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
            else:
                compiledOutput += compiledFileContent

            if not compiledOutput.endswith(
                    ";") and not compiledOutput.endswith("\n"):
                compiledOutput += ";"

        if not options.verbose:
            print

        print "  * Storing output as %s..." % options.compiledScriptFile
        filetool.save(options.compiledScriptFile, compiledOutput,
                      options.scriptOutputEncoding)
Пример #10
0
def execute(fileDb, moduleDb, options, pkgid="", names=[]):

  additionalOutput = []


  ######################################################################
  #  SORT OF INCLUDE LIST
  ######################################################################

  print
  print "  SORT OF INCLUDE LIST:"
  print "----------------------------------------------------------------------------"

  if options.verbose:
    print "  * Include (with dependencies): %s" % options.includeWithDeps
    print "  * Include (without dependencies): %s" % options.includeWithoutDeps
    print "  * Exclude (with dependencies): %s" % options.excludeWithDeps
    print "  * Exclude (without dependencies): %s" % options.excludeWithoutDeps

  print "  * Sorting %s classes..." % len(fileDb)

  sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)

  if len(sortedIncludeList) == 0:
  	print "    - No class files to include. Exciting!"
  	sys.exit(1)
  else:
  	print "    - Including %s classes" % len(sortedIncludeList)

  if options.printIncludes:
    print
    print "  PRINT OF INCLUDE ORDER:"
    print "----------------------------------------------------------------------------"
    print "  * The files will be included in this order:"
    for fileId in sortedIncludeList:
      print "    - %s" % fileId

  if options.printDeps:
    print
    print "  OUTPUT OF DEPENDENCIES:"
    print "----------------------------------------------------------------------------"
    print "  * These are all included files with their dependencies:"
    for fileId in sortedIncludeList:
      print "    - %s" % fileId
      if len(fileDb[fileId]["loadtimeDeps"]) > 0:
        print "      - Loadtime: "
        for depEntry in fileDb[fileId]["loadtimeDeps"]:
          print "        - %s" % depEntry

      if len(fileDb[fileId]["afterDeps"]) > 0:
        print "      - After: "
        for depEntry in fileDb[fileId]["afterDeps"]:
          print "        - %s" % depEntry

      if len(fileDb[fileId]["runtimeDeps"]) > 0:
        print "      - Runtime: "
        for depEntry in fileDb[fileId]["runtimeDeps"]:
          print "        - %s" % depEntry

      if len(fileDb[fileId]["loadDeps"]) > 0:
        print "      - Load: "
        for depEntry in fileDb[fileId]["loadDeps"]:
          print "        - %s" % depEntry

      if len(fileDb[fileId]["optionalDeps"]) > 0:
        print "      - Optional: "
        for depEntry in fileDb[fileId]["optionalDeps"]:
          print "        - %s" % depEntry





  ######################################################################
  #  GRAPHVIZ OUTPUT
  ######################################################################

  if options.depDotFile:
    graph.store(fileDb, sortedIncludeList, options)






  ######################################################################
  #  SOURCE MIGRATION
  ######################################################################

  if options.migrateSource:
    print
    print "  SOURCE MIGRATION:"
    print "----------------------------------------------------------------------------"

    print "  * Migrate Source Code..."

    migrator.handle(sortedIncludeList, fileDb, options)

    # Return after migration: Ignore other jobs
    return





  ######################################################################
  #  GENERATION OF PRETTY PRINTED CODE
  ######################################################################

  if options.fixSource:
    print
    print "  FIX SOURCE CODE:"
    print "----------------------------------------------------------------------------"

    if options.verbose:
      print "  * Fixing code..."
    else:
      print "  * Fixing code: ",

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - Reading %s" % fileId

      fileEntry = fileDb[fileId]

      filePath = fileEntry["path"]
      fileEncoding = fileEntry["encoding"]

      fileContent = filetool.read(filePath, fileEncoding)
      fixedContent = textutil.removeTrailingSpaces(textutil.tab2Space(textutil.any2Unix(fileContent), 2))

      if fixedContent != fileContent:
        if options.verbose:
          print "      - Storing modifications..."
        else:
          sys.stdout.write("!")
          sys.stdout.flush()

        filetool.save(filePath, fixedContent, fileEncoding)

      elif not options.verbose:
        sys.stdout.write(".")
        sys.stdout.flush()

    if not options.verbose:
      print

    # Return after fixing: Ignore other jobs
    return






  ######################################################################
  #  GENERATION OF PRETTY PRINTED CODE
  ######################################################################

  if options.prettyPrint:
    print
    print "  GENERATION OF PRETTY PRINTED CODE:"
    print "----------------------------------------------------------------------------"

    if options.verbose:
      print "  * Pretty printing..."
    else:
      print "  * Pretty printing: ",

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - Compiling %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      prettyFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), True)

      if not prettyFileContent.endswith("\n"):
        prettyFileContent += "\n"

      filetool.save(fileDb[fileId]["path"], prettyFileContent)

    if not options.verbose:
      print

    # Return after pretty print: Ignore other jobs
    return



  ######################################################################
  #  STRING OPTIMIZATION
  ######################################################################

  if options.optimizeStrings:
    print
    print "  STRING OPTIMIZATION:"
    print "----------------------------------------------------------------------------"

    if options.verbose:
      print "  * Searching strings..."
    else:
      print "  * Searching strings: ",

    stringMap = {}

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      localMap = loader.getStrings(fileDb, fileId, options)

      for value in localMap:
        if value in stringMap:
          stringMap[value] += localMap[value]
        else:
          stringMap[value] = localMap[value]

    if not options.verbose:
      print

    counter = 0
    for value in stringMap:
      counter += stringMap[value]

    stringList = stringoptimizer.sort(stringMap)

    print "  * Found %s strings (used %s times)" % (len(stringMap), counter)

    if options.verbose:
      print "  * Replacing strings..."
    else:
      print "  * Replacing strings: ",

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      stringoptimizer.replace(loader.getTree(fileDb, fileId, options), stringList, "$" + pkgid, options.verbose)

    if not options.verbose:
      print

    print "  * Generating replacement..."
    additionalOutput.append(stringoptimizer.replacement(stringList, "$" + pkgid))






  ######################################################################
  #  LOCAL VARIABLE OPTIMIZATION
  ######################################################################

  if options.optimizeVariables:
    print
    print "  LOCAL VARIABLE OPTIMIZATION:"
    print "----------------------------------------------------------------------------"

    if options.verbose:
      print "  * Optimizing variables..."
    else:
      print "  * Optimizing variables: ",

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      variableoptimizer.search(loader.getTree(fileDb, fileId, options), [], 0, "$", skipPrefix = options.optimizeVariablesSkipPrefix, debug = options.enableDebug)

    if not options.verbose:
      print






  ######################################################################
  #  NAME OBFUSCATION
  ######################################################################

  if options.obfuscateIdentifiers:
    print
    print "  OBFUSCATE IDENTIFIERS:"
    print "----------------------------------------------------------------------------"

    if options.verbose:
      print "  * Obfuscating identifiers..."
    else:
      print "  * Obfuscating identifiers: ",

    counter = 0

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      counter += obfuscator.update(loader.getTree(fileDb, fileId, options), names, "$$")

    if not options.verbose:
      print

    print "  * Updated %s names" % counter






  ######################################################################
  #  TOKEN STORAGE
  ######################################################################

  if options.storeTokens:
    print
    print "  TOKEN STORAGE:"
    print "----------------------------------------------------------------------------"

    if options.tokenOutputDirectory == None:
      print "  * You must define the token output directory!"
      sys.exit(1)

    if options.verbose:
      print "  * Storing tokens..."
    else:
      print "  * Storing tokens: ",

    for fileId in sortedIncludeList:
      tokenString = tokenizer.convertTokensToString(loader.getTokens(fileDb, fileId, options))

      if options.verbose:
        print "    * writing tokens for %s (%s KB)..." % (fileIdm, len(tokenString) / 1000.0)
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      filetool.save(os.path.join(filetool.normalize(options.tokenOutputDirectory), fileId + config.TOKENEXT), tokenString)

    if not options.verbose:
      print




  ######################################################################
  #  TREE STORAGE
  ######################################################################

  if options.storeTree:
    print
    print "  TREE STORAGE:"
    print "----------------------------------------------------------------------------"

    if options.treeOutputDirectory == None:
      print "  * You must define the tree output directory!"
      sys.exit(1)

    if options.verbose:
      print "  * Storing tree..."
    else:
      print "  * Storing tree: ",

    for fileId in sortedIncludeList:
      treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(loader.getTree(fileDb, fileId, options))

      if options.verbose:
        print "    * writing tree for %s (%s KB)..." % (fileId, len(treeString) / 1000.0)
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      filetool.save(os.path.join(filetool.normalize(options.treeOutputDirectory), fileId + config.XMLEXT), treeString)

    if not options.verbose:
      print





  ######################################################################
  #  GENERATION OF API
  ######################################################################

  if options.generateApiDocumentation:
    print
    print "  GENERATION OF API:"
    print "----------------------------------------------------------------------------"

    if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
      print "  * You must define one of JSON or XML API documentation file!"

    docTree = None

    if options.verbose:
      print "  * Generating API tree..."
    else:
      print "  * Generating API tree: ",

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      docTree = api.createDoc(loader.getTree(fileDb, fileId, options), docTree)

    if not options.verbose:
      print

    if docTree:
      print "  * Finalizing tree..."
      api.postWorkPackage(docTree, docTree)

    if options.apiDocumentationXmlFile != None:
      print "  * Writing XML API file to %s" % options.apiDocumentationXmlFile

      xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"

      if options.addNewLines:
        xmlContent += "\n" + tree.nodeToXmlString(docTree)
      else:
        xmlContent += tree.nodeToXmlString(docTree, "", "", "")

      filetool.save(options.apiDocumentationXmlFile, xmlContent, options.xmlOutputEncoding)

    if options.apiDocumentationJsonFile != None:
      print "  * Writing JSON API file to %s" % options.apiDocumentationJsonFile

      if options.addNewLines:
        jsonContent = tree.nodeToJsonString(docTree)
      else:
        jsonContent = tree.nodeToJsonString(docTree, "", "", "")

      filetool.save(options.apiDocumentationJsonFile, jsonContent, options.scriptOutputEncoding)





  ######################################################################
  #  CREATE COPY OF RESOURCES
  ######################################################################

  if options.copyResources:

    print
    print "  CREATE COPY OF RESOURCES:"
    print "----------------------------------------------------------------------------"

    resources.copy(options, sortedIncludeList, fileDb)






  ######################################################################
  #  GENERATION OF SETTINGS
  ######################################################################

  if options.generateSourceScript or options.generateCompiledScript:
    settingsStr = ""

    if len(options.defineRuntimeSetting) != 0:
      print
      print "  GENERATION OF SETTINGS:"
      print "----------------------------------------------------------------------------"

      print "  * Processing input data..."
      settingsStr = settings.generate(options)

      if options.settingsScriptFile:
        print "   * Storing result to %s" % options.settingsScriptFile
        filetool.save(options.settingsScriptFile, settingsStr)

        # clear settings for build and source
        settingsStr = ""





  ######################################################################
  #  GENERATION OF SOURCE VERSION
  ######################################################################

  if options.generateSourceScript:
    print
    print "  GENERATION OF SOURCE SCRIPT:"
    print "----------------------------------------------------------------------------"

    if options.sourceScriptFile == None and (options.sourceTemplateInputFile == None or options.sourceTemplateOutputFile == None):
      print "  * You must define at least one source script file or template input/output."
      sys.exit(1)

    if options.sourceScriptFile:
      options.sourceScriptFile = os.path.normpath(options.sourceScriptFile)

    if options.sourceTemplateInputFile:
      options.sourceTemplateInputFile = os.path.normpath(options.sourceTemplateInputFile)

    if options.sourceTemplateOutputFile:
      options.sourceTemplateOutputFile = os.path.normpath(options.sourceTemplateOutputFile)


    print "  * Generating script block..."

    # Handling line feed setting
    sourceLineFeed = "";
    if options.addNewLines:
      sourceLineFeed = "\n";


    # Generating inline code...
    inlineCode = ""
    inlineCode += settingsStr + sourceLineFeed
    inlineCode += "qx.IS_SOURCE=true;%s" % sourceLineFeed
    inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, sourceLineFeed)
    inlineCode += "".join(additionalOutput)


    # Generating script block
    scriptBlocks = ""
    scriptBlocks += '<script type="text/javascript">%s</script>' % inlineCode
    for fileId in sortedIncludeList:
      if fileDb[fileId]["classUri"] == None:
        print "  * Missing class URI definition for class path %s." % fileDb[fileId]["classPath"]
        sys.exit(1)

      scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (os.path.join(fileDb[fileId]["classUri"], fileDb[fileId]["pathId"].replace(".", os.sep)), config.JSEXT)
      scriptBlocks += sourceLineFeed



    if options.sourceScriptFile != None:
      print "  * Storing includer as %s..." % options.sourceScriptFile
      sourceScript = "document.write('%s');" % scriptBlocks.replace("'", "\\'")
      if options.addNewLines:
        sourceScript = sourceScript.replace("\n", "\\\n")
      filetool.save(options.sourceScriptFile, sourceScript, options.scriptOutputEncoding)

    if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
      print "  * Patching template: %s => %s" % (options.sourceTemplateInputFile, options.sourceTemplateOutputFile)
      tmpl = filetool.read(options.sourceTemplateInputFile)
      res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
      filetool.save(options.sourceTemplateOutputFile, res, options.scriptOutputEncoding)





  ######################################################################
  #  GENERATION OF COMPILED VERSION
  ######################################################################

  if options.generateCompiledScript:
    print
    print "  GENERATION OF COMPILED SCRIPT:"
    print "----------------------------------------------------------------------------"

    buildLineFeed = "";
    if options.addNewLines:
      buildLineFeed = "\n";

    inlineCode = ""
    inlineCode += settingsStr + buildLineFeed
    inlineCode += "qx.IS_SOURCE=false;%s" % buildLineFeed
    inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, buildLineFeed)
    inlineCode += "".join(additionalOutput)

    compiledOutput = inlineCode

    if options.compiledScriptFile == None:
      print "  * You must define the compiled script file!"
      sys.exit(1)

    if options.verbose:
      print "  * Compiling..."
    else:
      print "  * Compiling: ",

    for fileId in sortedIncludeList:
      if options.verbose:
        print "    - Compiling %s" % fileId
      else:
        sys.stdout.write(".")
        sys.stdout.flush()

      compiledFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), False, options.addNewLines, options.enableDebug)

      if options.addFileIds:
        compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
      else:
        compiledOutput += compiledFileContent

      if not compiledOutput.endswith(";") and not compiledOutput.endswith("\n"):
        compiledOutput += ";"

    if not options.verbose:
      print

    print "  * Storing output as %s..." % options.compiledScriptFile
    filetool.save(options.compiledScriptFile, compiledOutput, options.scriptOutputEncoding)
Пример #11
0
def execute(fileDb, moduleDb, options, pkgid="", names=[]):


    ######################################################################
    #  SORT OF INCLUDE LIST
    ######################################################################

    print
    print "  GENERATE CLASS INCLUDE LIST:"
    print "----------------------------------------------------------------------------"

    sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)

    if options.printIncludes:
        print
        print "  PRINT OF INCLUDE ORDER:"
        print "----------------------------------------------------------------------------"
        print "  * The files will be included in this order:"
        for fileId in sortedIncludeList:
            print "    - %s" % fileId


    if options.printIncludesFile:
        includeFile = open(options.printIncludesFile, "w")
        for fileId in sortedIncludeList:
            includeFile.write(fileId + "\n")
        includeFile.close()


    if options.printDependencies:
        print
        print "  OUTPUT OF DEPENDENCIES:"
        print "----------------------------------------------------------------------------"
        print "  * These are all included files with their dependencies:"
        for fileId in sortedIncludeList:
            print "    - %s" % fileId
            if len(fileDb[fileId]["loadtimeDeps"]) > 0:
                print "      - Loadtime: "
                for depEntry in fileDb[fileId]["loadtimeDeps"]:
                    print "        - %s" % depEntry

            if len(fileDb[fileId]["runtimeDeps"]) > 0:
                print "      - Runtime: "
                for depEntry in fileDb[fileId]["runtimeDeps"]:
                    print "        - %s" % depEntry

            if len(fileDb[fileId]["optionalDeps"]) > 0:
                print "      - Optional: "
                for depEntry in fileDb[fileId]["optionalDeps"]:
                    print "        - %s" % depEntry





    ######################################################################
    #  GRAPHVIZ OUTPUT
    ######################################################################

    if options.depDotFile:
        graph.store(fileDb, sortedIncludeList, options)






    ######################################################################
    #  SOURCE MIGRATION
    ######################################################################

    if options.migrateSource:
        print
        print "  SOURCE MIGRATION:"
        print "----------------------------------------------------------------------------"

        print "  * Migrate Source Code..."

        migrator.handle(fileDb, options, options.migrationTarget,
                        options.migrationInput, options.verbose)

        # Return after migration: Ignore other jobs
        return





    ######################################################################
    #  GENERATION OF FIXED CODE
    ######################################################################

    if options.fixSource:
        print
        print "  FIX SOURCE CODE:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Fixing code..."
        else:
            print "  * Fixing code: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - Reading %s" % fileId

            fileEntry = fileDb[fileId]

            filePath = fileEntry["path"]
            fileEncoding = fileEntry["encoding"]

            fileContent = filetool.read(filePath, fileEncoding)
            fixedContent = textutil.removeTrailingSpaces(textutil.tab2Space(textutil.any2Unix(fileContent), 2))

            if fixedContent != fileContent:
                if options.verbose:
                    print "      - Storing modifications..."
                else:
                    sys.stdout.write("!")
                    sys.stdout.flush()

                filetool.save(filePath, fixedContent, fileEncoding)

            elif not options.verbose:
                sys.stdout.write(".")
                sys.stdout.flush()

        if not options.verbose:
            print

        # Return after fixing: Ignore other jobs
        return






    ######################################################################
    #  GENERATION OF PRETTY PRINTED CODE
    ######################################################################

    if options.prettyPrint:
        print
        print "  GENERATION OF PRETTY PRINTED CODE:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Pretty printing..."
        else:
            print "  * Pretty printing: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - Compiling %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            prettyFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), options)

            if not prettyFileContent.endswith("\n"):
                prettyFileContent += "\n"

            filetool.save(fileDb[fileId]["path"], prettyFileContent)

        if not options.verbose:
            print

        # Return after pretty print: Ignore other jobs
        return





    ######################################################################
    #  INLINE CODE
    ######################################################################

    inlineSourceCode = []
    inlineCompiledCode = []







    ######################################################################
    #  SUPPORT FOR VARIANTS
    ######################################################################

    if len(options.useVariant) > 0 and not options.generateSourceScript:
        print
        print "  VARIANT OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        variantMap = {}
        for variant in options.useVariant:
            keyValue = variant.split(":")
            if len(keyValue) != 2:
                print "  * Error: Variants must be specified as key value pair separated by ':'!"
                sys.exit(1)

            variantMap[keyValue[0]] = keyValue[1]

        if options.verbose:
            print "  * Optimizing for variant setup..."
        else:
            print "  * Optimizing for variant setup: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            if variantoptimizer.search(loader.getTree(fileDb, fileId, options), variantMap, fileId, options.verbose):
                if options.verbose:
                    print "      - Modified!"

        if not options.verbose:
            print









    ######################################################################
    #  BASE CALL OPTIMIZATION
    ######################################################################

    if options.optimizeBaseCall:
        print
        print "  BASE CALL OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Optimizing this.base calls..."
        else:
            print "  * Optimizing this.base calls: ",

        counter = 0

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            counter += basecalloptimizer.patch(loader.getTree(fileDb, fileId, options))

        if not options.verbose:
            print

        print "  * Optimized %s calls" % counter





    ######################################################################
    #  ACCESSOR OBFUSCATION
    ######################################################################

    if options.obfuscateAccessors:
        print
        print "  ACCESSOR OBFUSCATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Obfuscating..."
        else:
            print "  * Obfuscating: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            accessorobfuscator.process(loader.getTree(fileDb, fileId, options), options.verbose)

        if not options.verbose:
            print




    ######################################################################
    #  STRING OPTIMIZATION
    ######################################################################

    if options.optimizeStrings:
        print
        print "  STRING OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Searching strings..."
        else:
            print "  * Searching strings: ",

        stringMap = {}

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            localMap = stringoptimizer.search(loader.getTree(fileDb, fileId, options), options.verbose)

            for value in localMap:
                if value in stringMap:
                    stringMap[value] += localMap[value]
                else:
                    stringMap[value] = localMap[value]

        if not options.verbose:
            print

        counter = 0
        for value in stringMap:
            counter += stringMap[value]

        stringList = stringoptimizer.sort(stringMap)

        print "  * Found %s string instances (%s unique)" % (counter, len(stringMap))

        if options.verbose:
            print "  * Replacing strings..."
        else:
            print "  * Replacing strings: ",

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            stringoptimizer.replace(loader.getTree(fileDb, fileId, options), stringList, "$" + pkgid, options.verbose)

        if not options.verbose:
            print

        print "  * Generating replacement..."
        inlineCompiledCode.append(stringoptimizer.replacement(stringList, "$" + pkgid))









    ######################################################################
    #  LOCAL VARIABLE OPTIMIZATION
    ######################################################################

    if options.optimizeVariables or options.obfuscate:
        print
        print "  LOCAL VARIABLE OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Optimizing variables..."
        else:
            print "  * Optimizing variables: ",

        counter = 0

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            counter += variableoptimizer.search(
                loader.getTree(fileDb, fileId, options),
                [], 0, 0, "$",
                skipPrefix = options.optimizeVariablesSkipPrefix,
                verbose = options.verbose
            )

        if not options.verbose:
            print

        print "  * Optimized %s variables" % counter





    ######################################################################
    #  PRIVATE MEMBER OPTIMIZATION
    ######################################################################

    if options.optimizePrivate:
        print
        print "  PRIVATE MEMBER OPTIMIZATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Optimizing private members..."
        else:
            print "  * Optimizing private members: ",

        counter = 0

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            counter += privateoptimizer.patch("",loader.getTree(fileDb, fileId, options), {}, "$", options.verbose)

        if not options.verbose:
            print

        print "  * Optimized %s keys" % counter





    ######################################################################
    #  OBFUSCATION
    ######################################################################

    if options.obfuscate:
        print
        print "  OBFUSCATION:"
        print "----------------------------------------------------------------------------"

        if options.verbose:
            print "  * Searching for assignments..."
        else:
            print "  * Searching for assignments: ",

        identifiers = {}

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            obfuscator.search(loader.getTree(fileDb, fileId, options), identifiers, options.verbose)

        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            obfuscator.patch(loader.getTree(fileDb, fileId, options), identifiers, options.verbose)

        if not options.verbose:
            print





    ######################################################################
    #  TOKEN STORAGE
    ######################################################################

    if options.storeTokens:
        print
        print "  TOKEN STORAGE:"
        print "----------------------------------------------------------------------------"

        if options.tokenOutputDirectory == None:
            print "  * You must define the token output directory!"
            sys.exit(1)

        if options.verbose:
            print "  * Storing tokens..."
        else:
            print "  * Storing tokens: ",

        for fileId in sortedIncludeList:
            tokenString = tokenizer.convertTokensToString(loader.getTokens(fileDb, fileId, options))

            if options.verbose:
                print "    * Writing tokens for %s (%s KB)..." % (fileIdm, len(tokenString) / 1000.0)
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            filetool.save(os.path.join(filetool.normalize(options.tokenOutputDirectory), fileId + config.TOKENEXT), tokenString)

        if not options.verbose:
            print







    ######################################################################
    #  TREE STORAGE
    ######################################################################

    if options.storeTree:
        print
        print "  TREE STORAGE:"
        print "----------------------------------------------------------------------------"

        if options.treeOutputDirectory == None:
            print "  * You must define the tree output directory!"
            sys.exit(1)

        if options.verbose:
            print "  * Storing tree..."
        else:
            print "  * Storing tree: ",

        for fileId in sortedIncludeList:
            treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(loader.getTree(fileDb, fileId, options))

            if options.verbose:
                print "    * Writing tree for %s (%s KB)..." % (fileId, len(treeString) / 1000.0)
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            filetool.save(os.path.join(filetool.normalize(options.treeOutputDirectory), fileId + config.XMLEXT), treeString)

        if not options.verbose:
            print








    ######################################################################
    #  GENERATION OF API
    ######################################################################

    if options.generateApiDocumentation:
        print
        print "  GENERATION OF API:"
        print "----------------------------------------------------------------------------"

        if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
            print "  * You must define one of JSON or XML API documentation file!"

        docTree = None

        if options.verbose:
            print "  * Generating API tree..."
        else:
            print "  * Generating API tree: ",

        hasDocError = False
        for fileId in sortedIncludeList:
            if options.verbose:
                print "    - %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()


            (docTree, error) = api.createDoc(loader.getTree(fileDb, fileId, options), docTree)
            hasDocError = hasDocError or error

        if hasDocError:
            print
            print "  * Building API failed!!"
            sys.exit(0)

        if not options.verbose:
            print

        if docTree:
            print "  * Finalizing tree..."
            api.postWorkPackage(docTree, docTree)

        if options.addNewLines:
            childPrefix = "  "
            newLine = "\n"
        else:
            childPrefix = ""
            newLine = ""


        if options.apiDocumentationXmlFile != None:
            print "  * Writing XML API file to %s" % options.apiDocumentationXmlFile

            if options.apiSeparateFiles:
                packages = api.packagesToXmlString(docTree, "", childPrefix, newLine)
                filetool.save(options.apiDocumentationXmlFile, packages, options.scriptOutputEncoding)

                for cls in api.classNodeIterator(docTree):
                    classContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
                    classContent += tree.nodeToXmlString(cls, "", childPrefix, newLine)
                    fileName = os.path.join(os.path.dirname(options.apiDocumentationXmlFile), cls.get("fullName") + ".xml")
                    filetool.save(fileName, classContent, options.xmlOutputEncoding)

            else:
                xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
                xmlContent += tree.nodeToXmlString(docTree, "", childPrefix, newLine)
                filetool.save(options.apiDocumentationXmlFile, xmlContent, options.xmlOutputEncoding)



        if options.apiDocumentationJsonFile != None:
            print "  * Writing JSON API file to %s" % options.apiDocumentationJsonFile

            if options.apiSeparateFiles:
                packages = api.packagesToJsonString(docTree, "", childPrefix, newLine)
                filetool.save(options.apiDocumentationJsonFile, packages, options.scriptOutputEncoding)

                for cls in api.classNodeIterator(docTree):
                    classContent = tree.nodeToJsonString(cls, "", childPrefix, newLine)
                    fileName = os.path.join(os.path.dirname(options.apiDocumentationJsonFile), cls.get("fullName") + ".js")
                    filetool.save(fileName, classContent, options.scriptOutputEncoding)

            else:
                jsonContent = tree.nodeToJsonString(docTree, "", childPrefix, newLine)
                filetool.save(options.apiDocumentationJsonFile, jsonContent, options.scriptOutputEncoding)

        # create search index
        if options.apiDocumentationIndexFile != None:
            print "  * Generating API index..."
            jsonContent = tree.nodeToIndexString(docTree, "", childPrefix, newLine)
            print "  * Writing API search index to %s" % options.apiDocumentationIndexFile
            filetool.save(options.apiDocumentationIndexFile, jsonContent, options.scriptOutputEncoding)






    ######################################################################
    #  CREATE COPY OF RESOURCES
    ######################################################################

    if options.copyResources:

        print
        print "  CREATE COPY OF RESOURCES:"
        print "----------------------------------------------------------------------------"

        resources.copy(options, sortedIncludeList, fileDb)






    ######################################################################
    #  GENERATION OF SETTINGS
    ######################################################################

    if options.generateSourceScript or options.generateCompiledScript:
        print
        print "  INCLUSION OF SETTINGS:"
        print "----------------------------------------------------------------------------"

        if options.generateSourceScript:
            print "  * Processing settings of source version..."

            settingsArr = options.useSetting[:]
            settingsArr.append("qx.version:%s" % options.version)
            settingsArr.append("qx.isSource:true")
            inlineSourceCode.append(settings.generate(settingsArr, options.addNewLines))

        if options.generateCompiledScript:
            print "  * Processing settings of compiled version..."

            settingsArr = options.useSetting[:]
            settingsArr.append("qx.version:%s" % options.version)
            settingsArr.append("qx.isSource:false")
            inlineCompiledCode.append(settings.generate(settingsArr, options.addNewLines))






    ######################################################################
    #  GENERATION OF VARIANTS
    ######################################################################

    if options.generateSourceScript or options.generateCompiledScript:
        print
        print "  INCLUSION OF VARIANTS:"
        print "----------------------------------------------------------------------------"

        if options.generateSourceScript:
            print "  * Processing variants of source version..."

            variantsArr = options.useVariant[:]
            inlineSourceCode.append(variants.generate(variantsArr, options.addNewLines))

        if options.generateCompiledScript:
            print "  * Processing variants of compiled version..."

            variantsArr = options.useVariant[:]
            inlineCompiledCode.append(variants.generate(variantsArr, options.addNewLines))







    ######################################################################
    #  GENERATION OF SOURCE VERSION
    ######################################################################

    if options.generateSourceScript:
        print
        print "  GENERATION OF SOURCE SCRIPT:"
        print "----------------------------------------------------------------------------"

        if options.sourceScriptFile == None and (options.sourceTemplateInputFile == None or options.sourceTemplateOutputFile == None):
            print "  * You must define at least one source script file or template input/output."
            sys.exit(1)

        if options.sourceScriptFile:
            options.sourceScriptFile = os.path.normpath(options.sourceScriptFile)

        if options.sourceTemplateInputFile:
            options.sourceTemplateInputFile = os.path.normpath(options.sourceTemplateInputFile)

        if options.sourceTemplateOutputFile:
            options.sourceTemplateOutputFile = os.path.normpath(options.sourceTemplateOutputFile)


        print "  * Generating script block..."

        # Handling line feed setting
        sourceLineFeed = "";
        if options.addNewLines:
            sourceLineFeed = "\n";

        # Generating script block
        scriptBlocks = ""
        scriptBlocks += '<script type="text/javascript">%s</script>' % "".join(inlineSourceCode)
        for fileId in sortedIncludeList:
            if fileDb[fileId]["meta"]:
              continue

            if fileDb[fileId]["classUri"] == None:
                print "  * Missing class URI definition for class path %s." % fileDb[fileId]["classPath"]
                sys.exit(1)

            scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (os.path.join(fileDb[fileId]["classUri"], fileDb[fileId]["pathId"].replace(".", '/')), config.JSEXT)
            scriptBlocks += sourceLineFeed

        # Writing includer
        if options.sourceScriptFile != None:
            print "  * Storing includer as %s..." % options.sourceScriptFile
            sourceScript = "document.write('%s');" % scriptBlocks.replace("'", "\\'")
            if options.addNewLines:
                sourceScript = sourceScript.replace("\n", "\\\n")
            filetool.save(options.sourceScriptFile, sourceScript, options.scriptOutputEncoding)

        # Patching template
        if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
            print "  * Patching template: %s => %s" % (options.sourceTemplateInputFile, options.sourceTemplateOutputFile)
            tmpl = filetool.read(options.sourceTemplateInputFile)
            res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
            filetool.save(options.sourceTemplateOutputFile, res, options.scriptOutputEncoding)





    ######################################################################
    #  GENERATION OF COMPILED VERSION
    ######################################################################

    if options.generateCompiledScript:
        print
        print "  GENERATION OF COMPILED SCRIPT:"
        print "----------------------------------------------------------------------------"

        buildLineFeed = "";
        if options.addNewLines:
            buildLineFeed = "\n";

        compiledOutput = "".join(inlineCompiledCode)

        if options.compiledScriptFile == None:
            print "  * You must define the compiled script file!"
            sys.exit(1)

        if options.verbose:
            print "  * Compiling..."
        else:
            print "  * Compiling: ",

        for fileId in sortedIncludeList:
            if fileDb[fileId]["meta"]:
              continue

            if options.verbose:
                print "    - Compiling %s" % fileId
            else:
                sys.stdout.write(".")
                sys.stdout.flush()

            if options.prettyPrint:
                options.prettyPrint = False  # make sure it's disabled

            compiledFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), options, options.addNewLines, options.verbose)

            if options.addFileIds:
                compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
            else:
                compiledOutput += compiledFileContent

            if not compiledOutput.endswith(";") and not compiledOutput.endswith("\n"):
                compiledOutput += ";"

        if not options.verbose:
            print

        print "  * Storing output as %s..." % options.compiledScriptFile
        filetool.save(options.compiledScriptFile, compiledOutput, options.scriptOutputEncoding)
Пример #12
0
def handle(fileList, fileDb, options):
  confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget)

  infoPath = os.path.join(confPath, "info")
  patchPath = os.path.join(confPath, "patches")

  importedModule = False
  infoList = []
  patchList = []
  htmlList = getHtmlList(options)




  print "  * Number of script input files: %s" % len(fileList)
  print "  * Number of HTML input files: %s" % len(htmlList)
  print "  * Update to version: %s" % options.migrationTarget



  print "  * Searching for patch module..."

  for root, dirs, files in os.walk(confPath):

    # Filter ignored directories
    for ignoredDir in config.DIRIGNORE:
      if ignoredDir in dirs:
        dirs.remove(ignoredDir)

    # Searching for files
    for fileName in files:
      filePath = os.path.join(root, fileName)

      if os.path.splitext(fileName)[1] != config.PYEXT:
        continue

      if fileName == "patch.py":
        print "    - Importing..."

        if not root in sys.path:
          sys.path.insert(0, root)

        import patch
        importedModule = True







  emptyLine = re.compile("^\s*$")



  print "  * Searching for info expression data..."

  for root, dirs, files in os.walk(infoPath):

    # Filter ignored directories
    for ignoredDir in config.DIRIGNORE:
      if ignoredDir in dirs:
        dirs.remove(ignoredDir)

    # Searching for files
    for fileName in files:
      filePath = os.path.join(root, fileName)

      fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
      infoList.append({"path":filePath, "content":fileContent.split("\n")})

      if options.verbose:
        print "    - %s" % filePath

  print "    - Number of info files: %s" % len(infoList)

  print "    - Compiling expressions..."

  compiledInfos = []

  for infoFile in infoList:
    print "      - %s" % os.path.basename(infoFile["path"])
    for line in infoFile["content"]:
      if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
        continue

      compiled = entryCompiler(line)
      if compiled != None:
        compiledInfos.append(compiled)

  print "    - Number of infos: %s" % len(compiledInfos)




  print "  * Searching for patch expression data..."

  for root, dirs, files in os.walk(patchPath):

    # Filter ignored directories
    for ignoredDir in config.DIRIGNORE:
      if ignoredDir in dirs:
        dirs.remove(ignoredDir)

    # Searching for files
    for fileName in files:
      filePath = os.path.join(root, fileName)

      fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
      patchList.append({"path":filePath, "content":fileContent.split("\n")})

      if options.verbose:
        print "    - %s" % filePath

  print "    - Number of patch files: %s" % len(patchList)

  print "    - Compiling expressions..."

  compiledPatches = []

  for patchFile in patchList:
    print "      - %s" % os.path.basename(patchFile["path"])
    for line in patchFile["content"]:
      if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
        continue

      compiled = entryCompiler(line)
      if compiled != None:
        compiledPatches.append(compiled)

  print "    - Number of patches: %s" % len(compiledPatches)








  print
  print "  FILE PROCESSING:"
  print "----------------------------------------------------------------------------"

  if len(fileList) > 0:
    print "  * Processing script files:"

    for fileId in fileList:
      fileEntry = fileDb[fileId]

      filePath = fileEntry["path"]
      fileEncoding = fileEntry["encoding"]

      print "    - %s" % fileId

      # Read in original content
      fileContent = filetool.read(filePath, fileEncoding)
      patchedContent = fileContent

      # Apply patches
      if importedModule:
        tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent))

        # If there were any changes, compile the result
        if patch.patch(fileId, tree):
          patchedContent = compiler.compile(tree, True)

      patchedContent = regtool(patchedContent, compiledPatches, True, options)
      patchedContent = regtool(patchedContent, compiledInfos, False, options)

      # Write file
      if patchedContent != fileContent:
        print "      - Store modifications..."
        filetool.save(filePath, patchedContent, fileEncoding)

    print "  * Done"



  if len(htmlList) > 0:
    print "  * Processing HTML files:"

    for filePath in htmlList:
      print "    - %s" % filePath

      # Read in original content
      fileContent = filetool.read(filePath)

      patchedContent = fileContent
      patchedContent = regtool(patchedContent, compiledPatches, True, options)
      patchedContent = regtool(patchedContent, compiledInfos, False, options)

      # Write file
      if patchedContent != fileContent:
        print "      - Store modifications..."
        filetool.save(filePath, patchedContent)

    print "  * Done"