def gunzip(filePath, encoding="utf-8"):
if not filePath.endswith(".gz"):
filePath = filePath + ".gz"
inputFile = sys_gzip.open(filePath, "rb")
content = inputFile.read()
return textutil.any2Unix(unicode(content))
def gunzip(filePath, encoding="utf-8"):
if not filePath.endswith(".gz"):
filePath = filePath + ".gz"
inputFile = sys_gzip.open(filePath, "rb")
content = inputFile.read()
return textutil.any2Unix(unicode(content))
def read(filePath, encoding="utf_8"):
try:
ref = codecs.open(filePath, encoding=encoding, mode="r")
content = ref.read()
ref.close()
return textutil.any2Unix(unicode(content))
except IOError, (errno, strerror):
print " * I/O error(%s): %s" % (errno, strerror)
sys.exit(1)
def read(filePath, encoding="utf_8"):
try:
ref = codecs.open(filePath, encoding=encoding, mode="r")
content = ref.read()
ref.close()
return textutil.any2Unix(unicode(content))
except IOError, (errno, strerror):
print " * I/O error(%s): %s (%s)" % (errno, strerror, filePath)
sys.exit(1)
def readPatchInfoFiles(baseDir):
"""
Reads all patch/info files from a directory and compiles the containing
regular expressions.
Retuns a list comiled RE (the output of entryCompiler)
"""
patchList = []
emptyLine = re.compile("^\s*$")
for root, dirs, files in os.walk(baseDir):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
patchList.append({
"path": filePath,
"content": fileContent.split("\n")
})
logging.debug(" - %s" % filePath)
logging.debug(" - Compiling expressions...")
compiledPatches = []
for patchFile in patchList:
logging.debug(" - %s" % os.path.basename(patchFile["path"]))
for line in patchFile["content"]:
if emptyLine.match(line) or line.startswith(
"#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledPatches.append(compiled)
return compiledPatches
def read(filePath, encoding="utf_8"):
def getRowCol(text, pos):
row = len(text[:pos].split("\n"))
col = pos - text[:pos].rfind("\n")
if col < 0:
col = pos
return (row, col)
try:
ref = codecs.open(filePath, encoding=encoding, mode="r")
try:
content = ref.read()
except UnicodeDecodeError, e:
rowCol = getRowCol(e.object, e.start)
e.reason = e.reason + "\nFile %s Line %s Column %s" % (str(filePath), str(rowCol[0]), str(rowCol[1]))
raise e
finally:
ref.close()
return textutil.any2Unix(unicode(content))
def read(filePath, encoding="utf_8"):
def getRowCol(text, pos):
row = len(text[:pos].split("\n"))
col = pos - text[:pos].rfind("\n")
if col < 0:
col = pos
return (row, col)
try:
ref = codecs.open(filePath, encoding=encoding, mode="r")
try:
content = ref.read()
except UnicodeDecodeError, e:
rowCol = getRowCol(e.object, e.start)
e.reason = e.reason + "\nFile %s Line %s Column %s" % (
str(filePath), str(rowCol[0]), str(rowCol[1]))
raise e
finally:
ref.close()
return textutil.any2Unix(unicode(content))
def readPatchInfoFiles(baseDir):
"""
Reads all patch/info files from a directory and compiles the containing
regular expressions.
Retuns a list comiled RE (the output of entryCompiler)
"""
patchList = []
emptyLine = re.compile("^\s*$")
for root, dirs, files in os.walk(baseDir):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
patchList.append({"path":filePath, "content":fileContent.split("\n")})
logging.debug(" - %s" % filePath)
logging.debug(" - Compiling expressions...")
compiledPatches = []
for patchFile in patchList:
logging.debug(" - %s" % os.path.basename(patchFile["path"]))
for line in patchFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledPatches.append(compiled)
return compiledPatches
def execute(fileDb, moduleDb, options, pkgid="", names=[]):
additionalOutput = []
######################################################################
# SORT OF INCLUDE LIST
######################################################################
print
print " SORT OF INCLUDE LIST:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Include (with dependencies): %s" % options.includeWithDeps
print " * Include (without dependencies): %s" % options.includeWithoutDeps
print " * Exclude (with dependencies): %s" % options.excludeWithDeps
print " * Exclude (without dependencies): %s" % options.excludeWithoutDeps
print " * Sorting %s classes..." % len(fileDb)
sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)
if len(sortedIncludeList) == 0:
print " - No class files to include. Exciting!"
sys.exit(1)
else:
print " - Including %s classes" % len(sortedIncludeList)
if options.printIncludes:
print
print " PRINT OF INCLUDE ORDER:"
print "----------------------------------------------------------------------------"
print " * The files will be included in this order:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if options.printDeps:
print
print " OUTPUT OF DEPENDENCIES:"
print "----------------------------------------------------------------------------"
print " * These are all included files with their dependencies:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
print " - Loadtime: "
for depEntry in fileDb[fileId]["loadtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["afterDeps"]) > 0:
print " - After: "
for depEntry in fileDb[fileId]["afterDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["runtimeDeps"]) > 0:
print " - Runtime: "
for depEntry in fileDb[fileId]["runtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["loadDeps"]) > 0:
print " - Load: "
for depEntry in fileDb[fileId]["loadDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["optionalDeps"]) > 0:
print " - Optional: "
for depEntry in fileDb[fileId]["optionalDeps"]:
print " - %s" % depEntry
######################################################################
# GRAPHVIZ OUTPUT
######################################################################
if options.depDotFile:
graph.store(fileDb, sortedIncludeList, options)
######################################################################
# SOURCE MIGRATION
######################################################################
if options.migrateSource:
print
print " SOURCE MIGRATION:"
print "----------------------------------------------------------------------------"
print " * Migrate Source Code..."
migrator.handle(sortedIncludeList, fileDb, options)
# Return after migration: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.fixSource:
print
print " FIX SOURCE CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Fixing code..."
else:
print " * Fixing code: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Reading %s" % fileId
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
fileContent = filetool.read(filePath, fileEncoding)
fixedContent = textutil.removeTrailingSpaces(
textutil.tab2Space(textutil.any2Unix(fileContent), 2))
if fixedContent != fileContent:
if options.verbose:
print " - Storing modifications..."
else:
sys.stdout.write("!")
sys.stdout.flush()
filetool.save(filePath, fixedContent, fileEncoding)
elif not options.verbose:
sys.stdout.write(".")
sys.stdout.flush()
if not options.verbose:
print
# Return after fixing: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.prettyPrint:
print
print " GENERATION OF PRETTY PRINTED CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Pretty printing..."
else:
print " * Pretty printing: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
prettyFileContent = compiler.compile(
loader.getTree(fileDb, fileId, options), True)
if not prettyFileContent.endswith("\n"):
prettyFileContent += "\n"
filetool.save(fileDb[fileId]["path"], prettyFileContent)
if not options.verbose:
print
# Return after pretty print: Ignore other jobs
return
######################################################################
# STRING OPTIMIZATION
######################################################################
if options.optimizeStrings:
print
print " STRING OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching strings..."
else:
print " * Searching strings: ",
stringMap = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
localMap = loader.getStrings(fileDb, fileId, options)
for value in localMap:
if value in stringMap:
stringMap[value] += localMap[value]
else:
stringMap[value] = localMap[value]
if not options.verbose:
print
counter = 0
for value in stringMap:
counter += stringMap[value]
stringList = stringoptimizer.sort(stringMap)
print " * Found %s strings (used %s times)" % (len(stringMap),
counter)
if options.verbose:
print " * Replacing strings..."
else:
print " * Replacing strings: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
stringoptimizer.replace(loader.getTree(fileDb, fileId, options),
stringList, "$" + pkgid, options.verbose)
if not options.verbose:
print
print " * Generating replacement..."
additionalOutput.append(
stringoptimizer.replacement(stringList, "$" + pkgid))
######################################################################
# LOCAL VARIABLE OPTIMIZATION
######################################################################
if options.optimizeVariables:
print
print " LOCAL VARIABLE OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing variables..."
else:
print " * Optimizing variables: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
variableoptimizer.search(
loader.getTree(fileDb, fileId, options), [],
0,
"$",
skipPrefix=options.optimizeVariablesSkipPrefix,
debug=options.enableDebug)
if not options.verbose:
print
######################################################################
# NAME OBFUSCATION
######################################################################
if options.obfuscateIdentifiers:
print
print " OBFUSCATE IDENTIFIERS:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Obfuscating identifiers..."
else:
print " * Obfuscating identifiers: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += obfuscator.update(
loader.getTree(fileDb, fileId, options), names, "$$")
if not options.verbose:
print
print " * Updated %s names" % counter
######################################################################
# TOKEN STORAGE
######################################################################
if options.storeTokens:
print
print " TOKEN STORAGE:"
print "----------------------------------------------------------------------------"
if options.tokenOutputDirectory == None:
print " * You must define the token output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tokens..."
else:
print " * Storing tokens: ",
for fileId in sortedIncludeList:
tokenString = tokenizer.convertTokensToString(
loader.getTokens(fileDb, fileId, options))
if options.verbose:
print " * writing tokens for %s (%s KB)..." % (
fileIdm, len(tokenString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(
os.path.join(filetool.normalize(options.tokenOutputDirectory),
fileId + config.TOKENEXT), tokenString)
if not options.verbose:
print
######################################################################
# TREE STORAGE
######################################################################
if options.storeTree:
print
print " TREE STORAGE:"
print "----------------------------------------------------------------------------"
if options.treeOutputDirectory == None:
print " * You must define the tree output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tree..."
else:
print " * Storing tree: ",
for fileId in sortedIncludeList:
treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(
loader.getTree(fileDb, fileId, options))
if options.verbose:
print " * writing tree for %s (%s KB)..." % (
fileId, len(treeString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(
os.path.join(filetool.normalize(options.treeOutputDirectory),
fileId + config.XMLEXT), treeString)
if not options.verbose:
print
######################################################################
# GENERATION OF API
######################################################################
if options.generateApiDocumentation:
print
print " GENERATION OF API:"
print "----------------------------------------------------------------------------"
if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
print " * You must define one of JSON or XML API documentation file!"
docTree = None
if options.verbose:
print " * Generating API tree..."
else:
print " * Generating API tree: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
docTree = api.createDoc(loader.getTree(fileDb, fileId, options),
docTree)
if not options.verbose:
print
if docTree:
print " * Finalizing tree..."
api.postWorkPackage(docTree, docTree)
if options.apiDocumentationXmlFile != None:
print " * Writing XML API file to %s" % options.apiDocumentationXmlFile
xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
if options.addNewLines:
xmlContent += "\n" + tree.nodeToXmlString(docTree)
else:
xmlContent += tree.nodeToXmlString(docTree, "", "", "")
filetool.save(options.apiDocumentationXmlFile, xmlContent,
options.xmlOutputEncoding)
if options.apiDocumentationJsonFile != None:
print " * Writing JSON API file to %s" % options.apiDocumentationJsonFile
if options.addNewLines:
jsonContent = tree.nodeToJsonString(docTree)
else:
jsonContent = tree.nodeToJsonString(docTree, "", "", "")
filetool.save(options.apiDocumentationJsonFile, jsonContent,
options.scriptOutputEncoding)
######################################################################
# CREATE COPY OF RESOURCES
######################################################################
if options.copyResources:
print
print " CREATE COPY OF RESOURCES:"
print "----------------------------------------------------------------------------"
resources.copy(options, sortedIncludeList, fileDb)
######################################################################
# GENERATION OF SETTINGS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
settingsStr = ""
if len(options.defineRuntimeSetting) != 0:
print
print " GENERATION OF SETTINGS:"
print "----------------------------------------------------------------------------"
print " * Processing input data..."
settingsStr = settings.generate(options)
if options.settingsScriptFile:
print " * Storing result to %s" % options.settingsScriptFile
filetool.save(options.settingsScriptFile, settingsStr)
# clear settings for build and source
settingsStr = ""
######################################################################
# GENERATION OF SOURCE VERSION
######################################################################
if options.generateSourceScript:
print
print " GENERATION OF SOURCE SCRIPT:"
print "----------------------------------------------------------------------------"
if options.sourceScriptFile == None and (
options.sourceTemplateInputFile == None
or options.sourceTemplateOutputFile == None):
print " * You must define at least one source script file or template input/output."
sys.exit(1)
if options.sourceScriptFile:
options.sourceScriptFile = os.path.normpath(
options.sourceScriptFile)
if options.sourceTemplateInputFile:
options.sourceTemplateInputFile = os.path.normpath(
options.sourceTemplateInputFile)
if options.sourceTemplateOutputFile:
options.sourceTemplateOutputFile = os.path.normpath(
options.sourceTemplateOutputFile)
print " * Generating script block..."
# Handling line feed setting
sourceLineFeed = ""
if options.addNewLines:
sourceLineFeed = "\n"
# Generating inline code...
inlineCode = ""
inlineCode += settingsStr + sourceLineFeed
inlineCode += "qx.IS_SOURCE=true;%s" % sourceLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version,
sourceLineFeed)
inlineCode += "".join(additionalOutput)
# Generating script block
scriptBlocks = ""
scriptBlocks += '<script type="text/javascript">%s</script>' % inlineCode
for fileId in sortedIncludeList:
if fileDb[fileId]["classUri"] == None:
print " * Missing class URI definition for class path %s." % fileDb[
fileId]["classPath"]
sys.exit(1)
scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (
os.path.join(fileDb[fileId]["classUri"],
fileDb[fileId]["pathId"].replace(
".", os.sep)), config.JSEXT)
scriptBlocks += sourceLineFeed
if options.sourceScriptFile != None:
print " * Storing includer as %s..." % options.sourceScriptFile
sourceScript = "document.write('%s');" % scriptBlocks.replace(
"'", "\\'")
if options.addNewLines:
sourceScript = sourceScript.replace("\n", "\\\n")
filetool.save(options.sourceScriptFile, sourceScript,
options.scriptOutputEncoding)
if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
print " * Patching template: %s => %s" % (
options.sourceTemplateInputFile,
options.sourceTemplateOutputFile)
tmpl = filetool.read(options.sourceTemplateInputFile)
res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
filetool.save(options.sourceTemplateOutputFile, res,
options.scriptOutputEncoding)
######################################################################
# GENERATION OF COMPILED VERSION
######################################################################
if options.generateCompiledScript:
print
print " GENERATION OF COMPILED SCRIPT:"
print "----------------------------------------------------------------------------"
buildLineFeed = ""
if options.addNewLines:
buildLineFeed = "\n"
inlineCode = ""
inlineCode += settingsStr + buildLineFeed
inlineCode += "qx.IS_SOURCE=false;%s" % buildLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, buildLineFeed)
inlineCode += "".join(additionalOutput)
compiledOutput = inlineCode
if options.compiledScriptFile == None:
print " * You must define the compiled script file!"
sys.exit(1)
if options.verbose:
print " * Compiling..."
else:
print " * Compiling: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
compiledFileContent = compiler.compile(
loader.getTree(fileDb, fileId, options), False,
options.addNewLines, options.enableDebug)
if options.addFileIds:
compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
else:
compiledOutput += compiledFileContent
if not compiledOutput.endswith(
";") and not compiledOutput.endswith("\n"):
compiledOutput += ";"
if not options.verbose:
print
print " * Storing output as %s..." % options.compiledScriptFile
filetool.save(options.compiledScriptFile, compiledOutput,
options.scriptOutputEncoding)
def execute(fileDb, moduleDb, options, pkgid="", names=[]):
additionalOutput = []
######################################################################
# SORT OF INCLUDE LIST
######################################################################
print
print " SORT OF INCLUDE LIST:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Include (with dependencies): %s" % options.includeWithDeps
print " * Include (without dependencies): %s" % options.includeWithoutDeps
print " * Exclude (with dependencies): %s" % options.excludeWithDeps
print " * Exclude (without dependencies): %s" % options.excludeWithoutDeps
print " * Sorting %s classes..." % len(fileDb)
sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)
if len(sortedIncludeList) == 0:
print " - No class files to include. Exciting!"
sys.exit(1)
else:
print " - Including %s classes" % len(sortedIncludeList)
if options.printIncludes:
print
print " PRINT OF INCLUDE ORDER:"
print "----------------------------------------------------------------------------"
print " * The files will be included in this order:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if options.printDeps:
print
print " OUTPUT OF DEPENDENCIES:"
print "----------------------------------------------------------------------------"
print " * These are all included files with their dependencies:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
print " - Loadtime: "
for depEntry in fileDb[fileId]["loadtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["afterDeps"]) > 0:
print " - After: "
for depEntry in fileDb[fileId]["afterDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["runtimeDeps"]) > 0:
print " - Runtime: "
for depEntry in fileDb[fileId]["runtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["loadDeps"]) > 0:
print " - Load: "
for depEntry in fileDb[fileId]["loadDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["optionalDeps"]) > 0:
print " - Optional: "
for depEntry in fileDb[fileId]["optionalDeps"]:
print " - %s" % depEntry
######################################################################
# GRAPHVIZ OUTPUT
######################################################################
if options.depDotFile:
graph.store(fileDb, sortedIncludeList, options)
######################################################################
# SOURCE MIGRATION
######################################################################
if options.migrateSource:
print
print " SOURCE MIGRATION:"
print "----------------------------------------------------------------------------"
print " * Migrate Source Code..."
migrator.handle(sortedIncludeList, fileDb, options)
# Return after migration: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.fixSource:
print
print " FIX SOURCE CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Fixing code..."
else:
print " * Fixing code: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Reading %s" % fileId
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
fileContent = filetool.read(filePath, fileEncoding)
fixedContent = textutil.removeTrailingSpaces(textutil.tab2Space(textutil.any2Unix(fileContent), 2))
if fixedContent != fileContent:
if options.verbose:
print " - Storing modifications..."
else:
sys.stdout.write("!")
sys.stdout.flush()
filetool.save(filePath, fixedContent, fileEncoding)
elif not options.verbose:
sys.stdout.write(".")
sys.stdout.flush()
if not options.verbose:
print
# Return after fixing: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.prettyPrint:
print
print " GENERATION OF PRETTY PRINTED CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Pretty printing..."
else:
print " * Pretty printing: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
prettyFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), True)
if not prettyFileContent.endswith("\n"):
prettyFileContent += "\n"
filetool.save(fileDb[fileId]["path"], prettyFileContent)
if not options.verbose:
print
# Return after pretty print: Ignore other jobs
return
######################################################################
# STRING OPTIMIZATION
######################################################################
if options.optimizeStrings:
print
print " STRING OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching strings..."
else:
print " * Searching strings: ",
stringMap = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
localMap = loader.getStrings(fileDb, fileId, options)
for value in localMap:
if value in stringMap:
stringMap[value] += localMap[value]
else:
stringMap[value] = localMap[value]
if not options.verbose:
print
counter = 0
for value in stringMap:
counter += stringMap[value]
stringList = stringoptimizer.sort(stringMap)
print " * Found %s strings (used %s times)" % (len(stringMap), counter)
if options.verbose:
print " * Replacing strings..."
else:
print " * Replacing strings: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
stringoptimizer.replace(loader.getTree(fileDb, fileId, options), stringList, "$" + pkgid, options.verbose)
if not options.verbose:
print
print " * Generating replacement..."
additionalOutput.append(stringoptimizer.replacement(stringList, "$" + pkgid))
######################################################################
# LOCAL VARIABLE OPTIMIZATION
######################################################################
if options.optimizeVariables:
print
print " LOCAL VARIABLE OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing variables..."
else:
print " * Optimizing variables: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
variableoptimizer.search(loader.getTree(fileDb, fileId, options), [], 0, "$", skipPrefix = options.optimizeVariablesSkipPrefix, debug = options.enableDebug)
if not options.verbose:
print
######################################################################
# NAME OBFUSCATION
######################################################################
if options.obfuscateIdentifiers:
print
print " OBFUSCATE IDENTIFIERS:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Obfuscating identifiers..."
else:
print " * Obfuscating identifiers: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += obfuscator.update(loader.getTree(fileDb, fileId, options), names, "$$")
if not options.verbose:
print
print " * Updated %s names" % counter
######################################################################
# TOKEN STORAGE
######################################################################
if options.storeTokens:
print
print " TOKEN STORAGE:"
print "----------------------------------------------------------------------------"
if options.tokenOutputDirectory == None:
print " * You must define the token output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tokens..."
else:
print " * Storing tokens: ",
for fileId in sortedIncludeList:
tokenString = tokenizer.convertTokensToString(loader.getTokens(fileDb, fileId, options))
if options.verbose:
print " * writing tokens for %s (%s KB)..." % (fileIdm, len(tokenString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.tokenOutputDirectory), fileId + config.TOKENEXT), tokenString)
if not options.verbose:
print
######################################################################
# TREE STORAGE
######################################################################
if options.storeTree:
print
print " TREE STORAGE:"
print "----------------------------------------------------------------------------"
if options.treeOutputDirectory == None:
print " * You must define the tree output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tree..."
else:
print " * Storing tree: ",
for fileId in sortedIncludeList:
treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(loader.getTree(fileDb, fileId, options))
if options.verbose:
print " * writing tree for %s (%s KB)..." % (fileId, len(treeString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.treeOutputDirectory), fileId + config.XMLEXT), treeString)
if not options.verbose:
print
######################################################################
# GENERATION OF API
######################################################################
if options.generateApiDocumentation:
print
print " GENERATION OF API:"
print "----------------------------------------------------------------------------"
if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
print " * You must define one of JSON or XML API documentation file!"
docTree = None
if options.verbose:
print " * Generating API tree..."
else:
print " * Generating API tree: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
docTree = api.createDoc(loader.getTree(fileDb, fileId, options), docTree)
if not options.verbose:
print
if docTree:
print " * Finalizing tree..."
api.postWorkPackage(docTree, docTree)
if options.apiDocumentationXmlFile != None:
print " * Writing XML API file to %s" % options.apiDocumentationXmlFile
xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
if options.addNewLines:
xmlContent += "\n" + tree.nodeToXmlString(docTree)
else:
xmlContent += tree.nodeToXmlString(docTree, "", "", "")
filetool.save(options.apiDocumentationXmlFile, xmlContent, options.xmlOutputEncoding)
if options.apiDocumentationJsonFile != None:
print " * Writing JSON API file to %s" % options.apiDocumentationJsonFile
if options.addNewLines:
jsonContent = tree.nodeToJsonString(docTree)
else:
jsonContent = tree.nodeToJsonString(docTree, "", "", "")
filetool.save(options.apiDocumentationJsonFile, jsonContent, options.scriptOutputEncoding)
######################################################################
# CREATE COPY OF RESOURCES
######################################################################
if options.copyResources:
print
print " CREATE COPY OF RESOURCES:"
print "----------------------------------------------------------------------------"
resources.copy(options, sortedIncludeList, fileDb)
######################################################################
# GENERATION OF SETTINGS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
settingsStr = ""
if len(options.defineRuntimeSetting) != 0:
print
print " GENERATION OF SETTINGS:"
print "----------------------------------------------------------------------------"
print " * Processing input data..."
settingsStr = settings.generate(options)
if options.settingsScriptFile:
print " * Storing result to %s" % options.settingsScriptFile
filetool.save(options.settingsScriptFile, settingsStr)
# clear settings for build and source
settingsStr = ""
######################################################################
# GENERATION OF SOURCE VERSION
######################################################################
if options.generateSourceScript:
print
print " GENERATION OF SOURCE SCRIPT:"
print "----------------------------------------------------------------------------"
if options.sourceScriptFile == None and (options.sourceTemplateInputFile == None or options.sourceTemplateOutputFile == None):
print " * You must define at least one source script file or template input/output."
sys.exit(1)
if options.sourceScriptFile:
options.sourceScriptFile = os.path.normpath(options.sourceScriptFile)
if options.sourceTemplateInputFile:
options.sourceTemplateInputFile = os.path.normpath(options.sourceTemplateInputFile)
if options.sourceTemplateOutputFile:
options.sourceTemplateOutputFile = os.path.normpath(options.sourceTemplateOutputFile)
print " * Generating script block..."
# Handling line feed setting
sourceLineFeed = "";
if options.addNewLines:
sourceLineFeed = "\n";
# Generating inline code...
inlineCode = ""
inlineCode += settingsStr + sourceLineFeed
inlineCode += "qx.IS_SOURCE=true;%s" % sourceLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, sourceLineFeed)
inlineCode += "".join(additionalOutput)
# Generating script block
scriptBlocks = ""
scriptBlocks += '<script type="text/javascript">%s</script>' % inlineCode
for fileId in sortedIncludeList:
if fileDb[fileId]["classUri"] == None:
print " * Missing class URI definition for class path %s." % fileDb[fileId]["classPath"]
sys.exit(1)
scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (os.path.join(fileDb[fileId]["classUri"], fileDb[fileId]["pathId"].replace(".", os.sep)), config.JSEXT)
scriptBlocks += sourceLineFeed
if options.sourceScriptFile != None:
print " * Storing includer as %s..." % options.sourceScriptFile
sourceScript = "document.write('%s');" % scriptBlocks.replace("'", "\\'")
if options.addNewLines:
sourceScript = sourceScript.replace("\n", "\\\n")
filetool.save(options.sourceScriptFile, sourceScript, options.scriptOutputEncoding)
if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
print " * Patching template: %s => %s" % (options.sourceTemplateInputFile, options.sourceTemplateOutputFile)
tmpl = filetool.read(options.sourceTemplateInputFile)
res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
filetool.save(options.sourceTemplateOutputFile, res, options.scriptOutputEncoding)
######################################################################
# GENERATION OF COMPILED VERSION
######################################################################
if options.generateCompiledScript:
print
print " GENERATION OF COMPILED SCRIPT:"
print "----------------------------------------------------------------------------"
buildLineFeed = "";
if options.addNewLines:
buildLineFeed = "\n";
inlineCode = ""
inlineCode += settingsStr + buildLineFeed
inlineCode += "qx.IS_SOURCE=false;%s" % buildLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, buildLineFeed)
inlineCode += "".join(additionalOutput)
compiledOutput = inlineCode
if options.compiledScriptFile == None:
print " * You must define the compiled script file!"
sys.exit(1)
if options.verbose:
print " * Compiling..."
else:
print " * Compiling: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
compiledFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), False, options.addNewLines, options.enableDebug)
if options.addFileIds:
compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
else:
compiledOutput += compiledFileContent
if not compiledOutput.endswith(";") and not compiledOutput.endswith("\n"):
compiledOutput += ";"
if not options.verbose:
print
print " * Storing output as %s..." % options.compiledScriptFile
filetool.save(options.compiledScriptFile, compiledOutput, options.scriptOutputEncoding)
def execute(fileDb, moduleDb, options, pkgid="", names=[]):
######################################################################
# SORT OF INCLUDE LIST
######################################################################
print
print " GENERATE CLASS INCLUDE LIST:"
print "----------------------------------------------------------------------------"
sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)
if options.printIncludes:
print
print " PRINT OF INCLUDE ORDER:"
print "----------------------------------------------------------------------------"
print " * The files will be included in this order:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if options.printIncludesFile:
includeFile = open(options.printIncludesFile, "w")
for fileId in sortedIncludeList:
includeFile.write(fileId + "\n")
includeFile.close()
if options.printDependencies:
print
print " OUTPUT OF DEPENDENCIES:"
print "----------------------------------------------------------------------------"
print " * These are all included files with their dependencies:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
print " - Loadtime: "
for depEntry in fileDb[fileId]["loadtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["runtimeDeps"]) > 0:
print " - Runtime: "
for depEntry in fileDb[fileId]["runtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["optionalDeps"]) > 0:
print " - Optional: "
for depEntry in fileDb[fileId]["optionalDeps"]:
print " - %s" % depEntry
######################################################################
# GRAPHVIZ OUTPUT
######################################################################
if options.depDotFile:
graph.store(fileDb, sortedIncludeList, options)
######################################################################
# SOURCE MIGRATION
######################################################################
if options.migrateSource:
print
print " SOURCE MIGRATION:"
print "----------------------------------------------------------------------------"
print " * Migrate Source Code..."
migrator.handle(fileDb, options, options.migrationTarget,
options.migrationInput, options.verbose)
# Return after migration: Ignore other jobs
return
######################################################################
# GENERATION OF FIXED CODE
######################################################################
if options.fixSource:
print
print " FIX SOURCE CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Fixing code..."
else:
print " * Fixing code: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Reading %s" % fileId
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
fileContent = filetool.read(filePath, fileEncoding)
fixedContent = textutil.removeTrailingSpaces(textutil.tab2Space(textutil.any2Unix(fileContent), 2))
if fixedContent != fileContent:
if options.verbose:
print " - Storing modifications..."
else:
sys.stdout.write("!")
sys.stdout.flush()
filetool.save(filePath, fixedContent, fileEncoding)
elif not options.verbose:
sys.stdout.write(".")
sys.stdout.flush()
if not options.verbose:
print
# Return after fixing: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.prettyPrint:
print
print " GENERATION OF PRETTY PRINTED CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Pretty printing..."
else:
print " * Pretty printing: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
prettyFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), options)
if not prettyFileContent.endswith("\n"):
prettyFileContent += "\n"
filetool.save(fileDb[fileId]["path"], prettyFileContent)
if not options.verbose:
print
# Return after pretty print: Ignore other jobs
return
######################################################################
# INLINE CODE
######################################################################
inlineSourceCode = []
inlineCompiledCode = []
######################################################################
# SUPPORT FOR VARIANTS
######################################################################
if len(options.useVariant) > 0 and not options.generateSourceScript:
print
print " VARIANT OPTIMIZATION:"
print "----------------------------------------------------------------------------"
variantMap = {}
for variant in options.useVariant:
keyValue = variant.split(":")
if len(keyValue) != 2:
print " * Error: Variants must be specified as key value pair separated by ':'!"
sys.exit(1)
variantMap[keyValue[0]] = keyValue[1]
if options.verbose:
print " * Optimizing for variant setup..."
else:
print " * Optimizing for variant setup: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
if variantoptimizer.search(loader.getTree(fileDb, fileId, options), variantMap, fileId, options.verbose):
if options.verbose:
print " - Modified!"
if not options.verbose:
print
######################################################################
# BASE CALL OPTIMIZATION
######################################################################
if options.optimizeBaseCall:
print
print " BASE CALL OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing this.base calls..."
else:
print " * Optimizing this.base calls: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += basecalloptimizer.patch(loader.getTree(fileDb, fileId, options))
if not options.verbose:
print
print " * Optimized %s calls" % counter
######################################################################
# ACCESSOR OBFUSCATION
######################################################################
if options.obfuscateAccessors:
print
print " ACCESSOR OBFUSCATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Obfuscating..."
else:
print " * Obfuscating: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
accessorobfuscator.process(loader.getTree(fileDb, fileId, options), options.verbose)
if not options.verbose:
print
######################################################################
# STRING OPTIMIZATION
######################################################################
if options.optimizeStrings:
print
print " STRING OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching strings..."
else:
print " * Searching strings: ",
stringMap = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
localMap = stringoptimizer.search(loader.getTree(fileDb, fileId, options), options.verbose)
for value in localMap:
if value in stringMap:
stringMap[value] += localMap[value]
else:
stringMap[value] = localMap[value]
if not options.verbose:
print
counter = 0
for value in stringMap:
counter += stringMap[value]
stringList = stringoptimizer.sort(stringMap)
print " * Found %s string instances (%s unique)" % (counter, len(stringMap))
if options.verbose:
print " * Replacing strings..."
else:
print " * Replacing strings: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
stringoptimizer.replace(loader.getTree(fileDb, fileId, options), stringList, "$" + pkgid, options.verbose)
if not options.verbose:
print
print " * Generating replacement..."
inlineCompiledCode.append(stringoptimizer.replacement(stringList, "$" + pkgid))
######################################################################
# LOCAL VARIABLE OPTIMIZATION
######################################################################
if options.optimizeVariables or options.obfuscate:
print
print " LOCAL VARIABLE OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing variables..."
else:
print " * Optimizing variables: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += variableoptimizer.search(
loader.getTree(fileDb, fileId, options),
[], 0, 0, "$",
skipPrefix = options.optimizeVariablesSkipPrefix,
verbose = options.verbose
)
if not options.verbose:
print
print " * Optimized %s variables" % counter
######################################################################
# PRIVATE MEMBER OPTIMIZATION
######################################################################
if options.optimizePrivate:
print
print " PRIVATE MEMBER OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing private members..."
else:
print " * Optimizing private members: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += privateoptimizer.patch("",loader.getTree(fileDb, fileId, options), {}, "$", options.verbose)
if not options.verbose:
print
print " * Optimized %s keys" % counter
######################################################################
# OBFUSCATION
######################################################################
if options.obfuscate:
print
print " OBFUSCATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching for assignments..."
else:
print " * Searching for assignments: ",
identifiers = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
obfuscator.search(loader.getTree(fileDb, fileId, options), identifiers, options.verbose)
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
obfuscator.patch(loader.getTree(fileDb, fileId, options), identifiers, options.verbose)
if not options.verbose:
print
######################################################################
# TOKEN STORAGE
######################################################################
if options.storeTokens:
print
print " TOKEN STORAGE:"
print "----------------------------------------------------------------------------"
if options.tokenOutputDirectory == None:
print " * You must define the token output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tokens..."
else:
print " * Storing tokens: ",
for fileId in sortedIncludeList:
tokenString = tokenizer.convertTokensToString(loader.getTokens(fileDb, fileId, options))
if options.verbose:
print " * Writing tokens for %s (%s KB)..." % (fileIdm, len(tokenString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.tokenOutputDirectory), fileId + config.TOKENEXT), tokenString)
if not options.verbose:
print
######################################################################
# TREE STORAGE
######################################################################
if options.storeTree:
print
print " TREE STORAGE:"
print "----------------------------------------------------------------------------"
if options.treeOutputDirectory == None:
print " * You must define the tree output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tree..."
else:
print " * Storing tree: ",
for fileId in sortedIncludeList:
treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(loader.getTree(fileDb, fileId, options))
if options.verbose:
print " * Writing tree for %s (%s KB)..." % (fileId, len(treeString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.treeOutputDirectory), fileId + config.XMLEXT), treeString)
if not options.verbose:
print
######################################################################
# GENERATION OF API
######################################################################
if options.generateApiDocumentation:
print
print " GENERATION OF API:"
print "----------------------------------------------------------------------------"
if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
print " * You must define one of JSON or XML API documentation file!"
docTree = None
if options.verbose:
print " * Generating API tree..."
else:
print " * Generating API tree: ",
hasDocError = False
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
(docTree, error) = api.createDoc(loader.getTree(fileDb, fileId, options), docTree)
hasDocError = hasDocError or error
if hasDocError:
print
print " * Building API failed!!"
sys.exit(0)
if not options.verbose:
print
if docTree:
print " * Finalizing tree..."
api.postWorkPackage(docTree, docTree)
if options.addNewLines:
childPrefix = " "
newLine = "\n"
else:
childPrefix = ""
newLine = ""
if options.apiDocumentationXmlFile != None:
print " * Writing XML API file to %s" % options.apiDocumentationXmlFile
if options.apiSeparateFiles:
packages = api.packagesToXmlString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationXmlFile, packages, options.scriptOutputEncoding)
for cls in api.classNodeIterator(docTree):
classContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
classContent += tree.nodeToXmlString(cls, "", childPrefix, newLine)
fileName = os.path.join(os.path.dirname(options.apiDocumentationXmlFile), cls.get("fullName") + ".xml")
filetool.save(fileName, classContent, options.xmlOutputEncoding)
else:
xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
xmlContent += tree.nodeToXmlString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationXmlFile, xmlContent, options.xmlOutputEncoding)
if options.apiDocumentationJsonFile != None:
print " * Writing JSON API file to %s" % options.apiDocumentationJsonFile
if options.apiSeparateFiles:
packages = api.packagesToJsonString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationJsonFile, packages, options.scriptOutputEncoding)
for cls in api.classNodeIterator(docTree):
classContent = tree.nodeToJsonString(cls, "", childPrefix, newLine)
fileName = os.path.join(os.path.dirname(options.apiDocumentationJsonFile), cls.get("fullName") + ".js")
filetool.save(fileName, classContent, options.scriptOutputEncoding)
else:
jsonContent = tree.nodeToJsonString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationJsonFile, jsonContent, options.scriptOutputEncoding)
# create search index
if options.apiDocumentationIndexFile != None:
print " * Generating API index..."
jsonContent = tree.nodeToIndexString(docTree, "", childPrefix, newLine)
print " * Writing API search index to %s" % options.apiDocumentationIndexFile
filetool.save(options.apiDocumentationIndexFile, jsonContent, options.scriptOutputEncoding)
######################################################################
# CREATE COPY OF RESOURCES
######################################################################
if options.copyResources:
print
print " CREATE COPY OF RESOURCES:"
print "----------------------------------------------------------------------------"
resources.copy(options, sortedIncludeList, fileDb)
######################################################################
# GENERATION OF SETTINGS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
print
print " INCLUSION OF SETTINGS:"
print "----------------------------------------------------------------------------"
if options.generateSourceScript:
print " * Processing settings of source version..."
settingsArr = options.useSetting[:]
settingsArr.append("qx.version:%s" % options.version)
settingsArr.append("qx.isSource:true")
inlineSourceCode.append(settings.generate(settingsArr, options.addNewLines))
if options.generateCompiledScript:
print " * Processing settings of compiled version..."
settingsArr = options.useSetting[:]
settingsArr.append("qx.version:%s" % options.version)
settingsArr.append("qx.isSource:false")
inlineCompiledCode.append(settings.generate(settingsArr, options.addNewLines))
######################################################################
# GENERATION OF VARIANTS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
print
print " INCLUSION OF VARIANTS:"
print "----------------------------------------------------------------------------"
if options.generateSourceScript:
print " * Processing variants of source version..."
variantsArr = options.useVariant[:]
inlineSourceCode.append(variants.generate(variantsArr, options.addNewLines))
if options.generateCompiledScript:
print " * Processing variants of compiled version..."
variantsArr = options.useVariant[:]
inlineCompiledCode.append(variants.generate(variantsArr, options.addNewLines))
######################################################################
# GENERATION OF SOURCE VERSION
######################################################################
if options.generateSourceScript:
print
print " GENERATION OF SOURCE SCRIPT:"
print "----------------------------------------------------------------------------"
if options.sourceScriptFile == None and (options.sourceTemplateInputFile == None or options.sourceTemplateOutputFile == None):
print " * You must define at least one source script file or template input/output."
sys.exit(1)
if options.sourceScriptFile:
options.sourceScriptFile = os.path.normpath(options.sourceScriptFile)
if options.sourceTemplateInputFile:
options.sourceTemplateInputFile = os.path.normpath(options.sourceTemplateInputFile)
if options.sourceTemplateOutputFile:
options.sourceTemplateOutputFile = os.path.normpath(options.sourceTemplateOutputFile)
print " * Generating script block..."
# Handling line feed setting
sourceLineFeed = "";
if options.addNewLines:
sourceLineFeed = "\n";
# Generating script block
scriptBlocks = ""
scriptBlocks += '<script type="text/javascript">%s</script>' % "".join(inlineSourceCode)
for fileId in sortedIncludeList:
if fileDb[fileId]["meta"]:
continue
if fileDb[fileId]["classUri"] == None:
print " * Missing class URI definition for class path %s." % fileDb[fileId]["classPath"]
sys.exit(1)
scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (os.path.join(fileDb[fileId]["classUri"], fileDb[fileId]["pathId"].replace(".", '/')), config.JSEXT)
scriptBlocks += sourceLineFeed
# Writing includer
if options.sourceScriptFile != None:
print " * Storing includer as %s..." % options.sourceScriptFile
sourceScript = "document.write('%s');" % scriptBlocks.replace("'", "\\'")
if options.addNewLines:
sourceScript = sourceScript.replace("\n", "\\\n")
filetool.save(options.sourceScriptFile, sourceScript, options.scriptOutputEncoding)
# Patching template
if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
print " * Patching template: %s => %s" % (options.sourceTemplateInputFile, options.sourceTemplateOutputFile)
tmpl = filetool.read(options.sourceTemplateInputFile)
res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
filetool.save(options.sourceTemplateOutputFile, res, options.scriptOutputEncoding)
######################################################################
# GENERATION OF COMPILED VERSION
######################################################################
if options.generateCompiledScript:
print
print " GENERATION OF COMPILED SCRIPT:"
print "----------------------------------------------------------------------------"
buildLineFeed = "";
if options.addNewLines:
buildLineFeed = "\n";
compiledOutput = "".join(inlineCompiledCode)
if options.compiledScriptFile == None:
print " * You must define the compiled script file!"
sys.exit(1)
if options.verbose:
print " * Compiling..."
else:
print " * Compiling: ",
for fileId in sortedIncludeList:
if fileDb[fileId]["meta"]:
continue
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
if options.prettyPrint:
options.prettyPrint = False # make sure it's disabled
compiledFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), options, options.addNewLines, options.verbose)
if options.addFileIds:
compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
else:
compiledOutput += compiledFileContent
if not compiledOutput.endswith(";") and not compiledOutput.endswith("\n"):
compiledOutput += ";"
if not options.verbose:
print
print " * Storing output as %s..." % options.compiledScriptFile
filetool.save(options.compiledScriptFile, compiledOutput, options.scriptOutputEncoding)
def handle(fileList, fileDb, options):
confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget)
infoPath = os.path.join(confPath, "info")
patchPath = os.path.join(confPath, "patches")
importedModule = False
infoList = []
patchList = []
htmlList = getHtmlList(options)
print " * Number of script input files: %s" % len(fileList)
print " * Number of HTML input files: %s" % len(htmlList)
print " * Update to version: %s" % options.migrationTarget
print " * Searching for patch module..."
for root, dirs, files in os.walk(confPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
if os.path.splitext(fileName)[1] != config.PYEXT:
continue
if fileName == "patch.py":
print " - Importing..."
if not root in sys.path:
sys.path.insert(0, root)
import patch
importedModule = True
emptyLine = re.compile("^\s*$")
print " * Searching for info expression data..."
for root, dirs, files in os.walk(infoPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
infoList.append({"path":filePath, "content":fileContent.split("\n")})
if options.verbose:
print " - %s" % filePath
print " - Number of info files: %s" % len(infoList)
print " - Compiling expressions..."
compiledInfos = []
for infoFile in infoList:
print " - %s" % os.path.basename(infoFile["path"])
for line in infoFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledInfos.append(compiled)
print " - Number of infos: %s" % len(compiledInfos)
print " * Searching for patch expression data..."
for root, dirs, files in os.walk(patchPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
patchList.append({"path":filePath, "content":fileContent.split("\n")})
if options.verbose:
print " - %s" % filePath
print " - Number of patch files: %s" % len(patchList)
print " - Compiling expressions..."
compiledPatches = []
for patchFile in patchList:
print " - %s" % os.path.basename(patchFile["path"])
for line in patchFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledPatches.append(compiled)
print " - Number of patches: %s" % len(compiledPatches)
print
print " FILE PROCESSING:"
print "----------------------------------------------------------------------------"
if len(fileList) > 0:
print " * Processing script files:"
for fileId in fileList:
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
print " - %s" % fileId
# Read in original content
fileContent = filetool.read(filePath, fileEncoding)
patchedContent = fileContent
# Apply patches
if importedModule:
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
patchedContent = compiler.compile(tree, True)
patchedContent = regtool(patchedContent, compiledPatches, True, options)
patchedContent = regtool(patchedContent, compiledInfos, False, options)
# Write file
if patchedContent != fileContent:
print " - Store modifications..."
filetool.save(filePath, patchedContent, fileEncoding)
print " * Done"
if len(htmlList) > 0:
print " * Processing HTML files:"
for filePath in htmlList:
print " - %s" % filePath
# Read in original content
fileContent = filetool.read(filePath)
patchedContent = fileContent
patchedContent = regtool(patchedContent, compiledPatches, True, options)
patchedContent = regtool(patchedContent, compiledInfos, False, options)
# Write file
if patchedContent != fileContent:
print " - Store modifications..."
filetool.save(filePath, patchedContent)
print " * Done"
