def main():
allowed = ["any2Dos", "any2Mac", "any2Unix", "convertDos2Mac", "convertDos2Unix", "convertMac2Dos", "convertMac2Unix", "convertUnix2Dos", "convertUnix2Mac", "spaces2Tab", "tab2Space", "removeTrailingSpaces", "normalizeWhiteSpace"]
parser = optparse.OptionParser()
parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=False, help="Quiet output mode.")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Verbose output mode.")
parser.add_option("-c", "--command", dest="command", help="Normalize a file")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if not options.command in allowed:
raise RuntimeError( "Unallowed command: %s" % options.command)
if len(args) == 0:
raise RuntimeError( "Needs one or more arguments (files) to modify!")
for fileName in args:
if options.verbose:
print " * Running %s on: %s" % (options.command, fileName)
ref = codecs.open(fileName, encoding=options.encoding, mode="r")
origFileContent = ref.read()
ref.close()
patchedFileContent = eval(options.command + "(origFileContent)")
if patchedFileContent != origFileContent:
if options.verbose:
print " * Store modifications..."
filetool.save(fileName, patchedFileContent, options.encoding)
def main():
parser = optparse.OptionParser()
parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default=".tokenized")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to tokenize!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
else:
print "Compiling %s => stdout" % fileName
tokenString = convertTokensToString(parseFile(fileName, fileName, options.encoding))
if options.write:
filetool.save(fileName + options.extension, tokenString, options.encoding)
else:
try:
print tokenString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def main():
parser = optparse.OptionParser()
parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default=".tokenized")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to tokenize!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
else:
print "Compiling %s => stdout" % fileName
tokenString = convertTokensToString(parseFile(fileName, "", options.encoding))
if options.write:
filetool.save(fileName + options.extension, tokenString, options.encoding)
else:
try:
print tokenString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def store(fileDb, sortedIncludeList, options):
content = '''digraph "qooxdoo" {
node [style=filled];
'''
largest = 0
for fileId in sortedIncludeList:
size = os.path.getsize(fileDb[fileId]["path"])
if size > largest:
largest = size
for fileId in sortedIncludeList:
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
for depEntry in fileDb[fileId]["loadtimeDeps"]:
content += dotLine(fileDb, fileId, depEntry, largest)
if len(fileDb[fileId]["afterDeps"]) > 0:
for depEntry in fileDb[fileId]["afterDeps"]:
content += dotLine(fileDb, fileId, depEntry, largest)
if len(fileDb[fileId]["runtimeDeps"]) > 0:
for depEntry in fileDb[fileId]["runtimeDeps"]:
content += dotLine(fileDb, fileId, depEntry, largest)
if len(fileDb[fileId]["loadDeps"]) > 0:
for depEntry in fileDb[fileId]["loadDeps"]:
content += dotLine(fileDb, fileId, depEntry, largest)
content += '}'
filetool.save(options.depDotFile, content)
def migrateFile(
filePath, compiledPatches, compiledInfos,
hasPatchModule=False, options=None, encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = loader.extractFileContentId(fileContent);
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
patchedContent = compiler.compile(tree, options)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." % filePath)
filetool.save(filePath, patchedContent, encoding)
def main():
parser = optparse.OptionParser()
parser.add_option("-d",
"--define-runtime-setting",
action="append",
dest="defineRuntimeSetting",
metavar="NAMESPACE.KEY:VALUE",
default=[],
help="Define a setting.")
parser.add_option("-s",
"--settings-script-file",
dest="settingsScriptFile",
metavar="FILENAME",
help="Name of settings script file.")
parser.add_option("-n",
"--add-new-lines",
action="store_true",
dest="addNewLines",
default=False,
help="Keep newlines in compiled files.")
(options, args) = parser.parse_args()
if options.settingsScriptFile == None:
print " * Please define the output file!"
sys.exit(1)
if len(options.defineRuntimeSetting) == 0:
print " * Please define at least one runtime setting!"
sys.exit(1)
print " * Saving settings to %s" % options.settingsScriptFile
filetool.save(options.settingsScriptFile, generate(options))
def main():
allowed = [ "any2Dos", "any2Mac", "any2Unix", "convertDos2Mac", "convertDos2Unix", "convertMac2Dos", "convertMac2Unix", "convertUnix2Dos", "convertUnix2Mac", "spaces2Tab", "tab2Space" ]
parser = optparse.OptionParser()
parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=False, help="Quiet output mode.")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Verbose output mode.")
parser.add_option("-c", "--command", dest="command", default="normalize", help="Normalize a file")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if not options.command in allowed:
print "Unallowed command: %s" % options.command
sys.exit(1)
if len(args) == 0:
print "Needs one or more arguments (files) to modify!"
sys.exit(1)
for fileName in args:
if options.verbose:
print " * Running %s on: %s" % (options.command, fileName)
origFileContent = filetool.read(fileName, options.encoding)
patchedFileContent = eval(options.command + "(origFileContent)")
if patchedFileContent != origFileContent:
filetool.save(fileName, patchedFileContent, options.encoding)
def main():
allowed = ["any2Dos", "any2Mac", "any2Unix", "convertDos2Mac", "convertDos2Unix", "convertMac2Dos", "convertMac2Unix", "convertUnix2Dos", "convertUnix2Mac", "spaces2Tab", "tab2Space"]
parser = optparse.OptionParser()
parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=False, help="Quiet output mode.")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Verbose output mode.")
parser.add_option("-c", "--command", dest="command", help="Normalize a file")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if not options.command in allowed:
print "Unallowed command: %s" % options.command
sys.exit(1)
if len(args) == 0:
print "Needs one or more arguments (files) to modify!"
sys.exit(1)
for fileName in args:
if options.verbose:
print " * Running %s on: %s" % (options.command, fileName)
ref = codecs.open(fileName, encoding=options.encoding, mode="r")
origFileContent = ref.read()
ref.close()
patchedFileContent = eval(options.command + "(origFileContent)")
if patchedFileContent != origFileContent:
if options.verbose:
print " * Store modifications..."
filetool.save(fileName, patchedFileContent, options.encoding)
def main():
parser = optparse.OptionParser()
parser.add_option("-w",
"--write",
action="store_true",
dest="write",
default=False,
help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e",
"--extension",
dest="extension",
metavar="EXTENSION",
help="The EXTENSION to use",
default=".compiled")
parser.add_option("--optimize-variables",
action="store_true",
dest="optimizeVariables",
default=False,
help="Optimize variables. Reducing size.")
parser.add_option("--encoding",
dest="encoding",
default="utf-8",
metavar="ENCODING",
help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Generating tree of %s => %s%s" % (fileName, fileName,
options.extension)
else:
print "Generating tree of %s => stdout" % fileName
restree = createSyntaxTree(
tokenizer.parseFile(fileName, "", options.encoding))
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, "$")
compiledString = tree.nodeToXmlString(restree)
if options.write:
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def main():
parser = optparse.OptionParser("usage: %prog [options]", option_class=optparseext.ExtendAction)
parser.add_option("-d", "--use-variant", action="extend", dest="useVariant", type="string", metavar="NAMESPACE.KEY:VALUE", default=[], help="Define a variant.")
parser.add_option("-o", "--output-file", dest="outputFile", metavar="FILENAME", help="Name of variants script file.")
parser.add_option("-n", "--add-new-lines", action="store_true", dest="addNewLines", default=False, help="Keep newlines in compiled files.")
(options, args) = parser.parse_args()
if options.outputFile == None:
print generate(options.useVariant, options.addNewLines)
else:
print " * Saving variants to %s" % options.outputFile
filetool.save(options.outputFile, generate(options))
def main():
parser = optparse.OptionParser()
parser.add_option("-d", "--define-runtime-setting", action="append", dest="defineRuntimeSetting", metavar="NAMESPACE.KEY:VALUE", default=[], help="Define a setting.")
parser.add_option("-s", "--settings-script-file", dest="settingsScriptFile", metavar="FILENAME", help="Name of settings script file.")
parser.add_option("-n", "--add-new-lines", action="store_true", dest="addNewLines", default=False, help="Keep newlines in compiled files.")
(options, args) = parser.parse_args()
if options.settingsScriptFile == None:
print " * Please define the output file!"
sys.exit(1)
if len(options.defineRuntimeSetting) == 0:
print " * Please define at least one runtime setting!"
sys.exit(1)
print " * Saving settings to %s" % options.settingsScriptFile
filetool.save(options.settingsScriptFile, generate(options))
def main():
parser = optparse.OptionParser()
parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="")
parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False)
parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
else:
print "Compiling %s => stdout" % fileName
restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, "", options.encoding))
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, "$")
compiledString = compile(restree, not options.compress)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def store(fileDb, sortedIncludeList, options):
content = '''digraph "qooxdoo" {
node [style=filled];
'''
largest = 0
for fileId in sortedIncludeList:
size = os.path.getsize(fileDb[fileId]["path"])
if size > largest:
largest = size
for fileId in sortedIncludeList:
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
for depEntry in fileDb[fileId]["loadtimeDeps"]:
content += dotLine(fileDb, fileId, depEntry, largest)
if len(fileDb[fileId]["runtimeDeps"]) > 0:
for depEntry in fileDb[fileId]["runtimeDeps"]:
content += dotLine(fileDb, fileId, depEntry, largest)
content += '}'
filetool.save(options.depDotFile, content)
def migrateFile(filePath,
compiledPatches,
compiledInfos,
hasPatchModule=False,
options=None,
encoding="UTF-8"):
logging.info(" - File: %s" % filePath)
# Read in original content
fileContent = filetool.read(filePath, encoding)
fileId = loader.extractFileContentId(fileContent)
# Apply patches
patchedContent = fileContent
if hasPatchModule and fileId is not None:
import patch
tree = treegenerator.createSyntaxTree(
tokenizer.parseStream(fileContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
options.prettyPrint = True # make sure it's set
patchedContent = compiler.compile(tree, options)
# apply RE patches
patchedContent = regtool(patchedContent, compiledPatches, True, filePath)
patchedContent = regtool(patchedContent, compiledInfos, False, filePath)
# Write file
if patchedContent != fileContent:
logging.info(" - %s has been modified. Storing modifications ..." %
filePath)
filetool.save(filePath, patchedContent, encoding)
def argparser(cmdlineargs):
# Parse arguments
(options, args) = getparser().parse_args(cmdlineargs)
# Export to file
if options.exportToFile != None:
print
print " EXPORTING:"
print "----------------------------------------------------------------------------"
print " * Translating options..."
optionString = "# Exported configuration from build.py\n\n"
ignoreValue = True
lastWasKey = False
for arg in cmdlineargs:
if arg == "--export-to-file":
ignoreValue = True
elif arg.startswith("--"):
if lastWasKey:
optionString += "\n"
optionString += arg[2:]
ignoreValue = False
lastWasKey = True
elif arg.startswith("-"):
print " - Could not export short argument: %s" % arg
optionString += "\n# Ignored short argument %s\n" % arg
ignoreValue = True
elif not ignoreValue:
optionString += " = %s\n" % arg
ignoreValue = True
lastWasKey = False
print " * Export to file: %s" % options.exportToFile
filetool.save(options.exportToFile, optionString)
sys.exit(0)
# Read from file
elif options.fromFile != None:
print
print " INITIALIZATION:"
print "----------------------------------------------------------------------------"
print " * Reading configuration..."
# Convert file content into arguments
fileargs = {}
fileargpos = 0
fileargid = "default"
currentfileargs = []
fileargs[fileargid] = currentfileargs
alternativeFormatBegin = re.compile("\s*\[\s*")
alternativeFormatEnd = re.compile("\s*\]\s*=\s*")
emptyLine = re.compile("^\s*$")
for line in file(options.fromFile).read().split("\n"):
line = line.strip()
if emptyLine.match(line) or line.startswith(
"#") or line.startswith("//"):
continue
# Translating...
line = alternativeFormatBegin.sub(" = ", line)
line = alternativeFormatEnd.sub(":", line)
# Splitting line
line = line.split("=")
# Extract key element
key = line.pop(0).strip()
# Separate packages
if key == "package":
fileargpos += 1
fileargid = line[0].strip()
print " - Found package: %s" % fileargid
currentfileargs = []
fileargs[fileargid] = currentfileargs
continue
currentfileargs.append("--%s" % key)
if len(line) > 0:
value = line[0].strip()
currentfileargs.append(value)
# Parse
defaultargs = fileargs["default"]
if len(fileargs) > 1:
(fileDb, moduleDb) = load(getparser().parse_args(defaultargs)[0])
if options.obfuscateIdentifiers:
sharednames = {}
for filearg in fileargs:
if filearg == "default":
continue
combinedargs = []
combinedargs.extend(defaultargs)
combinedargs.extend(fileargs[filearg])
options = getparser().parse_args(defaultargs)[0]
findnames(fileDb, moduleDb, options, sharednames)
names = obfuscator.sort(sharednames)
for filearg in fileargs:
if filearg == "default":
continue
print
print
print
print
print " PACKAGE: %s" % filearg
print "----------------------------------------------------------------------------"
combinedargs = []
combinedargs.extend(defaultargs)
combinedargs.extend(fileargs[filearg])
options = getparser().parse_args(combinedargs)[0]
if options.obfuscateIdentifiers:
execute(fileDb, moduleDb, options, filearg, names)
else:
execute(fileDb, moduleDb, options, filearg)
else:
options = getparser().parse_args(defaultargs)[0]
(fileDb, moduleDb) = load(options)
if options.obfuscateIdentifiers:
execute(fileDb, moduleDb, options, "",
obfuscator.sort(findnames(fileDb, moduleDb, options)))
else:
execute(fileDb, moduleDb, options, "")
else:
print
print " INITIALIZATION:"
print "----------------------------------------------------------------------------"
print " * Processing arguments..."
(fileDb, moduleDb) = load(options)
if options.obfuscateIdentifiers:
execute(fileDb, moduleDb, options, options.packageId,
obfuscator.sort(findnames(fileDb, moduleDb, options)))
else:
execute(fileDb, moduleDb, options, options.packageId)
def handle(fileList, fileDb, options):
confPath = os.path.join(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "migration"), options.migrationTarget)
infoPath = os.path.join(confPath, "info")
patchPath = os.path.join(confPath, "patches")
importedModule = False
infoList = []
patchList = []
htmlList = getHtmlList(options)
print " * Number of script input files: %s" % len(fileList)
print " * Number of HTML input files: %s" % len(htmlList)
print " * Update to version: %s" % options.migrationTarget
print " * Searching for patch module..."
for root, dirs, files in os.walk(confPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
if os.path.splitext(fileName)[1] != config.PYEXT:
continue
if fileName == "patch.py":
print " - Importing..."
if not root in sys.path:
sys.path.insert(0, root)
import patch
importedModule = True
emptyLine = re.compile("^\s*$")
print " * Searching for info expression data..."
for root, dirs, files in os.walk(infoPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
infoList.append({"path":filePath, "content":fileContent.split("\n")})
if options.verbose:
print " - %s" % filePath
print " - Number of info files: %s" % len(infoList)
print " - Compiling expressions..."
compiledInfos = []
for infoFile in infoList:
print " - %s" % os.path.basename(infoFile["path"])
for line in infoFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledInfos.append(compiled)
print " - Number of infos: %s" % len(compiledInfos)
print " * Searching for patch expression data..."
for root, dirs, files in os.walk(patchPath):
# Filter ignored directories
for ignoredDir in config.DIRIGNORE:
if ignoredDir in dirs:
dirs.remove(ignoredDir)
# Searching for files
for fileName in files:
filePath = os.path.join(root, fileName)
fileContent = textutil.any2Unix(filetool.read(filePath, "utf-8"))
patchList.append({"path":filePath, "content":fileContent.split("\n")})
if options.verbose:
print " - %s" % filePath
print " - Number of patch files: %s" % len(patchList)
print " - Compiling expressions..."
compiledPatches = []
for patchFile in patchList:
print " - %s" % os.path.basename(patchFile["path"])
for line in patchFile["content"]:
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
compiled = entryCompiler(line)
if compiled != None:
compiledPatches.append(compiled)
print " - Number of patches: %s" % len(compiledPatches)
print
print " FILE PROCESSING:"
print "----------------------------------------------------------------------------"
if len(fileList) > 0:
print " * Processing script files:"
for fileId in fileList:
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
print " - %s" % fileId
# Read in original content
fileContent = filetool.read(filePath, fileEncoding)
patchedContent = fileContent
# Apply patches
if importedModule:
tree = treegenerator.createSyntaxTree(tokenizer.parseStream(patchedContent))
# If there were any changes, compile the result
if patch.patch(fileId, tree):
patchedContent = compiler.compile(tree, True)
patchedContent = regtool(patchedContent, compiledPatches, True, options)
patchedContent = regtool(patchedContent, compiledInfos, False, options)
# Write file
if patchedContent != fileContent:
print " - Store modifications..."
filetool.save(filePath, patchedContent, fileEncoding)
print " * Done"
if len(htmlList) > 0:
print " * Processing HTML files:"
for filePath in htmlList:
print " - %s" % filePath
# Read in original content
fileContent = filetool.read(filePath)
patchedContent = fileContent
patchedContent = regtool(patchedContent, compiledPatches, True, options)
patchedContent = regtool(patchedContent, compiledInfos, False, options)
# Write file
if patchedContent != fileContent:
print " - Store modifications..."
filetool.save(filePath, patchedContent)
print " * Done"
def main():
global options
parser = optparse.OptionParser(option_class=optparseext.ExtendAction)
parser.add_option("-w",
"--write",
action="store_true",
dest="write",
default=False,
help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e",
"--extension",
dest="extension",
metavar="EXTENSION",
help="The EXTENSION to use",
default="")
parser.add_option("-c",
"--compress",
action="store_true",
dest="compress",
help="Enable compression",
default=False)
parser.add_option("--optimize-variables",
action="store_true",
dest="optimizeVariables",
default=False,
help="Optimize variables. Reducing size.")
parser.add_option(
"--optimize-privates",
action="store_true",
dest="optimizePrivates",
default=False,
help="Optimize privates. Protected them and reducing size.")
parser.add_option("--obfuscate-accessors",
action="store_true",
dest="obfuscateAccessors",
default=False,
help="Enable accessor obfuscation")
parser.add_option("--encoding",
dest="encoding",
default="utf-8",
metavar="ENCODING",
help="Defines the encoding expected for input files.")
parser.add_option("--use-variant",
action="extend",
dest="useVariant",
type="string",
metavar="NAMESPACE.KEY:VALUE",
default=[],
help="Optimize for the given variant.")
# Options for pretty printing
addCommandLineOptions(parser)
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName,
options.extension)
restree = treegenerator.createSyntaxTree(
tokenizer.parseFile(fileName, fileName, options.encoding))
if len(options.useVariant) > 0:
variantMap = {}
for variant in options.useVariant:
keyValue = variant.split(":")
if len(keyValue) != 2:
print " * Error: Variants must be specified as key value pair separated by ':'!"
sys.exit(1)
variantMap[keyValue[0]] = keyValue[1]
variantoptimizer.search(restree, variantMap, fileName)
if options.obfuscateAccessors:
accessorobfuscator.process(restree)
if options.optimizePrivates:
privateoptimizer.patch("A", restree, {})
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, 0, "$")
if options.compress:
options.prettyPrint = False # make sure it's set
else:
options.prettyPrint = True
compiledString = compile(restree, options)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def main():
global options
parser = optparse.OptionParser(option_class=optparseext.ExtendAction)
parser.add_option("-w", "--write", action="store_true", dest="write", default=False, help="Writes file to incoming fileName + EXTENSION.")
parser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="The EXTENSION to use", default="")
parser.add_option("-c", "--compress", action="store_true", dest="compress", help="Enable compression", default=False)
parser.add_option("--optimize-variables", action="store_true", dest="optimizeVariables", default=False, help="Optimize variables. Reducing size.")
parser.add_option("--optimize-privates", action="store_true", dest="optimizePrivates", default=False, help="Optimize privates. Protected them and reducing size.")
parser.add_option("--obfuscate-accessors", action="store_true", dest="obfuscateAccessors", default=False, help="Enable accessor obfuscation")
parser.add_option("--encoding", dest="encoding", default="utf-8", metavar="ENCODING", help="Defines the encoding expected for input files.")
parser.add_option("--use-variant", action="extend", dest="useVariant", type="string", metavar="NAMESPACE.KEY:VALUE", default=[], help="Optimize for the given variant.")
# Options for pretty printing
addCommandLineOptions(parser)
(options, args) = parser.parse_args()
if len(args) == 0:
print "Needs one or more arguments (files) to compile!"
sys.exit(1)
for fileName in args:
if options.write:
print "Compiling %s => %s%s" % (fileName, fileName, options.extension)
restree = treegenerator.createSyntaxTree(tokenizer.parseFile(fileName, fileName, options.encoding))
if len(options.useVariant) > 0:
variantMap = {}
for variant in options.useVariant:
keyValue = variant.split(":")
if len(keyValue) != 2:
print " * Error: Variants must be specified as key value pair separated by ':'!"
sys.exit(1)
variantMap[keyValue[0]] = keyValue[1]
variantoptimizer.search(restree, variantMap, fileName)
if options.obfuscateAccessors:
accessorobfuscator.process(restree)
if options.optimizePrivates:
privateoptimizer.patch("A", restree, {})
if options.optimizeVariables:
variableoptimizer.search(restree, [], 0, 0, "$")
if options.compress:
options.prettyPrint = False # make sure it's set
else:
options.prettyPrint = True
compiledString = compile(restree, options)
if options.write:
if compiledString != "" and not compiledString.endswith("\n"):
compiledString += "\n"
filetool.save(fileName + options.extension, compiledString)
else:
try:
print compiledString
except UnicodeEncodeError:
print " * Could not encode result to ascii. Use '-w' instead."
sys.exit(1)
def execute(fileDb, moduleDb, options, pkgid="", names=[]):
######################################################################
# SORT OF INCLUDE LIST
######################################################################
print
print " GENERATE CLASS INCLUDE LIST:"
print "----------------------------------------------------------------------------"
sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)
if options.printIncludes:
print
print " PRINT OF INCLUDE ORDER:"
print "----------------------------------------------------------------------------"
print " * The files will be included in this order:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if options.printIncludesFile:
includeFile = open(options.printIncludesFile, "w")
for fileId in sortedIncludeList:
includeFile.write(fileId + "\n")
includeFile.close()
if options.printDependencies:
print
print " OUTPUT OF DEPENDENCIES:"
print "----------------------------------------------------------------------------"
print " * These are all included files with their dependencies:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
print " - Loadtime: "
for depEntry in fileDb[fileId]["loadtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["runtimeDeps"]) > 0:
print " - Runtime: "
for depEntry in fileDb[fileId]["runtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["optionalDeps"]) > 0:
print " - Optional: "
for depEntry in fileDb[fileId]["optionalDeps"]:
print " - %s" % depEntry
######################################################################
# GRAPHVIZ OUTPUT
######################################################################
if options.depDotFile:
graph.store(fileDb, sortedIncludeList, options)
######################################################################
# SOURCE MIGRATION
######################################################################
if options.migrateSource:
print
print " SOURCE MIGRATION:"
print "----------------------------------------------------------------------------"
print " * Migrate Source Code..."
migrator.handle(fileDb, options, options.migrationTarget,
options.migrationInput, options.verbose)
# Return after migration: Ignore other jobs
return
######################################################################
# GENERATION OF FIXED CODE
######################################################################
if options.fixSource:
print
print " FIX SOURCE CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Fixing code..."
else:
print " * Fixing code: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Reading %s" % fileId
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
fileContent = filetool.read(filePath, fileEncoding)
fixedContent = textutil.removeTrailingSpaces(textutil.tab2Space(textutil.any2Unix(fileContent), 2))
if fixedContent != fileContent:
if options.verbose:
print " - Storing modifications..."
else:
sys.stdout.write("!")
sys.stdout.flush()
filetool.save(filePath, fixedContent, fileEncoding)
elif not options.verbose:
sys.stdout.write(".")
sys.stdout.flush()
if not options.verbose:
print
# Return after fixing: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.prettyPrint:
print
print " GENERATION OF PRETTY PRINTED CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Pretty printing..."
else:
print " * Pretty printing: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
prettyFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), options)
if not prettyFileContent.endswith("\n"):
prettyFileContent += "\n"
filetool.save(fileDb[fileId]["path"], prettyFileContent)
if not options.verbose:
print
# Return after pretty print: Ignore other jobs
return
######################################################################
# INLINE CODE
######################################################################
inlineSourceCode = []
inlineCompiledCode = []
######################################################################
# SUPPORT FOR VARIANTS
######################################################################
if len(options.useVariant) > 0 and not options.generateSourceScript:
print
print " VARIANT OPTIMIZATION:"
print "----------------------------------------------------------------------------"
variantMap = {}
for variant in options.useVariant:
keyValue = variant.split(":")
if len(keyValue) != 2:
print " * Error: Variants must be specified as key value pair separated by ':'!"
sys.exit(1)
variantMap[keyValue[0]] = keyValue[1]
if options.verbose:
print " * Optimizing for variant setup..."
else:
print " * Optimizing for variant setup: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
if variantoptimizer.search(loader.getTree(fileDb, fileId, options), variantMap, fileId, options.verbose):
if options.verbose:
print " - Modified!"
if not options.verbose:
print
######################################################################
# BASE CALL OPTIMIZATION
######################################################################
if options.optimizeBaseCall:
print
print " BASE CALL OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing this.base calls..."
else:
print " * Optimizing this.base calls: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += basecalloptimizer.patch(loader.getTree(fileDb, fileId, options))
if not options.verbose:
print
print " * Optimized %s calls" % counter
######################################################################
# ACCESSOR OBFUSCATION
######################################################################
if options.obfuscateAccessors:
print
print " ACCESSOR OBFUSCATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Obfuscating..."
else:
print " * Obfuscating: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
accessorobfuscator.process(loader.getTree(fileDb, fileId, options), options.verbose)
if not options.verbose:
print
######################################################################
# STRING OPTIMIZATION
######################################################################
if options.optimizeStrings:
print
print " STRING OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching strings..."
else:
print " * Searching strings: ",
stringMap = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
localMap = stringoptimizer.search(loader.getTree(fileDb, fileId, options), options.verbose)
for value in localMap:
if value in stringMap:
stringMap[value] += localMap[value]
else:
stringMap[value] = localMap[value]
if not options.verbose:
print
counter = 0
for value in stringMap:
counter += stringMap[value]
stringList = stringoptimizer.sort(stringMap)
print " * Found %s string instances (%s unique)" % (counter, len(stringMap))
if options.verbose:
print " * Replacing strings..."
else:
print " * Replacing strings: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
stringoptimizer.replace(loader.getTree(fileDb, fileId, options), stringList, "$" + pkgid, options.verbose)
if not options.verbose:
print
print " * Generating replacement..."
inlineCompiledCode.append(stringoptimizer.replacement(stringList, "$" + pkgid))
######################################################################
# LOCAL VARIABLE OPTIMIZATION
######################################################################
if options.optimizeVariables or options.obfuscate:
print
print " LOCAL VARIABLE OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing variables..."
else:
print " * Optimizing variables: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += variableoptimizer.search(
loader.getTree(fileDb, fileId, options),
[], 0, 0, "$",
skipPrefix = options.optimizeVariablesSkipPrefix,
verbose = options.verbose
)
if not options.verbose:
print
print " * Optimized %s variables" % counter
######################################################################
# PRIVATE MEMBER OPTIMIZATION
######################################################################
if options.optimizePrivate:
print
print " PRIVATE MEMBER OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing private members..."
else:
print " * Optimizing private members: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += privateoptimizer.patch("",loader.getTree(fileDb, fileId, options), {}, "$", options.verbose)
if not options.verbose:
print
print " * Optimized %s keys" % counter
######################################################################
# OBFUSCATION
######################################################################
if options.obfuscate:
print
print " OBFUSCATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching for assignments..."
else:
print " * Searching for assignments: ",
identifiers = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
obfuscator.search(loader.getTree(fileDb, fileId, options), identifiers, options.verbose)
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
obfuscator.patch(loader.getTree(fileDb, fileId, options), identifiers, options.verbose)
if not options.verbose:
print
######################################################################
# TOKEN STORAGE
######################################################################
if options.storeTokens:
print
print " TOKEN STORAGE:"
print "----------------------------------------------------------------------------"
if options.tokenOutputDirectory == None:
print " * You must define the token output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tokens..."
else:
print " * Storing tokens: ",
for fileId in sortedIncludeList:
tokenString = tokenizer.convertTokensToString(loader.getTokens(fileDb, fileId, options))
if options.verbose:
print " * Writing tokens for %s (%s KB)..." % (fileIdm, len(tokenString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.tokenOutputDirectory), fileId + config.TOKENEXT), tokenString)
if not options.verbose:
print
######################################################################
# TREE STORAGE
######################################################################
if options.storeTree:
print
print " TREE STORAGE:"
print "----------------------------------------------------------------------------"
if options.treeOutputDirectory == None:
print " * You must define the tree output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tree..."
else:
print " * Storing tree: ",
for fileId in sortedIncludeList:
treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(loader.getTree(fileDb, fileId, options))
if options.verbose:
print " * Writing tree for %s (%s KB)..." % (fileId, len(treeString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.treeOutputDirectory), fileId + config.XMLEXT), treeString)
if not options.verbose:
print
######################################################################
# GENERATION OF API
######################################################################
if options.generateApiDocumentation:
print
print " GENERATION OF API:"
print "----------------------------------------------------------------------------"
if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
print " * You must define one of JSON or XML API documentation file!"
docTree = None
if options.verbose:
print " * Generating API tree..."
else:
print " * Generating API tree: ",
hasDocError = False
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
(docTree, error) = api.createDoc(loader.getTree(fileDb, fileId, options), docTree)
hasDocError = hasDocError or error
if hasDocError:
print
print " * Building API failed!!"
sys.exit(0)
if not options.verbose:
print
if docTree:
print " * Finalizing tree..."
api.postWorkPackage(docTree, docTree)
if options.addNewLines:
childPrefix = " "
newLine = "\n"
else:
childPrefix = ""
newLine = ""
if options.apiDocumentationXmlFile != None:
print " * Writing XML API file to %s" % options.apiDocumentationXmlFile
if options.apiSeparateFiles:
packages = api.packagesToXmlString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationXmlFile, packages, options.scriptOutputEncoding)
for cls in api.classNodeIterator(docTree):
classContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
classContent += tree.nodeToXmlString(cls, "", childPrefix, newLine)
fileName = os.path.join(os.path.dirname(options.apiDocumentationXmlFile), cls.get("fullName") + ".xml")
filetool.save(fileName, classContent, options.xmlOutputEncoding)
else:
xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
xmlContent += tree.nodeToXmlString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationXmlFile, xmlContent, options.xmlOutputEncoding)
if options.apiDocumentationJsonFile != None:
print " * Writing JSON API file to %s" % options.apiDocumentationJsonFile
if options.apiSeparateFiles:
packages = api.packagesToJsonString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationJsonFile, packages, options.scriptOutputEncoding)
for cls in api.classNodeIterator(docTree):
classContent = tree.nodeToJsonString(cls, "", childPrefix, newLine)
fileName = os.path.join(os.path.dirname(options.apiDocumentationJsonFile), cls.get("fullName") + ".js")
filetool.save(fileName, classContent, options.scriptOutputEncoding)
else:
jsonContent = tree.nodeToJsonString(docTree, "", childPrefix, newLine)
filetool.save(options.apiDocumentationJsonFile, jsonContent, options.scriptOutputEncoding)
# create search index
if options.apiDocumentationIndexFile != None:
print " * Generating API index..."
jsonContent = tree.nodeToIndexString(docTree, "", childPrefix, newLine)
print " * Writing API search index to %s" % options.apiDocumentationIndexFile
filetool.save(options.apiDocumentationIndexFile, jsonContent, options.scriptOutputEncoding)
######################################################################
# CREATE COPY OF RESOURCES
######################################################################
if options.copyResources:
print
print " CREATE COPY OF RESOURCES:"
print "----------------------------------------------------------------------------"
resources.copy(options, sortedIncludeList, fileDb)
######################################################################
# GENERATION OF SETTINGS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
print
print " INCLUSION OF SETTINGS:"
print "----------------------------------------------------------------------------"
if options.generateSourceScript:
print " * Processing settings of source version..."
settingsArr = options.useSetting[:]
settingsArr.append("qx.version:%s" % options.version)
settingsArr.append("qx.isSource:true")
inlineSourceCode.append(settings.generate(settingsArr, options.addNewLines))
if options.generateCompiledScript:
print " * Processing settings of compiled version..."
settingsArr = options.useSetting[:]
settingsArr.append("qx.version:%s" % options.version)
settingsArr.append("qx.isSource:false")
inlineCompiledCode.append(settings.generate(settingsArr, options.addNewLines))
######################################################################
# GENERATION OF VARIANTS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
print
print " INCLUSION OF VARIANTS:"
print "----------------------------------------------------------------------------"
if options.generateSourceScript:
print " * Processing variants of source version..."
variantsArr = options.useVariant[:]
inlineSourceCode.append(variants.generate(variantsArr, options.addNewLines))
if options.generateCompiledScript:
print " * Processing variants of compiled version..."
variantsArr = options.useVariant[:]
inlineCompiledCode.append(variants.generate(variantsArr, options.addNewLines))
######################################################################
# GENERATION OF SOURCE VERSION
######################################################################
if options.generateSourceScript:
print
print " GENERATION OF SOURCE SCRIPT:"
print "----------------------------------------------------------------------------"
if options.sourceScriptFile == None and (options.sourceTemplateInputFile == None or options.sourceTemplateOutputFile == None):
print " * You must define at least one source script file or template input/output."
sys.exit(1)
if options.sourceScriptFile:
options.sourceScriptFile = os.path.normpath(options.sourceScriptFile)
if options.sourceTemplateInputFile:
options.sourceTemplateInputFile = os.path.normpath(options.sourceTemplateInputFile)
if options.sourceTemplateOutputFile:
options.sourceTemplateOutputFile = os.path.normpath(options.sourceTemplateOutputFile)
print " * Generating script block..."
# Handling line feed setting
sourceLineFeed = "";
if options.addNewLines:
sourceLineFeed = "\n";
# Generating script block
scriptBlocks = ""
scriptBlocks += '<script type="text/javascript">%s</script>' % "".join(inlineSourceCode)
for fileId in sortedIncludeList:
if fileDb[fileId]["meta"]:
continue
if fileDb[fileId]["classUri"] == None:
print " * Missing class URI definition for class path %s." % fileDb[fileId]["classPath"]
sys.exit(1)
scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (os.path.join(fileDb[fileId]["classUri"], fileDb[fileId]["pathId"].replace(".", '/')), config.JSEXT)
scriptBlocks += sourceLineFeed
# Writing includer
if options.sourceScriptFile != None:
print " * Storing includer as %s..." % options.sourceScriptFile
sourceScript = "document.write('%s');" % scriptBlocks.replace("'", "\\'")
if options.addNewLines:
sourceScript = sourceScript.replace("\n", "\\\n")
filetool.save(options.sourceScriptFile, sourceScript, options.scriptOutputEncoding)
# Patching template
if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
print " * Patching template: %s => %s" % (options.sourceTemplateInputFile, options.sourceTemplateOutputFile)
tmpl = filetool.read(options.sourceTemplateInputFile)
res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
filetool.save(options.sourceTemplateOutputFile, res, options.scriptOutputEncoding)
######################################################################
# GENERATION OF COMPILED VERSION
######################################################################
if options.generateCompiledScript:
print
print " GENERATION OF COMPILED SCRIPT:"
print "----------------------------------------------------------------------------"
buildLineFeed = "";
if options.addNewLines:
buildLineFeed = "\n";
compiledOutput = "".join(inlineCompiledCode)
if options.compiledScriptFile == None:
print " * You must define the compiled script file!"
sys.exit(1)
if options.verbose:
print " * Compiling..."
else:
print " * Compiling: ",
for fileId in sortedIncludeList:
if fileDb[fileId]["meta"]:
continue
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
if options.prettyPrint:
options.prettyPrint = False # make sure it's disabled
compiledFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), options, options.addNewLines, options.verbose)
if options.addFileIds:
compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
else:
compiledOutput += compiledFileContent
if not compiledOutput.endswith(";") and not compiledOutput.endswith("\n"):
compiledOutput += ";"
if not options.verbose:
print
print " * Storing output as %s..." % options.compiledScriptFile
filetool.save(options.compiledScriptFile, compiledOutput, options.scriptOutputEncoding)
def argparser(cmdlineargs):
# Parse arguments
(options, args) = getparser().parse_args(cmdlineargs)
# Export to file
if options.exportToFile != None:
print
print " EXPORTING:"
print "----------------------------------------------------------------------------"
print " * Translating options..."
optionString = "# Exported configuration from build.py\n\n"
ignoreValue = True
lastWasKey = False
for arg in cmdlineargs:
if arg == "--export-to-file":
ignoreValue = True
elif arg.startswith("--"):
if lastWasKey:
optionString += "\n"
optionString += arg[2:]
ignoreValue = False
lastWasKey = True
elif arg.startswith("-"):
print " - Could not export short argument: %s" % arg
optionString += "\n# Ignored short argument %s\n" % arg
ignoreValue = True
elif not ignoreValue:
optionString += " = %s\n" % arg
ignoreValue = True
lastWasKey = False
print " * Export to file: %s" % options.exportToFile
filetool.save(options.exportToFile, optionString)
sys.exit(0)
# Read from file
elif options.fromFile != None:
print
print " INITIALIZATION:"
print "----------------------------------------------------------------------------"
print " * Reading configuration..."
# Convert file content into arguments
fileargs = {}
fileargpos = 0
fileargid = "default"
currentfileargs = []
fileargs[fileargid] = currentfileargs
alternativeFormatBegin = re.compile("\s*\[\s*")
alternativeFormatEnd = re.compile("\s*\]\s*=\s*")
emptyLine = re.compile("^\s*$")
for line in file(options.fromFile).read().split("\n"):
line = line.strip()
if emptyLine.match(line) or line.startswith("#") or line.startswith("//"):
continue
# Translating...
line = alternativeFormatBegin.sub(" = ", line)
line = alternativeFormatEnd.sub(":", line)
# Splitting line
line = line.split("=")
# Extract key element
key = line.pop(0).strip()
# Separate packages
if key == "package":
fileargpos += 1
fileargid = line[0].strip()
print " - Found package: %s" % fileargid
currentfileargs = []
fileargs[fileargid] = currentfileargs
continue
currentfileargs.append("--%s" % key)
if len(line) > 0:
value = line[0].strip()
currentfileargs.append(value)
# Parse
defaultargs = fileargs["default"]
if len(fileargs) > 1:
(fileDb, moduleDb) = load(getparser().parse_args(defaultargs)[0])
if options.obfuscateIdentifiers:
sharednames = {}
for filearg in fileargs:
if filearg == "default":
continue
combinedargs = []
combinedargs.extend(defaultargs)
combinedargs.extend(fileargs[filearg])
options = getparser().parse_args(defaultargs)[0]
findnames(fileDb, moduleDb, options, sharednames)
names = obfuscator.sort(sharednames)
for filearg in fileargs:
if filearg == "default":
continue
print
print
print
print
print " PACKAGE: %s" % filearg
print "----------------------------------------------------------------------------"
combinedargs = []
combinedargs.extend(defaultargs)
combinedargs.extend(fileargs[filearg])
options = getparser().parse_args(combinedargs)[0]
if options.obfuscateIdentifiers:
execute(fileDb, moduleDb, options, filearg, names)
else:
execute(fileDb, moduleDb, options, filearg)
else:
options = getparser().parse_args(defaultargs)[0]
(fileDb, moduleDb) = load(options)
if options.obfuscateIdentifiers:
execute(fileDb, moduleDb, options, "", obfuscator.sort(findnames(fileDb, moduleDb, options)))
else:
execute(fileDb, moduleDb, options, "")
else:
print
print " INITIALIZATION:"
print "----------------------------------------------------------------------------"
print " * Processing arguments..."
(fileDb, moduleDb) = load(options)
if options.obfuscateIdentifiers:
execute(fileDb, moduleDb, options, options.packageId, obfuscator.sort(findnames(fileDb, moduleDb, options)))
else:
execute(fileDb, moduleDb, options, options.packageId)
def execute(fileDb, moduleDb, options, pkgid="", names=[]):
additionalOutput = []
######################################################################
# SORT OF INCLUDE LIST
######################################################################
print
print " SORT OF INCLUDE LIST:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Include (with dependencies): %s" % options.includeWithDeps
print " * Include (without dependencies): %s" % options.includeWithoutDeps
print " * Exclude (with dependencies): %s" % options.excludeWithDeps
print " * Exclude (without dependencies): %s" % options.excludeWithoutDeps
print " * Sorting %s classes..." % len(fileDb)
sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)
if len(sortedIncludeList) == 0:
print " - No class files to include. Exciting!"
sys.exit(1)
else:
print " - Including %s classes" % len(sortedIncludeList)
if options.printIncludes:
print
print " PRINT OF INCLUDE ORDER:"
print "----------------------------------------------------------------------------"
print " * The files will be included in this order:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if options.printDeps:
print
print " OUTPUT OF DEPENDENCIES:"
print "----------------------------------------------------------------------------"
print " * These are all included files with their dependencies:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
print " - Loadtime: "
for depEntry in fileDb[fileId]["loadtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["afterDeps"]) > 0:
print " - After: "
for depEntry in fileDb[fileId]["afterDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["runtimeDeps"]) > 0:
print " - Runtime: "
for depEntry in fileDb[fileId]["runtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["loadDeps"]) > 0:
print " - Load: "
for depEntry in fileDb[fileId]["loadDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["optionalDeps"]) > 0:
print " - Optional: "
for depEntry in fileDb[fileId]["optionalDeps"]:
print " - %s" % depEntry
######################################################################
# GRAPHVIZ OUTPUT
######################################################################
if options.depDotFile:
graph.store(fileDb, sortedIncludeList, options)
######################################################################
# SOURCE MIGRATION
######################################################################
if options.migrateSource:
print
print " SOURCE MIGRATION:"
print "----------------------------------------------------------------------------"
print " * Migrate Source Code..."
migrator.handle(sortedIncludeList, fileDb, options)
# Return after migration: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.fixSource:
print
print " FIX SOURCE CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Fixing code..."
else:
print " * Fixing code: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Reading %s" % fileId
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
fileContent = filetool.read(filePath, fileEncoding)
fixedContent = textutil.removeTrailingSpaces(textutil.tab2Space(textutil.any2Unix(fileContent), 2))
if fixedContent != fileContent:
if options.verbose:
print " - Storing modifications..."
else:
sys.stdout.write("!")
sys.stdout.flush()
filetool.save(filePath, fixedContent, fileEncoding)
elif not options.verbose:
sys.stdout.write(".")
sys.stdout.flush()
if not options.verbose:
print
# Return after fixing: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.prettyPrint:
print
print " GENERATION OF PRETTY PRINTED CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Pretty printing..."
else:
print " * Pretty printing: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
prettyFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), True)
if not prettyFileContent.endswith("\n"):
prettyFileContent += "\n"
filetool.save(fileDb[fileId]["path"], prettyFileContent)
if not options.verbose:
print
# Return after pretty print: Ignore other jobs
return
######################################################################
# STRING OPTIMIZATION
######################################################################
if options.optimizeStrings:
print
print " STRING OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching strings..."
else:
print " * Searching strings: ",
stringMap = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
localMap = loader.getStrings(fileDb, fileId, options)
for value in localMap:
if value in stringMap:
stringMap[value] += localMap[value]
else:
stringMap[value] = localMap[value]
if not options.verbose:
print
counter = 0
for value in stringMap:
counter += stringMap[value]
stringList = stringoptimizer.sort(stringMap)
print " * Found %s strings (used %s times)" % (len(stringMap), counter)
if options.verbose:
print " * Replacing strings..."
else:
print " * Replacing strings: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
stringoptimizer.replace(loader.getTree(fileDb, fileId, options), stringList, "$" + pkgid, options.verbose)
if not options.verbose:
print
print " * Generating replacement..."
additionalOutput.append(stringoptimizer.replacement(stringList, "$" + pkgid))
######################################################################
# LOCAL VARIABLE OPTIMIZATION
######################################################################
if options.optimizeVariables:
print
print " LOCAL VARIABLE OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing variables..."
else:
print " * Optimizing variables: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
variableoptimizer.search(loader.getTree(fileDb, fileId, options), [], 0, "$", skipPrefix = options.optimizeVariablesSkipPrefix, debug = options.enableDebug)
if not options.verbose:
print
######################################################################
# NAME OBFUSCATION
######################################################################
if options.obfuscateIdentifiers:
print
print " OBFUSCATE IDENTIFIERS:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Obfuscating identifiers..."
else:
print " * Obfuscating identifiers: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += obfuscator.update(loader.getTree(fileDb, fileId, options), names, "$$")
if not options.verbose:
print
print " * Updated %s names" % counter
######################################################################
# TOKEN STORAGE
######################################################################
if options.storeTokens:
print
print " TOKEN STORAGE:"
print "----------------------------------------------------------------------------"
if options.tokenOutputDirectory == None:
print " * You must define the token output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tokens..."
else:
print " * Storing tokens: ",
for fileId in sortedIncludeList:
tokenString = tokenizer.convertTokensToString(loader.getTokens(fileDb, fileId, options))
if options.verbose:
print " * writing tokens for %s (%s KB)..." % (fileIdm, len(tokenString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.tokenOutputDirectory), fileId + config.TOKENEXT), tokenString)
if not options.verbose:
print
######################################################################
# TREE STORAGE
######################################################################
if options.storeTree:
print
print " TREE STORAGE:"
print "----------------------------------------------------------------------------"
if options.treeOutputDirectory == None:
print " * You must define the tree output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tree..."
else:
print " * Storing tree: ",
for fileId in sortedIncludeList:
treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(loader.getTree(fileDb, fileId, options))
if options.verbose:
print " * writing tree for %s (%s KB)..." % (fileId, len(treeString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(os.path.join(filetool.normalize(options.treeOutputDirectory), fileId + config.XMLEXT), treeString)
if not options.verbose:
print
######################################################################
# GENERATION OF API
######################################################################
if options.generateApiDocumentation:
print
print " GENERATION OF API:"
print "----------------------------------------------------------------------------"
if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
print " * You must define one of JSON or XML API documentation file!"
docTree = None
if options.verbose:
print " * Generating API tree..."
else:
print " * Generating API tree: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
docTree = api.createDoc(loader.getTree(fileDb, fileId, options), docTree)
if not options.verbose:
print
if docTree:
print " * Finalizing tree..."
api.postWorkPackage(docTree, docTree)
if options.apiDocumentationXmlFile != None:
print " * Writing XML API file to %s" % options.apiDocumentationXmlFile
xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
if options.addNewLines:
xmlContent += "\n" + tree.nodeToXmlString(docTree)
else:
xmlContent += tree.nodeToXmlString(docTree, "", "", "")
filetool.save(options.apiDocumentationXmlFile, xmlContent, options.xmlOutputEncoding)
if options.apiDocumentationJsonFile != None:
print " * Writing JSON API file to %s" % options.apiDocumentationJsonFile
if options.addNewLines:
jsonContent = tree.nodeToJsonString(docTree)
else:
jsonContent = tree.nodeToJsonString(docTree, "", "", "")
filetool.save(options.apiDocumentationJsonFile, jsonContent, options.scriptOutputEncoding)
######################################################################
# CREATE COPY OF RESOURCES
######################################################################
if options.copyResources:
print
print " CREATE COPY OF RESOURCES:"
print "----------------------------------------------------------------------------"
resources.copy(options, sortedIncludeList, fileDb)
######################################################################
# GENERATION OF SETTINGS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
settingsStr = ""
if len(options.defineRuntimeSetting) != 0:
print
print " GENERATION OF SETTINGS:"
print "----------------------------------------------------------------------------"
print " * Processing input data..."
settingsStr = settings.generate(options)
if options.settingsScriptFile:
print " * Storing result to %s" % options.settingsScriptFile
filetool.save(options.settingsScriptFile, settingsStr)
# clear settings for build and source
settingsStr = ""
######################################################################
# GENERATION OF SOURCE VERSION
######################################################################
if options.generateSourceScript:
print
print " GENERATION OF SOURCE SCRIPT:"
print "----------------------------------------------------------------------------"
if options.sourceScriptFile == None and (options.sourceTemplateInputFile == None or options.sourceTemplateOutputFile == None):
print " * You must define at least one source script file or template input/output."
sys.exit(1)
if options.sourceScriptFile:
options.sourceScriptFile = os.path.normpath(options.sourceScriptFile)
if options.sourceTemplateInputFile:
options.sourceTemplateInputFile = os.path.normpath(options.sourceTemplateInputFile)
if options.sourceTemplateOutputFile:
options.sourceTemplateOutputFile = os.path.normpath(options.sourceTemplateOutputFile)
print " * Generating script block..."
# Handling line feed setting
sourceLineFeed = "";
if options.addNewLines:
sourceLineFeed = "\n";
# Generating inline code...
inlineCode = ""
inlineCode += settingsStr + sourceLineFeed
inlineCode += "qx.IS_SOURCE=true;%s" % sourceLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, sourceLineFeed)
inlineCode += "".join(additionalOutput)
# Generating script block
scriptBlocks = ""
scriptBlocks += '<script type="text/javascript">%s</script>' % inlineCode
for fileId in sortedIncludeList:
if fileDb[fileId]["classUri"] == None:
print " * Missing class URI definition for class path %s." % fileDb[fileId]["classPath"]
sys.exit(1)
scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (os.path.join(fileDb[fileId]["classUri"], fileDb[fileId]["pathId"].replace(".", os.sep)), config.JSEXT)
scriptBlocks += sourceLineFeed
if options.sourceScriptFile != None:
print " * Storing includer as %s..." % options.sourceScriptFile
sourceScript = "document.write('%s');" % scriptBlocks.replace("'", "\\'")
if options.addNewLines:
sourceScript = sourceScript.replace("\n", "\\\n")
filetool.save(options.sourceScriptFile, sourceScript, options.scriptOutputEncoding)
if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
print " * Patching template: %s => %s" % (options.sourceTemplateInputFile, options.sourceTemplateOutputFile)
tmpl = filetool.read(options.sourceTemplateInputFile)
res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
filetool.save(options.sourceTemplateOutputFile, res, options.scriptOutputEncoding)
######################################################################
# GENERATION OF COMPILED VERSION
######################################################################
if options.generateCompiledScript:
print
print " GENERATION OF COMPILED SCRIPT:"
print "----------------------------------------------------------------------------"
buildLineFeed = "";
if options.addNewLines:
buildLineFeed = "\n";
inlineCode = ""
inlineCode += settingsStr + buildLineFeed
inlineCode += "qx.IS_SOURCE=false;%s" % buildLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, buildLineFeed)
inlineCode += "".join(additionalOutput)
compiledOutput = inlineCode
if options.compiledScriptFile == None:
print " * You must define the compiled script file!"
sys.exit(1)
if options.verbose:
print " * Compiling..."
else:
print " * Compiling: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
compiledFileContent = compiler.compile(loader.getTree(fileDb, fileId, options), False, options.addNewLines, options.enableDebug)
if options.addFileIds:
compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
else:
compiledOutput += compiledFileContent
if not compiledOutput.endswith(";") and not compiledOutput.endswith("\n"):
compiledOutput += ";"
if not options.verbose:
print
print " * Storing output as %s..." % options.compiledScriptFile
filetool.save(options.compiledScriptFile, compiledOutput, options.scriptOutputEncoding)
def execute(fileDb, moduleDb, options, pkgid="", names=[]):
additionalOutput = []
######################################################################
# SORT OF INCLUDE LIST
######################################################################
print
print " SORT OF INCLUDE LIST:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Include (with dependencies): %s" % options.includeWithDeps
print " * Include (without dependencies): %s" % options.includeWithoutDeps
print " * Exclude (with dependencies): %s" % options.excludeWithDeps
print " * Exclude (without dependencies): %s" % options.excludeWithoutDeps
print " * Sorting %s classes..." % len(fileDb)
sortedIncludeList = loader.getSortedList(options, fileDb, moduleDb)
if len(sortedIncludeList) == 0:
print " - No class files to include. Exciting!"
sys.exit(1)
else:
print " - Including %s classes" % len(sortedIncludeList)
if options.printIncludes:
print
print " PRINT OF INCLUDE ORDER:"
print "----------------------------------------------------------------------------"
print " * The files will be included in this order:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if options.printDeps:
print
print " OUTPUT OF DEPENDENCIES:"
print "----------------------------------------------------------------------------"
print " * These are all included files with their dependencies:"
for fileId in sortedIncludeList:
print " - %s" % fileId
if len(fileDb[fileId]["loadtimeDeps"]) > 0:
print " - Loadtime: "
for depEntry in fileDb[fileId]["loadtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["afterDeps"]) > 0:
print " - After: "
for depEntry in fileDb[fileId]["afterDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["runtimeDeps"]) > 0:
print " - Runtime: "
for depEntry in fileDb[fileId]["runtimeDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["loadDeps"]) > 0:
print " - Load: "
for depEntry in fileDb[fileId]["loadDeps"]:
print " - %s" % depEntry
if len(fileDb[fileId]["optionalDeps"]) > 0:
print " - Optional: "
for depEntry in fileDb[fileId]["optionalDeps"]:
print " - %s" % depEntry
######################################################################
# GRAPHVIZ OUTPUT
######################################################################
if options.depDotFile:
graph.store(fileDb, sortedIncludeList, options)
######################################################################
# SOURCE MIGRATION
######################################################################
if options.migrateSource:
print
print " SOURCE MIGRATION:"
print "----------------------------------------------------------------------------"
print " * Migrate Source Code..."
migrator.handle(sortedIncludeList, fileDb, options)
# Return after migration: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.fixSource:
print
print " FIX SOURCE CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Fixing code..."
else:
print " * Fixing code: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Reading %s" % fileId
fileEntry = fileDb[fileId]
filePath = fileEntry["path"]
fileEncoding = fileEntry["encoding"]
fileContent = filetool.read(filePath, fileEncoding)
fixedContent = textutil.removeTrailingSpaces(
textutil.tab2Space(textutil.any2Unix(fileContent), 2))
if fixedContent != fileContent:
if options.verbose:
print " - Storing modifications..."
else:
sys.stdout.write("!")
sys.stdout.flush()
filetool.save(filePath, fixedContent, fileEncoding)
elif not options.verbose:
sys.stdout.write(".")
sys.stdout.flush()
if not options.verbose:
print
# Return after fixing: Ignore other jobs
return
######################################################################
# GENERATION OF PRETTY PRINTED CODE
######################################################################
if options.prettyPrint:
print
print " GENERATION OF PRETTY PRINTED CODE:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Pretty printing..."
else:
print " * Pretty printing: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
prettyFileContent = compiler.compile(
loader.getTree(fileDb, fileId, options), True)
if not prettyFileContent.endswith("\n"):
prettyFileContent += "\n"
filetool.save(fileDb[fileId]["path"], prettyFileContent)
if not options.verbose:
print
# Return after pretty print: Ignore other jobs
return
######################################################################
# STRING OPTIMIZATION
######################################################################
if options.optimizeStrings:
print
print " STRING OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Searching strings..."
else:
print " * Searching strings: ",
stringMap = {}
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
localMap = loader.getStrings(fileDb, fileId, options)
for value in localMap:
if value in stringMap:
stringMap[value] += localMap[value]
else:
stringMap[value] = localMap[value]
if not options.verbose:
print
counter = 0
for value in stringMap:
counter += stringMap[value]
stringList = stringoptimizer.sort(stringMap)
print " * Found %s strings (used %s times)" % (len(stringMap),
counter)
if options.verbose:
print " * Replacing strings..."
else:
print " * Replacing strings: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
stringoptimizer.replace(loader.getTree(fileDb, fileId, options),
stringList, "$" + pkgid, options.verbose)
if not options.verbose:
print
print " * Generating replacement..."
additionalOutput.append(
stringoptimizer.replacement(stringList, "$" + pkgid))
######################################################################
# LOCAL VARIABLE OPTIMIZATION
######################################################################
if options.optimizeVariables:
print
print " LOCAL VARIABLE OPTIMIZATION:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Optimizing variables..."
else:
print " * Optimizing variables: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
variableoptimizer.search(
loader.getTree(fileDb, fileId, options), [],
0,
"$",
skipPrefix=options.optimizeVariablesSkipPrefix,
debug=options.enableDebug)
if not options.verbose:
print
######################################################################
# NAME OBFUSCATION
######################################################################
if options.obfuscateIdentifiers:
print
print " OBFUSCATE IDENTIFIERS:"
print "----------------------------------------------------------------------------"
if options.verbose:
print " * Obfuscating identifiers..."
else:
print " * Obfuscating identifiers: ",
counter = 0
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
counter += obfuscator.update(
loader.getTree(fileDb, fileId, options), names, "$$")
if not options.verbose:
print
print " * Updated %s names" % counter
######################################################################
# TOKEN STORAGE
######################################################################
if options.storeTokens:
print
print " TOKEN STORAGE:"
print "----------------------------------------------------------------------------"
if options.tokenOutputDirectory == None:
print " * You must define the token output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tokens..."
else:
print " * Storing tokens: ",
for fileId in sortedIncludeList:
tokenString = tokenizer.convertTokensToString(
loader.getTokens(fileDb, fileId, options))
if options.verbose:
print " * writing tokens for %s (%s KB)..." % (
fileIdm, len(tokenString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(
os.path.join(filetool.normalize(options.tokenOutputDirectory),
fileId + config.TOKENEXT), tokenString)
if not options.verbose:
print
######################################################################
# TREE STORAGE
######################################################################
if options.storeTree:
print
print " TREE STORAGE:"
print "----------------------------------------------------------------------------"
if options.treeOutputDirectory == None:
print " * You must define the tree output directory!"
sys.exit(1)
if options.verbose:
print " * Storing tree..."
else:
print " * Storing tree: ",
for fileId in sortedIncludeList:
treeString = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n" + tree.nodeToXmlString(
loader.getTree(fileDb, fileId, options))
if options.verbose:
print " * writing tree for %s (%s KB)..." % (
fileId, len(treeString) / 1000.0)
else:
sys.stdout.write(".")
sys.stdout.flush()
filetool.save(
os.path.join(filetool.normalize(options.treeOutputDirectory),
fileId + config.XMLEXT), treeString)
if not options.verbose:
print
######################################################################
# GENERATION OF API
######################################################################
if options.generateApiDocumentation:
print
print " GENERATION OF API:"
print "----------------------------------------------------------------------------"
if options.apiDocumentationJsonFile == None and options.apiDocumentationXmlFile == None:
print " * You must define one of JSON or XML API documentation file!"
docTree = None
if options.verbose:
print " * Generating API tree..."
else:
print " * Generating API tree: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
docTree = api.createDoc(loader.getTree(fileDb, fileId, options),
docTree)
if not options.verbose:
print
if docTree:
print " * Finalizing tree..."
api.postWorkPackage(docTree, docTree)
if options.apiDocumentationXmlFile != None:
print " * Writing XML API file to %s" % options.apiDocumentationXmlFile
xmlContent = "<?xml version=\"1.0\" encoding=\"" + options.xmlOutputEncoding + "\"?>\n"
if options.addNewLines:
xmlContent += "\n" + tree.nodeToXmlString(docTree)
else:
xmlContent += tree.nodeToXmlString(docTree, "", "", "")
filetool.save(options.apiDocumentationXmlFile, xmlContent,
options.xmlOutputEncoding)
if options.apiDocumentationJsonFile != None:
print " * Writing JSON API file to %s" % options.apiDocumentationJsonFile
if options.addNewLines:
jsonContent = tree.nodeToJsonString(docTree)
else:
jsonContent = tree.nodeToJsonString(docTree, "", "", "")
filetool.save(options.apiDocumentationJsonFile, jsonContent,
options.scriptOutputEncoding)
######################################################################
# CREATE COPY OF RESOURCES
######################################################################
if options.copyResources:
print
print " CREATE COPY OF RESOURCES:"
print "----------------------------------------------------------------------------"
resources.copy(options, sortedIncludeList, fileDb)
######################################################################
# GENERATION OF SETTINGS
######################################################################
if options.generateSourceScript or options.generateCompiledScript:
settingsStr = ""
if len(options.defineRuntimeSetting) != 0:
print
print " GENERATION OF SETTINGS:"
print "----------------------------------------------------------------------------"
print " * Processing input data..."
settingsStr = settings.generate(options)
if options.settingsScriptFile:
print " * Storing result to %s" % options.settingsScriptFile
filetool.save(options.settingsScriptFile, settingsStr)
# clear settings for build and source
settingsStr = ""
######################################################################
# GENERATION OF SOURCE VERSION
######################################################################
if options.generateSourceScript:
print
print " GENERATION OF SOURCE SCRIPT:"
print "----------------------------------------------------------------------------"
if options.sourceScriptFile == None and (
options.sourceTemplateInputFile == None
or options.sourceTemplateOutputFile == None):
print " * You must define at least one source script file or template input/output."
sys.exit(1)
if options.sourceScriptFile:
options.sourceScriptFile = os.path.normpath(
options.sourceScriptFile)
if options.sourceTemplateInputFile:
options.sourceTemplateInputFile = os.path.normpath(
options.sourceTemplateInputFile)
if options.sourceTemplateOutputFile:
options.sourceTemplateOutputFile = os.path.normpath(
options.sourceTemplateOutputFile)
print " * Generating script block..."
# Handling line feed setting
sourceLineFeed = ""
if options.addNewLines:
sourceLineFeed = "\n"
# Generating inline code...
inlineCode = ""
inlineCode += settingsStr + sourceLineFeed
inlineCode += "qx.IS_SOURCE=true;%s" % sourceLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version,
sourceLineFeed)
inlineCode += "".join(additionalOutput)
# Generating script block
scriptBlocks = ""
scriptBlocks += '<script type="text/javascript">%s</script>' % inlineCode
for fileId in sortedIncludeList:
if fileDb[fileId]["classUri"] == None:
print " * Missing class URI definition for class path %s." % fileDb[
fileId]["classPath"]
sys.exit(1)
scriptBlocks += '<script type="text/javascript" src="%s%s"></script>' % (
os.path.join(fileDb[fileId]["classUri"],
fileDb[fileId]["pathId"].replace(
".", os.sep)), config.JSEXT)
scriptBlocks += sourceLineFeed
if options.sourceScriptFile != None:
print " * Storing includer as %s..." % options.sourceScriptFile
sourceScript = "document.write('%s');" % scriptBlocks.replace(
"'", "\\'")
if options.addNewLines:
sourceScript = sourceScript.replace("\n", "\\\n")
filetool.save(options.sourceScriptFile, sourceScript,
options.scriptOutputEncoding)
if options.sourceTemplateInputFile != None and options.sourceTemplateOutputFile != None:
print " * Patching template: %s => %s" % (
options.sourceTemplateInputFile,
options.sourceTemplateOutputFile)
tmpl = filetool.read(options.sourceTemplateInputFile)
res = tmpl.replace(options.sourceTemplateReplace, scriptBlocks)
filetool.save(options.sourceTemplateOutputFile, res,
options.scriptOutputEncoding)
######################################################################
# GENERATION OF COMPILED VERSION
######################################################################
if options.generateCompiledScript:
print
print " GENERATION OF COMPILED SCRIPT:"
print "----------------------------------------------------------------------------"
buildLineFeed = ""
if options.addNewLines:
buildLineFeed = "\n"
inlineCode = ""
inlineCode += settingsStr + buildLineFeed
inlineCode += "qx.IS_SOURCE=false;%s" % buildLineFeed
inlineCode += "qx.VERSION=\"%s\";%s" % (options.version, buildLineFeed)
inlineCode += "".join(additionalOutput)
compiledOutput = inlineCode
if options.compiledScriptFile == None:
print " * You must define the compiled script file!"
sys.exit(1)
if options.verbose:
print " * Compiling..."
else:
print " * Compiling: ",
for fileId in sortedIncludeList:
if options.verbose:
print " - Compiling %s" % fileId
else:
sys.stdout.write(".")
sys.stdout.flush()
compiledFileContent = compiler.compile(
loader.getTree(fileDb, fileId, options), False,
options.addNewLines, options.enableDebug)
if options.addFileIds:
compiledOutput += "\n\n\n/* ID: " + fileId + " */\n" + compiledFileContent + "\n"
else:
compiledOutput += compiledFileContent
if not compiledOutput.endswith(
";") and not compiledOutput.endswith("\n"):
compiledOutput += ";"
if not options.verbose:
print
print " * Storing output as %s..." % options.compiledScriptFile
filetool.save(options.compiledScriptFile, compiledOutput,
options.scriptOutputEncoding)