def writeI18NFiles(self, globalCodes, script):
# for each locale code, collect mappings
transKeys = globalCodes['Translations'].keys()
localeKeys = globalCodes['Locales'].keys()
newParts = {} # language codes to part objects, {"C": part}
newPackages= {} # language codes to private package objects, {"C": package}
for localeCode in set(transKeys + localeKeys):
# new: also provide a localeCode "part" with corresponding packages
part = Part(localeCode)
part.bit_mask = script.getPartBitMask()
newParts[localeCode] = part
package = Package(part.bit_mask) # this might be modified later
newPackages[localeCode] = package
part.packages.append(package)
data = {}
data[localeCode] = { 'Translations': {}, 'Locales': {} } # we want to have the locale code in the data
if localeCode in transKeys:
data[localeCode]['Translations'] = globalCodes['Translations'][localeCode]
package.data.translations[localeCode] = globalCodes['Translations'][localeCode]
if localeCode in localeKeys:
data[localeCode]['Locales'] = globalCodes['Locales'][localeCode]
package.data.locales[localeCode] = globalCodes['Locales'][localeCode]
# write to file
#dataS = json.dumpsCode(data)
hash, dataS = package.packageContent() # TODO: this currently works only for pure data packages
dataS = dataS.replace('\\\\\\', '\\').replace(r'\\', '\\') # undo damage done by simplejson to raw strings with escapes \\ -> \
package.compiled = dataS
fPath = self._resolveFileName(script.baseScriptPath, script.variants, script.settings, localeCode)
self.writePackage(dataS, fPath, script)
package.file = os.path.basename(fPath) # TODO: the use of os.path.basename is a hack
package.hash = hash
# Finalize the new packages and parts
# - add prerequisite languages to parts; e.g. ["C", "en", "en_EN"]
for partId, part in newParts.items():
if newPackages["C"] not in part.packages:
package = newPackages["C"]
part.packages.append(package) # all need "C"
package.id |= part.bit_mask # adapt package's bit string
if len(partId) > 2 and partId[2] == "_": # it's a sub-language -> include main language
mainlang = partId[:2]
if mainlang not in newPackages:
raise RuntimeError("Locale '%s' specified, but not base locale '%s'" % (partId, mainlang))
if newPackages[mainlang] not in part.packages:
part.packages.append(newPackages[mainlang]) # add main language
newPackages[mainlang].id |= part.bit_mask # adapt package's bit string
# finally, sort packages
for part in newParts.values():
part.packagesSorted
# - add to script object
script.parts.update([(x.name, x) for x in newParts.values()]) # TODO: update might overwrite exist. entries!
script.packages.extend(newPackages.values())
return globalCodes
def _findMergeTarget(self, mergePackage, packages):
##
# if another package id has the same bits turned on, it is available
# in the same parts.
def areInSameParts(mergePackage, package):
return (mergePackage.id & package.id) == mergePackage.id
##
# check if any of the deps of mergePackage depend on targetPackage -
# if merging mergePackage into targetPackage, this would be creating
# circular dependencies
def noCircularDeps(mergePackage, targetPackage):
for package in mergePackage.packageDeps:
if targetPackage in package.packageDeps:
return False
return True
##
# check that the targetPackage is loaded in those parts
# where mergePackage's deps are loaded
def depsAvailWhereTarget(mergePackage, targetPackage):
for depsPackage in mergePackage.packageDeps:
if not areInSameParts(targetPackage, depsPackage):
return False
return True
# ----------------------------------------------------------------------
allPackages = reversed(Package.sort(packages))
# sorting and reversing assures we try "smaller" package id's first
addtl_merge_constraints = self._jobconf.get(
"packages/additional-merge-constraints", True)
for targetPackage in allPackages:
if mergePackage.id == targetPackage.id: # no self-merging ;)
continue
if not areInSameParts(mergePackage, targetPackage):
self._console.debug("Problematic #%d (different parts using)" %
targetPackage.id)
continue
if not noCircularDeps(mergePackage, targetPackage):
self._console.debug("Problematic #%d (circular dependencies)" %
targetPackage.id)
if addtl_merge_constraints:
continue
# why accept this by default?
if not depsAvailWhereTarget(mergePackage, targetPackage):
self._console.debug(
"Problematic #%d (dependencies not always available)" %
targetPackage.id)
if addtl_merge_constraints:
continue
# why accept this by default?
yield targetPackage
yield None
def _getPackages(self, script):
self._console.indent()
parts = script.parts
# Generating list of all classes
allClasses = {}
for part in parts.values():
for classId in part.deps:
allClasses[classId] = True
# Check for each class which part is using it;
# create a package for each set of classes which
# are used by the same combination of parts;
# track how many parts are using a particular package
packages = {}
for classId in allClasses.keys():
pkgId = 0
for part in parts.values():
if classId in part.deps:
pkgId |= part.bit_mask
if not packages.has_key(pkgId):
package = Package(pkgId)
packages[pkgId] = package
packages[pkgId].classes.append(classId)
# Which packages does a part use - and vice versa
for package in packages.values():
for part in parts.values():
if package.id & part.bit_mask:
#part.packages.append(package.id)
part.packages.append(package)
#package.parts.append(part.name)
#package.parts.append(part)
#package.part_count = len(package.parts)
# Register dependencies between Packages
for package in packages.values():
# get all direct deps of this package
allDeps = set(())
for classId in package.classes:
classDeps, _ = self._depLoader.getCombinedDeps(
classId, script.variants, script.buildType)
loadDeps = set(x.name for x in classDeps['load'])
allDeps.update(loadDeps)
# record the other packages in which these dependencies are located
for classId in allDeps:
for otherpackage in packages.values():
if otherpackage != package and classId in otherpackage.classes:
#print "-- package %s adding dependent package %s" % (package.id, otherpackage.id)
package.packageDeps.add(otherpackage)
self._console.outdent()
return packages.values()
def getPackagesFromClasses(allClasses):
packages = {}
for classId in allClasses:
pkgId = allClasses[classId]
# create a Package if necessary
if pkgId not in packages:
package = Package(pkgId)
packages[pkgId] = package
# store classId with this package
packages[pkgId].classes.append(classId)
return packages.values()
def _findMergeTarget(self, mergePackage, packages):
##
# if another package id has the same bits turned on, it is available
# in the same parts.
def areInSameParts(mergePackage, package):
return (mergePackage.id & package.id) == mergePackage.id
##
# check if any of the deps of mergePackage depend on targetPackage -
# if merging mergePackage into targetPackage, this would be creating
# circular dependencies
def noCircularDeps(mergePackage, targetPackage):
for package in mergePackage.packageDeps:
if targetPackage in package.packageDeps:
return False
return True
##
# check that the targetPackage is loaded in those parts
# where mergePackage's deps are loaded
def depsAvailWhereTarget (mergePackage, targetPackage):
for depsPackage in mergePackage.packageDeps:
if not areInSameParts(targetPackage, depsPackage):
return False
return True
# ----------------------------------------------------------------------
allPackages = reversed(Package.sort(packages))
# sorting and reversing assures we try "smaller" package id's first
addtl_merge_constraints = self._jobconf.get("packages/additional-merge-constraints", False)
for targetPackage in allPackages:
if mergePackage.id == targetPackage.id: # no self-merging ;)
continue
if not areInSameParts(mergePackage, targetPackage):
self._console.debug("Problematic #%d (different parts using)" % targetPackage.id)
continue
if not noCircularDeps(mergePackage, targetPackage):
self._console.debug("Problematic #%d (circular dependencies)" % targetPackage.id)
if addtl_merge_constraints:
continue
# why accept this by default?
if not depsAvailWhereTarget(mergePackage, targetPackage):
self._console.debug("Problematic #%d (dependencies not always available)" % targetPackage.id)
if addtl_merge_constraints:
continue
# why accept this by default?
yield targetPackage
yield None
def packagesSortedSimple(self):
return Package.simpleSort(self.packages)
def generateI18NParts(self, script, globalCodes):
# for each locale code, collect mappings
transKeys = globalCodes['Translations'].keys()
localeKeys = globalCodes['Locales'].keys()
newParts = {} # language codes to part objects, {"C": part}
newPackages= {} # language codes to private package objects, {"C": package}
for localeCode in set(transKeys + localeKeys):
# new: also provide a localeCode "part" with corresponding packages
part = Part(localeCode)
part.bit_mask = script.getPartBitMask()
newParts[localeCode] = part
package = Package(part.bit_mask) # this might be modified later
newPackages[localeCode] = package
part.packages.append(package)
data = {}
data[localeCode] = { 'Translations': {}, 'Locales': {} } # we want to have the locale code in the data
if localeCode in transKeys:
data[localeCode]['Translations'] = globalCodes['Translations'][localeCode]
package.data.translations[localeCode] = globalCodes['Translations'][localeCode]
if localeCode in localeKeys:
data[localeCode]['Locales'] = globalCodes['Locales'][localeCode]
package.data.locales[localeCode] = globalCodes['Locales'][localeCode]
# file name and hash code
hash, dataS = package.packageContent() # TODO: this currently works only for pure data packages
dataS = dataS.replace('\\\\\\', '\\').replace(r'\\', '\\') # undo damage done by simplejson to raw strings with escapes \\ -> \
package.compiled = dataS
package.hash = hash
fPath = self._resolveFileName(script.baseScriptPath, script.variants, script.settings, localeCode)
package.file = os.path.basename(fPath)
if self._job.get("compile-options/paths/scripts-add-hash", False):
package.file = self._fileNameWithHash(package.file, package.hash)
setattr(package,"__localeflag", True) # TODO: temp. hack for writeI18NPackages()
# Finalize the new packages and parts
# - add prerequisite languages to parts; e.g. ["C", "en", "en_EN"]
for partId, part in newParts.items():
if newPackages["C"] not in part.packages:
package = newPackages["C"]
part.packages.append(package) # all need "C"
package.id |= part.bit_mask # adapt package's bit string
if len(partId) > 2 and partId[2] == "_": # it's a sub-language -> include main language
mainlang = partId[:2]
if mainlang not in newPackages:
raise RuntimeError("Locale '%s' specified, but not base locale '%s'" % (partId, mainlang))
if newPackages[mainlang] not in part.packages:
part.packages.append(newPackages[mainlang]) # add main language
newPackages[mainlang].id |= part.bit_mask # adapt package's bit string
# finally, sort packages
for part in newParts.values():
part.packagesSorted
# - add to script object
for partId in newParts:
if partId in script.parts:
raise RuntimeError("Name collison between code part and generated I18N part.")
script.parts[partId] = newParts[partId]
script.packages.extend(newPackages.values())
return script
def _getPreviousCommonPackage(self, searchPackage, packages):
# get the "smallest" package (in the sense of _sortPackages()) that is
# in all parts searchPackage is in, and is earlier in the corresponding
# packages lists
def isCommonAndGreaterPackage(searchPackage, package):
# the next takes advantage of the fact that the package id encodes
# the parts a package is used by. if another package id has the
# same bits turned on, it is in the same packages. this is only
# true for the searchId package itself and package id's that have
# more bits turned on (ie. are "greater"); hence, and due to
# _sortPackages, they are earlier in the packages list of the
# corresponding parts
if searchPackage.id & package.id == searchPackage.id:
return True
return False
##
# check if the deps of searchPackage have deps to targetPackage -
# if merging searchPackage into targetPackage, this would be creating
# circular dependencies
def noCircularDeps(searchPackage, targetPackage):
for package in searchPackage.packageDeps:
if targetPackage in package.packageDeps:
return False
return True
##
# check that the targetPackage is loaded in (at least) those parts
# where searchPackage's deps are also loaded
def depsAvailWhereTarget(searchPackage, targetPackage):
for depsPackage in searchPackage.packageDeps:
if not isCommonAndGreaterPackage(targetPackage, depsPackage):
#if not targetPackage.id & depsPackage.id == targetPackage.id:
return False
return True
# ----------------------------------------------------------------------
allPackages = reversed(Package.simpleSort(packages))
# sorting and reversing assures we try "smaller" package id's first
additional_constraints = self._jobconf.get(
"packages/additional-merge-constraints", False)
for targetPackage in allPackages:
if searchPackage.id == targetPackage.id: # no self-merging ;)
continue
if not isCommonAndGreaterPackage(searchPackage, targetPackage):
self._console.debug("Problematic #%d (different parts using)" %
targetPackage.id)
continue
if not noCircularDeps(searchPackage, targetPackage):
self._console.debug("Problematic #%d (circular dependencies)" %
targetPackage.id)
if additional_constraints:
continue
if not depsAvailWhereTarget(searchPackage, targetPackage):
self._console.debug(
"Problematic #%d (dependencies not always available)" %
targetPackage.id)
if additional_constraints:
continue
yield targetPackage
yield None
def packagesSorted(self):
return Package.sort(self.packages)
def runCompiled(self, script, treeCompiler, version="build"):
def getOutputFile(compileType):
filePath = compConf.get("paths/file")
if not filePath:
filePath = os.path.join(compileType, "script", self.getAppName() + ".js")
return filePath
def getFileUri(scriptUri):
appfile = os.path.basename(fileRelPath)
fileUri = os.path.join(scriptUri, appfile) # make complete with file name
fileUri = Path.posifyPath(fileUri)
return fileUri
def generateBootScript(globalCodes, script, bootPackage="", compileType="build"):
def packagesOfFiles(fileUri, packages):
# returns list of lists, each containing destination file name of the corresp. part
# npackages = [['script/gui-0.js'], ['script/gui-1.js'],...]
npackages = []
file = os.path.basename(fileUri)
if self._job.get("packages/loader-with-boot", True):
totalLen = len(packages)
else:
totalLen = len(packages) + 1
for packageId, packageFileName in enumerate(self.packagesFileNames(file, totalLen, classPackagesOnly=True)):
npackages.append((packageFileName,))
packages[packageId].file = packageFileName # TODO: very unnice to fix this here
return npackages
# besser: fixPackagesFiles()
def packagesOfFilesX(fileUri, packages):
# returns list of lists, each containing destination file name of the corresp. package
# npackages = [['script/gui-0.js'], ['script/gui-1.js'],...]
file = os.path.basename(fileUri)
loader_with_boot = self._job.get("packages/loader-with-boot", True)
for packageId, package in enumerate(packages):
if loader_with_boot:
suffix = packageId - 1
if suffix < 0:
suffix = ""
else:
suffix = packageId
packageFileName = self._resolveFileName(file, self._variants, self._settings, suffix)
package.file = packageFileName
return packages
# ----------------------------------------------------------------------------
self._console.info("Generating boot script...")
if not self._job.get("packages/i18n-with-boot", True):
globalCodes = self.writeI18NFiles(globalCodes, script)
# remove I18N info from globalCodes, so they don't go into the loader
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
else:
if compileType == "build":
# also remove them here, as this info is now with the packages
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
plugCodeFile = compConf.get("code/decode-uris-plug", False)
if compileType == "build":
filepackages = packagesOfFiles(fileUri, packages)
bootContent = self.generateBootCode(parts, filepackages, boot, script, compConf, variants, settings, bootPackage, globalCodes, compileType, plugCodeFile, format)
else:
filepackages = [x.classes for x in packages]
bootContent = self.generateBootCode(parts, filepackages, boot, script, compConf, variants={}, settings={}, bootCode=None, globalCodes=globalCodes, version=compileType, decodeUrisFile=plugCodeFile, format=format)
return bootContent
def getPackageData(package):
data = {}
data["resources"] = package.data.resources
data["translations"] = package.data.translations
data["locales"] = package.data.locales
data = json.dumpsCode(data)
data += ';\n'
return data
def compilePackage(packageIndex, package):
self._console.info("Compiling package #%s:" % packageIndex, False)
self._console.indent()
# Compile file content
pkgCode = self._treeCompiler.compileClasses(package.classes, variants, optimize, format)
pkgData = getPackageData(package)
hash = sha.getHash(pkgData + pkgCode)[:12] # first 12 chars should be enough
isBootPackage = packageIndex == 0
if isBootPackage:
compiledContent = ("qx.$$packageData['%s']=" % hash) + pkgData + pkgCode
else:
compiledContent = u'''qx.$$packageData['%s']=%s\n''' % (hash, pkgData)
compiledContent += u'''qx.Part.$$notifyLoad("%s", function() {\n%s\n});''' % (hash, pkgCode)
#
package.hash = hash # to fill qx.$$loader.packageHashes in generateBootScript()
self._console.debug("Done: %s" % self._computeContentSize(compiledContent))
self._console.outdent()
return compiledContent
# -- Main --------------------------------------------------------------
# Early return
compileType = self._job.get("compile/type", "")
if compileType not in ("build", "source"):
return
packages = script.packagesSortedSimple()
parts = script.parts
boot = script.boot
variants = script.variants
self._classList = script.classes
self._treeCompiler = treeCompiler
self._variants = variants
self._console.info("Generate %s version..." % compileType)
self._console.indent()
# - Evaluate job config ---------------------
# Compile config
compConf = self._job.get("compile-options")
compConf = ExtMap(compConf)
# Whether the code should be formatted
format = compConf.get("code/format", False)
script.scriptCompress = compConf.get("paths/gzip", False)
# Read in settings
settings = self.getSettings()
script.settings = settings
# Read libraries
libs = self._job.get("library", [])
# Get translation maps
locales = compConf.get("code/locales", [])
translationMaps = self.getTranslationMaps(packages, variants, locales)
# Read in base file name
fileRelPath = getOutputFile(compileType)
filePath = self._config.absPath(fileRelPath)
script.baseScriptPath = filePath
if compileType == "build":
# read in uri prefixes
scriptUri = compConf.get('uris/script', 'script')
scriptUri = Path.posifyPath(scriptUri)
fileUri = getFileUri(scriptUri)
# for resource list
resourceUri = compConf.get('uris/resource', 'resource')
resourceUri = Path.posifyPath(resourceUri)
else:
# source version needs place where the app HTML ("index.html") lives
self.approot = self._config.absPath(compConf.get("paths/app-root", ""))
resourceUri = None
scriptUri = None
# Get global script data (like qxlibraries, qxresources,...)
globalCodes = self.generateGlobalCodes(script, libs, translationMaps, settings, variants, format, resourceUri, scriptUri)
if compileType == "build":
# - Specific job config ---------------------
# read in compiler options
optimize = compConf.get("code/optimize", [])
self._treeCompiler.setOptimize(optimize)
# - Generating packages ---------------------
self._console.info("Generating packages...")
self._console.indent()
bootPackage = ""
for packageIndex, package in enumerate(packages):
package.compiled = compilePackage(packageIndex, package)
self._console.outdent()
if not len(packages):
raise RuntimeError("No valid boot package generated.")
# - Put loader and packages together -------
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# handle loader and boot package
if loader_with_boot:
bootCode = generateBootScript(globalCodes, script, packages[0].compiled)
packages[0].compiled = bootCode
else:
loaderCode = generateBootScript(globalCodes, script)
loadPackage = Package(0) # make a dummy Package for the loader
loadPackage.compiled = loaderCode
packages.insert(0, loadPackage)
# attach file names
for package, fileName in zip(packages, self.packagesFileNames(script.baseScriptPath, len(packages))):
package.file = fileName
# write packages
self.writePackages(packages, script)
# ---- 'source' version ------------------------------------------------
else:
sourceContent = generateBootScript(globalCodes, script, bootPackage="", compileType=compileType)
# Construct file name
resolvedFilePath = self._resolveFileName(filePath, variants, settings)
# Save result file
filetool.save(resolvedFilePath, sourceContent)
if compConf.get("paths/gzip"):
filetool.gzip(resolvedFilePath, sourceContent)
self._console.outdent()
self._console.debug("Done: %s" % self._computeContentSize(sourceContent))
self._console.outdent()
self._console.outdent()
return # runCompiled()
def _getPackages(self, script):
parts = script.parts
# Generating list of all classes
allClasses = {}
for part in parts.values():
for classId in part.deps:
allClasses[classId] = True
# Check for each class which part is using it;
# create a package for each set of classes which
# are used by the same combination of parts;
# track how many parts are using a particular package
packages = {}
for classId in allClasses.keys():
pkgId = 0
for part in parts.values():
if classId in part.deps:
pkgId |= part.bit_mask
if not packages.has_key(pkgId):
package = Package(pkgId)
packages[pkgId] = package
packages[pkgId].classes.append(classId)
# Which packages does a part use - and vice versa
for package in packages.values():
for part in parts.values():
if package.id & part.bit_mask:
#part.packages.append(package.id)
part.packages.append(package)
#package.parts.append(part.name)
package.parts.append(part)
package.part_count = len(package.parts)
# Sorting packages of parts
for part in parts.values():
#part.packageIdsSorted = self._sortPackages([x.id for x in part.packages], packages)
## re-map sorting to part.packages
#packObjs = []
#for pkgId in part.packageIdsSorted:
# packObjs.append(packages[pkgId])
#part.packages = packObjs
self._sortPartPackages(part)
# Register dependencies between Packages
for package in packages.values():
# get all direct deps of this package
allDeps = set(())
for classId in package.classes:
classDeps = self._depLoader.getCombinedDeps(classId, script.variants)
loadDeps = set(x.name for x in classDeps['load'])
allDeps.update(loadDeps)
# record the other packages in which these dependencies are located
for classId in allDeps:
for otherpackage in packages.values():
if otherpackage != package and classId in otherpackage.classes:
#print "-- package %s adding dependent package %s" % (package.id, otherpackage.id)
package.packageDeps.add(otherpackage)
return packages
def generateI18NParts(self, script, locales):
##
# collect translation and locale info from the packages
def getTranslationMaps(packages):
packageTranslations = []
for package in packages:
trans_dat = package.data.translations
loc_dat = package.data.locales
packageTranslations.append((trans_dat,loc_dat))
return packageTranslations
##
# takes an array of (po-data, locale-data) dict pairs
# merge all po data and all cldr data in a single dict each
def mergeTranslationMaps(transMaps):
poData = {}
cldrData = {}
for pac_dat, loc_dat in transMaps:
for loc in pac_dat:
if loc not in poData:
poData[loc] = {}
poData[loc].update(pac_dat[loc])
for loc in loc_dat:
if loc not in cldrData:
cldrData[loc] = {}
cldrData[loc].update(loc_dat[loc])
return (poData, cldrData)
# ----------------------------------------------------------------------
translationMaps = getTranslationMaps(script.packages)
translationData , \
localeData = mergeTranslationMaps(translationMaps)
# for each locale code, collect mappings
transKeys = translationData.keys()
localeKeys = localeData.keys()
newParts = {} # language codes to part objects, {"C": part}
newPackages= {} # language codes to private package objects, {"C": package}
for localeCode in set(transKeys + localeKeys):
# also provide a localeCode "part" with corresponding packages
intpart = Part(localeCode)
intpart.bit_mask = script.getPartBitMask()
newParts[localeCode] = intpart
intpackage = Package(intpart.bit_mask) # this might be modified later
newPackages[localeCode] = intpackage
intpart.packages.append(intpackage)
data = {}
data[localeCode] = { 'Translations': {}, 'Locales': {} } # we want to have the locale code in the data
if localeCode in transKeys:
data[localeCode]['Translations'] = translationData[localeCode]
intpackage.data.translations[localeCode] = translationData[localeCode]
if localeCode in localeKeys:
data[localeCode]['Locales'] = localeData[localeCode]
intpackage.data.locales[localeCode] = localeData[localeCode]
# file name and hash code
hash_, dataS = intpackage.packageContent() # TODO: this currently works only for pure data packages
dataS = dataS.replace('\\\\\\', '\\').replace(r'\\', '\\') # undo damage done by simplejson to raw strings with escapes \\ -> \
intpackage.compiled.append(dataS)
intpackage.hash = hash_
fPath = self._resolveFileName(script.baseScriptPath, script.variants, script.settings, localeCode)
intpackage.file = os.path.basename(fPath)
if self._job.get("compile-options/paths/scripts-add-hash", False):
intpackage.file = self._fileNameWithHash(intpackage.file, intpackage.hash)
intpackage.files = ["%s:%s" % ("__out__", intpackage.file)]
setattr(intpackage,"__localeflag", True) # TODO: temp. hack for writeI18NPackages()
# Finalize the new packages and parts
# - add prerequisite languages to parts; e.g. ["C", "en", "en_EN"]
for partId, intpart in newParts.items():
if newPackages["C"] not in intpart.packages:
intpackage = newPackages["C"]
intpart.packages.append(intpackage) # all need "C"
intpackage.part_mask |= intpart.bit_mask # adapt package's bit string
if len(partId) > 2 and partId[2] == "_": # it's a sub-language -> include main language
mainlang = partId[:2]
if mainlang not in newPackages:
raise RuntimeError("Locale '%s' specified, but not base locale '%s'" % (partId, mainlang))
if newPackages[mainlang] not in intpart.packages:
intpart.packages.append(newPackages[mainlang]) # add main language
newPackages[mainlang].part_mask |= intpart.bit_mask # adapt package's bit string
# finally, sort packages
for intpart in newParts.values():
intpart.packagesSorted
# - add to script object
for partId in newParts:
if partId in script.parts:
raise RuntimeError("Name collison between code part and generated I18N part.")
script.parts[partId] = newParts[partId]
script.packages.extend(newPackages.values())
return script
def generateI18NParts(self, script, globalCodes):
# for each locale code, collect mappings
transKeys = globalCodes['Translations'].keys()
localeKeys = globalCodes['Locales'].keys()
newParts = {} # language codes to part objects, {"C": part}
newPackages= {} # language codes to private package objects, {"C": package}
for localeCode in set(transKeys + localeKeys):
# new: also provide a localeCode "part" with corresponding packages
part = Part(localeCode)
part.bit_mask = script.getPartBitMask()
newParts[localeCode] = part
package = Package(part.bit_mask) # this might be modified later
newPackages[localeCode] = package
part.packages.append(package)
data = {}
data[localeCode] = { 'Translations': {}, 'Locales': {} } # we want to have the locale code in the data
if localeCode in transKeys:
data[localeCode]['Translations'] = globalCodes['Translations'][localeCode]
package.data.translations[localeCode] = globalCodes['Translations'][localeCode]
if localeCode in localeKeys:
data[localeCode]['Locales'] = globalCodes['Locales'][localeCode]
package.data.locales[localeCode] = globalCodes['Locales'][localeCode]
# file name and hash code
hash, dataS = package.packageContent() # TODO: this currently works only for pure data packages
dataS = dataS.replace('\\\\\\', '\\').replace(r'\\', '\\') # undo damage done by simplejson to raw strings with escapes \\ -> \
package.compiled = dataS
package.hash = hash
fPath = self._resolveFileName(script.baseScriptPath, script.variants, script.settings, localeCode)
package.file = os.path.basename(fPath)
if self._job.get("compile-options/paths/scripts-add-hash", False):
package.file = self._fileNameWithHash(package.file, package.hash)
setattr(package,"__localeflag", True) # TODO: temp. hack for writeI18NPackages()
# Finalize the new packages and parts
# - add prerequisite languages to parts; e.g. ["C", "en", "en_EN"]
for partId, part in newParts.items():
if newPackages["C"] not in part.packages:
package = newPackages["C"]
part.packages.append(package) # all need "C"
package.id |= part.bit_mask # adapt package's bit string
if len(partId) > 2 and partId[2] == "_": # it's a sub-language -> include main language
mainlang = partId[:2]
if mainlang not in newPackages:
raise RuntimeError("Locale '%s' specified, but not base locale '%s'" % (partId, mainlang))
if newPackages[mainlang] not in part.packages:
part.packages.append(newPackages[mainlang]) # add main language
newPackages[mainlang].id |= part.bit_mask # adapt package's bit string
# finally, sort packages
for part in newParts.values():
part.packagesSorted
# - add to script object
for partId in newParts:
if partId in script.parts:
raise RuntimeError("Name collison between code part and generated I18N part.")
script.parts[partId] = newParts[partId]
script.packages.extend(newPackages.values())
return script
def runCompiled(self, script, treeCompiler, version="build"):
def getOutputFile(compileType):
filePath = compConf.get("paths/file")
if not filePath:
filePath = os.path.join(compileType, "script", script.namespace + ".js")
return filePath
def getFileUri(scriptUri):
appfile = os.path.basename(fileRelPath)
fileUri = os.path.join(scriptUri, appfile) # make complete with file name
fileUri = Path.posifyPath(fileUri)
return fileUri
##
# returns the Javascript code for the initial ("boot") script as a string,
# using the loader.tmpl template and filling its placeholders
def generateBootCode(parts, packages, boot, script, compConf, variants, settings, bootCode, globalCodes, version="source", decodeUrisFile=None, format=False):
##
# create a map with part names as key and array of package id's and
# return as string
def partsMap(script):
partData = {}
packages = script.packagesSortedSimple()
#print "packages: %r" % packages
for part in script.parts:
partData[part] = script.parts[part].packagesAsIndices(packages)
#print "part '%s': %r" % (part, script.parts[part].packages)
partData = json.dumpsCode(partData)
return partData
def fillTemplate(vals, template):
# Fill the code template with various vals
templ = MyTemplate(template)
result = templ.safe_substitute(vals)
return result
def packageUrisToJS1(packages, version, namespace=None):
# Translate URI data to JavaScript
allUris = []
for packageId, package in enumerate(packages):
packageUris = []
for fileId in package:
if version == "build":
# TODO: gosh, the next is an ugly hack!
#namespace = self._resourceHandler._genobj._namespaces[0] # all name spaces point to the same paths in the libinfo struct, so any of them will do
if not namespace:
namespace = script.namespace # all name spaces point to the same paths in the libinfo struct, so any of them will do
relpath = OsPath(fileId)
else:
namespace = self._classes[fileId]["namespace"]
relpath = OsPath(self._classes[fileId]["relpath"])
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
allUris.append(packageUris)
return allUris
##
# Translate URI data to JavaScript
# using Package objects
def packageUrisToJS(packages, version):
allUris = []
for packageId, package in enumerate(packages):
packageUris = []
if package.file: # build
namespace = "__out__"
fileId = package.file
relpath = OsPath(fileId)
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
else: # "source" :
for clazz in package.classes:
namespace = self._classes[clazz]["namespace"]
relpath = OsPath(self._classes[clazz]["relpath"])
shortUri = Uri(relpath.toUri())
packageUris.append("%s:%s" % (namespace, shortUri.encodedValue()))
allUris.append(packageUris)
return allUris
def loadTemplate(bootCode):
# try custom loader templates
loaderFile = compConf.get("paths/loader-template", None)
if not loaderFile:
# use default templates
if version=="build":
#loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-build.tmpl.js")
# TODO: test-wise using generic template
loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js")
else:
#loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-source.tmpl.js")
loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js")
template = filetool.read(loaderFile)
return template
# ---------------------------------------------------------------
if not parts:
return ""
result = ""
vals = {}
packages = script.packagesSortedSimple()
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# stringify data in globalCodes
for entry in globalCodes:
globalCodes[entry] = json.dumpsCode(globalCodes[entry])
# undo damage done by simplejson to raw strings with escapes \\ -> \
globalCodes[entry] = globalCodes[entry].replace('\\\\\\', '\\').replace(r'\\', '\\') # " gets tripple escaped, therefore the first .replace()
vals.update(globalCodes)
if version=="build":
vals["Resources"] = json.dumpsCode({}) # TODO: undo Resources from globalCodes!!!
vals["Boot"] = '"%s"' % boot
if version == "build":
vals["BootPart"] = bootCode
else:
vals["BootPart"] = ""
# fake package data
for key, package in enumerate(packages):
vals["BootPart"] += "qx.$$packageData['%d']={};\n" % key
# Translate part information to JavaScript
vals["Parts"] = partsMap(script)
# Translate URI data to JavaScript
#vals["Uris"] = packageUrisToJS1(packages, version)
vals["Uris"] = packageUrisToJS(packages, version)
vals["Uris"] = json.dumpsCode(vals["Uris"])
# Add potential extra scripts
vals["UrisBefore"] = []
if self._job.get("add-script", False):
additional_scripts = self._job.get("add-script",[])
for additional_script in additional_scripts:
vals["UrisBefore"].append(additional_script["uri"])
vals["UrisBefore"] = json.dumpsCode(vals["UrisBefore"])
# Whether boot package is inline
if version == "source":
vals["BootIsInline"] = json.dumpsCode(False)
else:
vals["BootIsInline"] = json.dumpsCode(loader_with_boot)
# Closure package information
cParts = {}
if version == "build":
for part in script.parts:
if not loader_with_boot or part != "boot":
cParts[part] = True
vals["ClosureParts"] = json.dumpsCode(cParts)
# Package Hashes
vals["PackageHashes"] = {}
for key, package in enumerate(packages):
if package.hash:
vals["PackageHashes"][key] = package.hash
else:
vals["PackageHashes"][key] = "%d" % key # fake code package hashes in source ver.
vals["PackageHashes"] = json.dumpsCode(vals["PackageHashes"])
# Script hook for qx.$$loader.decodeUris() function
vals["DecodeUrisPlug"] = ""
if decodeUrisFile:
plugCode = filetool.read(self._config.absPath(decodeUrisFile)) # let it bomb if file can't be read
vals["DecodeUrisPlug"] = plugCode.strip()
# Enable "?nocache=...." for script loading?
vals["NoCacheParam"] = "true" if self._job.get("compile-options/uris/add-nocache-param", True) else "false"
# Locate and load loader basic script
template = loadTemplate(bootCode)
# Fill template gives result
result = fillTemplate(vals, template)
return result
##
# shallow layer above generateBootCode(), and its only client
def generateBootScript(globalCodes, script, bootPackage="", compileType="build"):
self._console.info("Generating boot script...")
if not self._job.get("packages/i18n-with-boot", True):
# remove I18N info from globalCodes, so they don't go into the loader
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
else:
if compileType == "build":
# also remove them here, as this info is now with the packages
globalCodes["Translations"] = {}
globalCodes["Locales"] = {}
plugCodeFile = compConf.get("code/decode-uris-plug", False)
if compileType == "build":
filepackages = [(x.file,) for x in packages]
bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants, settings, bootPackage, globalCodes, compileType, plugCodeFile, format)
else:
filepackages = [x.classes for x in packages]
bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants={}, settings={}, bootCode=None, globalCodes=globalCodes, version=compileType, decodeUrisFile=plugCodeFile, format=format)
return bootContent
def getPackageData(package):
data = {}
data["resources"] = package.data.resources
data["translations"] = package.data.translations
data["locales"] = package.data.locales
data = json.dumpsCode(data)
data += ';\n'
return data
def compilePackage(packageIndex, package):
self._console.info("Compiling package #%s:" % packageIndex, False)
self._console.indent()
# Compile file content
pkgCode = self._treeCompiler.compileClasses(package.classes, variants, optimize, format)
pkgData = getPackageData(package)
hash = sha.getHash(pkgData + pkgCode)[:12] # first 12 chars should be enough
isBootPackage = packageIndex == 0
if isBootPackage:
compiledContent = ("qx.$$packageData['%s']=" % hash) + pkgData + pkgCode
else:
compiledContent = u'''qx.$$packageData['%s']=%s\n''' % (hash, pkgData)
compiledContent += u'''qx.Part.$$notifyLoad("%s", function() {\n%s\n});''' % (hash, pkgCode)
#
package.hash = hash # to fill qx.$$loader.packageHashes in generateBootScript()
self._console.debug("Done: %s" % self._computeContentSize(compiledContent))
self._console.outdent()
return compiledContent
##
# takes an array of (po-data, locale-data) dict pairs
# merge all po data and all cldr data in a single dict each
def mergeTranslationMaps(transMaps):
poData = {}
cldrData = {}
for pac_dat, loc_dat in transMaps:
for loc in pac_dat:
if loc not in poData:
poData[loc] = {}
poData[loc].update(pac_dat[loc])
for loc in loc_dat:
if loc not in cldrData:
cldrData[loc] = {}
cldrData[loc].update(loc_dat[loc])
return (poData, cldrData)
# -- Main - runCompiled ------------------------------------------------
# Early return
compileType = self._job.get("compile/type", "")
if compileType not in ("build", "source"):
return
packages = script.packagesSortedSimple()
parts = script.parts
boot = script.boot
variants = script.variants
libraries = script.libraries
self._treeCompiler = treeCompiler
self._variants = variants
self._script = script
self._console.info("Generate %s version..." % compileType)
self._console.indent()
# - Evaluate job config ---------------------
# Compile config
compConf = self._job.get("compile-options")
compConf = ExtMap(compConf)
# Whether the code should be formatted
format = compConf.get("code/format", False)
script.scriptCompress = compConf.get("paths/gzip", False)
# Read in settings
settings = self.getSettings()
script.settings = settings
# Read libraries
libs = self._job.get("library", [])
# Get translation maps
locales = compConf.get("code/locales", [])
translationMaps = self.getTranslationMaps(packages, variants, locales)
# Read in base file name
fileRelPath = getOutputFile(compileType)
filePath = self._config.absPath(fileRelPath)
script.baseScriptPath = filePath
if compileType == "build":
# read in uri prefixes
scriptUri = compConf.get('uris/script', 'script')
scriptUri = Path.posifyPath(scriptUri)
fileUri = getFileUri(scriptUri)
# for resource list
resourceUri = compConf.get('uris/resource', 'resource')
resourceUri = Path.posifyPath(resourceUri)
else:
# source version needs place where the app HTML ("index.html") lives
self.approot = self._config.absPath(compConf.get("paths/app-root", ""))
resourceUri = None
scriptUri = None
# Get global script data (like qxlibraries, qxresources,...)
globalCodes = {}
globalCodes["Settings"] = settings
globalCodes["Variants"] = self.generateVariantsCode(variants)
globalCodes["Libinfo"] = self.generateLibInfoCode(libs, format, resourceUri, scriptUri)
# add synthetic output lib
if scriptUri: out_sourceUri= scriptUri
else:
out_sourceUri = self._computeResourceUri({'class': ".", 'path': os.path.dirname(script.baseScriptPath)}, OsPath(""), rType="class", appRoot=self.approot)
out_sourceUri = os.path.normpath(out_sourceUri.encodedValue())
globalCodes["Libinfo"]['__out__'] = { 'sourceUri': out_sourceUri }
globalCodes["Resources"] = self.generateResourceInfoCode(script, settings, libraries, format)
globalCodes["Translations"],\
globalCodes["Locales"] = mergeTranslationMaps(translationMaps)
# Potentally create dedicated I18N packages
i18n_as_parts = not self._job.get("packages/i18n-with-boot", True)
if i18n_as_parts:
script = self.generateI18NParts(script, globalCodes)
self.writePackages([p for p in script.packages if getattr(p, "__localeflag", False)], script)
if compileType == "build":
# - Specific job config ---------------------
# read in compiler options
optimize = compConf.get("code/optimize", [])
self._treeCompiler.setOptimize(optimize)
# - Generating packages ---------------------
self._console.info("Generating packages...")
self._console.indent()
bootPackage = ""
for packageIndex, package in enumerate(packages):
package.compiled = compilePackage(packageIndex, package)
self._console.outdent()
if not len(packages):
raise RuntimeError("No valid boot package generated.")
# - Put loader and packages together -------
loader_with_boot = self._job.get("packages/loader-with-boot", True)
# handle loader and boot package
if not loader_with_boot:
loadPackage = Package(0) # make a dummy Package for the loader
packages.insert(0, loadPackage)
# attach file names (do this before calling generateBootScript)
for package, fileName in zip(packages, self.packagesFileNames(script.baseScriptPath, len(packages))):
package.file = os.path.basename(fileName)
if self._job.get("compile-options/paths/scripts-add-hash", False):
package.file = self._fileNameWithHash(package.file, package.hash)
# generate and integrate boot code
if loader_with_boot:
# merge loader code with first package
bootCode = generateBootScript(globalCodes, script, packages[0].compiled)
packages[0].compiled = bootCode
else:
loaderCode = generateBootScript(globalCodes, script)
packages[0].compiled = loaderCode
# write packages
self.writePackages(packages, script)
# ---- 'source' version ------------------------------------------------
else:
sourceContent = generateBootScript(globalCodes, script, bootPackage="", compileType=compileType)
# Construct file name
resolvedFilePath = self._resolveFileName(filePath, variants, settings)
# Save result file
filetool.save(resolvedFilePath, sourceContent)
if compConf.get("paths/gzip"):
filetool.gzip(resolvedFilePath, sourceContent)
self._console.outdent()
self._console.debug("Done: %s" % self._computeContentSize(sourceContent))
self._console.outdent()
self._console.outdent()
return # runCompiled()
def packagesSorted(self):
return Package.sort(self.packages)
def packagesSortedSimple(self):
return Package.simpleSort(self.packages)
def partsConfigFromClassList(includeWithDeps, excludeWithDeps, script):
def evalPackagesConfig(excludeWithDeps, classList, variants):
# Reading configuration
partsCfg = self._job.get("packages/parts", {})
# Expanding expressions
self._console.debug("Expanding include expressions...")
partIncludes = {}
for partId in partsCfg:
partIncludes[partId] = textutil.expandGlobs(
partsCfg[partId]['include'], self._classesObj)
# Computing packages
#boot, partPackages, packageClasses = self._partBuilder.getPackages(partIncludes, excludeWithDeps, self._context, script)
partPackages, _ = self._partBuilder.getPackages(
partIncludes, excludeWithDeps, self._context, script)
packageClasses = script.packagesSorted()
#return boot, partPackages, packageClasses
return script.boot, script.parts, packageClasses
# -----------------------------------------------------------
classList = script.classes
variants = script.variants
self._partBuilder = PartBuilder(self._console, self._depLoader)
# Check for a 'packages' configuration in the job
if 0:
# this branch should work, but doesn't;
# create a synthetic job key and let evalPackagesConfig do the rest
if not self._job.get("packages"):
package_config = {
"parts": {
"boot": {
"include": includeWithDeps
}
},
"init": "boot"
}
self._job.setFeature("packages", package_config)
(
boot,
partPackages, # partPackages[partId]=[0,1,3]
packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...]
) = evalPackagesConfig(excludeWithDeps, classList, variants)
else:
if self._job.get("packages"):
(
boot,
partPackages, # partPackages[partId]=[0,1,3]
packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...]
) = evalPackagesConfig(excludeWithDeps, classList,
variants)
else:
# Emulate a 'boot' part
boot = "boot"
partPackages = {"boot": [0]}
packageClasses = [classList]
# patch script object
script.boot = boot
packageObj = Package(0)
packageObj.classes = script.classesObj
script.packages.append(packageObj)
partObj = Part("boot")
partObj.packages.append(packageObj)
initial_deps = list(
set(includeWithDeps).difference(script.excludes)
) # defining classes from config minus expanded excludes
partObj.initial_deps = initial_deps
partObj.deps = initial_deps[:]
script.parts = {"boot": partObj}
return boot, partPackages, packageClasses
def _getPreviousCommonPackage(self, searchPackage, packages):
# get the "smallest" package (in the sense of _sortPackages()) that is
# in all parts searchPackage is in, and is earlier in the corresponding
# packages lists
def isCommonAndGreaterPackage(searchPackage, package):
# the next takes advantage of the fact that the package id encodes
# the parts a package is used by. if another package id has the
# same bits turned on, it is in the same packages. this is only
# true for the searchId package itself and package id's that have
# more bits turned on (ie. are "greater"); hence, and due to
# _sortPackages, they are earlier in the packages list of the
# corresponding parts
if searchPackage.id & package.id == searchPackage.id:
return True
return False
##
# check if the deps of searchPackage have deps to targetPackage -
# if merging searchPackage into targetPackage, this would be creating
# circular dependencies
def noCircularDeps(searchPackage, targetPackage):
for package in searchPackage.packageDeps:
if targetPackage in package.packageDeps:
return False
return True
##
# check that the targetPackage is loaded in (at least) those parts
# where searchPackage's deps are also loaded
def depsAvailWhereTarget (searchPackage, targetPackage):
for depsPackage in searchPackage.packageDeps:
if not isCommonAndGreaterPackage(targetPackage, depsPackage):
#if not targetPackage.id & depsPackage.id == targetPackage.id:
return False
return True
# ----------------------------------------------------------------------
allPackages = reversed(Package.simpleSort(packages))
# sorting and reversing assures we try "smaller" package id's first
additional_constraints = self._jobconf.get("packages/additional-merge-constraints", False)
for targetPackage in allPackages:
if searchPackage.id == targetPackage.id: # no self-merging ;)
continue
if not isCommonAndGreaterPackage(searchPackage, targetPackage):
self._console.debug("Problematic #%d (different parts using)" % targetPackage.id)
continue
if not noCircularDeps(searchPackage, targetPackage):
self._console.debug("Problematic #%d (circular dependencies)" % targetPackage.id)
if additional_constraints:
continue
if not depsAvailWhereTarget(searchPackage, targetPackage):
self._console.debug("Problematic #%d (dependencies not always available)" % targetPackage.id)
if additional_constraints:
continue
yield targetPackage
yield None
def partsConfigFromClassList(includeWithDeps, excludeWithDeps, script):
def evalPackagesConfig(excludeWithDeps, classList, variants):
# Reading configuration
partsCfg = self._job.get("packages/parts", {})
# Expanding expressions
self._console.debug("Expanding include expressions...")
partIncludes = {}
for partId in partsCfg:
partIncludes[partId] = textutil.expandGlobs(partsCfg[partId]['include'], self._classesObj)
# Computing packages
#boot, partPackages, packageClasses = self._partBuilder.getPackages(partIncludes, excludeWithDeps, self._context, script)
partPackages, _ = self._partBuilder.getPackages(partIncludes, excludeWithDeps, self._context, script)
packageClasses = script.packagesSorted()
#return boot, partPackages, packageClasses
return script.boot, script.parts, packageClasses
# -----------------------------------------------------------
classList = script.classes
variants = script.variants
self._partBuilder = PartBuilder(self._console, self._depLoader)
# Check for a 'packages' configuration in the job
if 0:
# this branch should work, but doesn't;
# create a synthetic job key and let evalPackagesConfig do the rest
if not self._job.get("packages"):
package_config = {
"parts" : {
"boot" : {
"include" : includeWithDeps
}
},
"init" : "boot"
}
self._job.setFeature("packages", package_config)
(boot,
partPackages, # partPackages[partId]=[0,1,3]
packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...]
) = evalPackagesConfig(excludeWithDeps, classList, variants)
else:
if self._job.get("packages"):
(boot,
partPackages, # partPackages[partId]=[0,1,3]
packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...]
) = evalPackagesConfig(excludeWithDeps, classList, variants)
else:
# Emulate a 'boot' part
boot = "boot"
partPackages = { "boot" : [0] }
packageClasses = [classList]
# patch script object
script.boot = boot
packageObj = Package(0)
packageObj.classes = script.classesObj
script.packages.append(packageObj)
partObj = Part("boot")
partObj.packages.append(packageObj)
initial_deps = list(set(includeWithDeps).difference(script.excludes)) # defining classes from config minus expanded excludes
partObj.initial_deps = initial_deps
partObj.deps = initial_deps[:]
script.parts = { "boot" : partObj }
return boot, partPackages, packageClasses