def _getPackages(self, script): self._console.indent() parts = script.parts # Generating list of all classes allClasses = {} for part in parts.values(): for classId in part.deps: allClasses[classId] = True # Check for each class which part is using it; # create a package for each set of classes which # are used by the same combination of parts; # track how many parts are using a particular package packages = {} for classId in allClasses.keys(): pkgId = 0 for part in parts.values(): if classId in part.deps: pkgId |= part.bit_mask if not packages.has_key(pkgId): package = Package(pkgId) packages[pkgId] = package packages[pkgId].classes.append(classId) # Which packages does a part use - and vice versa for package in packages.values(): for part in parts.values(): if package.id & part.bit_mask: #part.packages.append(package.id) part.packages.append(package) #package.parts.append(part.name) #package.parts.append(part) #package.part_count = len(package.parts) # Register dependencies between Packages for package in packages.values(): # get all direct deps of this package allDeps = set(()) for classId in package.classes: classDeps, _ = self._depLoader.getCombinedDeps( classId, script.variants, script.buildType) loadDeps = set(x.name for x in classDeps['load']) allDeps.update(loadDeps) # record the other packages in which these dependencies are located for classId in allDeps: for otherpackage in packages.values(): if otherpackage != package and classId in otherpackage.classes: #print "-- package %s adding dependent package %s" % (package.id, otherpackage.id) package.packageDeps.add(otherpackage) self._console.outdent() return packages.values()
def getPackagesFromClasses(allClasses): packages = {} for classId in allClasses: pkgId = allClasses[classId] # create a Package if necessary if pkgId not in packages: package = Package(pkgId) packages[pkgId] = package # store classId with this package packages[pkgId].classes.append(classId) return packages.values()
def partsConfigFromClassList(includeWithDeps, excludeWithDeps, script): def evalPackagesConfig(excludeWithDeps, classList, variants): # Reading configuration partsCfg = self._job.get("packages/parts", {}) # Expanding expressions self._console.debug("Expanding include expressions...") partIncludes = {} for partId in partsCfg: partIncludes[partId] = textutil.expandGlobs( partsCfg[partId]['include'], self._classesObj) # Computing packages #boot, partPackages, packageClasses = self._partBuilder.getPackages(partIncludes, excludeWithDeps, self._context, script) partPackages, _ = self._partBuilder.getPackages( partIncludes, excludeWithDeps, self._context, script) packageClasses = script.packagesSorted() #return boot, partPackages, packageClasses return script.boot, script.parts, packageClasses # ----------------------------------------------------------- classList = script.classes variants = script.variants self._partBuilder = PartBuilder(self._console, self._depLoader) # Check for a 'packages' configuration in the job if 0: # this branch should work, but doesn't; # create a synthetic job key and let evalPackagesConfig do the rest if not self._job.get("packages"): package_config = { "parts": { "boot": { "include": includeWithDeps } }, "init": "boot" } self._job.setFeature("packages", package_config) ( boot, partPackages, # partPackages[partId]=[0,1,3] packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...] ) = evalPackagesConfig(excludeWithDeps, classList, variants) else: if self._job.get("packages"): ( boot, partPackages, # partPackages[partId]=[0,1,3] packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...] ) = evalPackagesConfig(excludeWithDeps, classList, variants) else: # Emulate a 'boot' part boot = "boot" partPackages = {"boot": [0]} packageClasses = [classList] # patch script object script.boot = boot packageObj = Package(0) packageObj.classes = script.classesObj script.packages.append(packageObj) partObj = Part("boot") partObj.packages.append(packageObj) initial_deps = list( set(includeWithDeps).difference(script.excludes) ) # defining classes from config minus expanded excludes partObj.initial_deps = initial_deps partObj.deps = initial_deps[:] script.parts = {"boot": partObj} return boot, partPackages, packageClasses
def generateI18NParts(self, script, globalCodes): # for each locale code, collect mappings transKeys = globalCodes['Translations'].keys() localeKeys = globalCodes['Locales'].keys() newParts = {} # language codes to part objects, {"C": part} newPackages= {} # language codes to private package objects, {"C": package} for localeCode in set(transKeys + localeKeys): # new: also provide a localeCode "part" with corresponding packages part = Part(localeCode) part.bit_mask = script.getPartBitMask() newParts[localeCode] = part package = Package(part.bit_mask) # this might be modified later newPackages[localeCode] = package part.packages.append(package) data = {} data[localeCode] = { 'Translations': {}, 'Locales': {} } # we want to have the locale code in the data if localeCode in transKeys: data[localeCode]['Translations'] = globalCodes['Translations'][localeCode] package.data.translations[localeCode] = globalCodes['Translations'][localeCode] if localeCode in localeKeys: data[localeCode]['Locales'] = globalCodes['Locales'][localeCode] package.data.locales[localeCode] = globalCodes['Locales'][localeCode] # file name and hash code hash, dataS = package.packageContent() # TODO: this currently works only for pure data packages dataS = dataS.replace('\\\\\\', '\\').replace(r'\\', '\\') # undo damage done by simplejson to raw strings with escapes \\ -> \ package.compiled = dataS package.hash = hash fPath = self._resolveFileName(script.baseScriptPath, script.variants, script.settings, localeCode) package.file = os.path.basename(fPath) if self._job.get("compile-options/paths/scripts-add-hash", False): package.file = self._fileNameWithHash(package.file, package.hash) setattr(package,"__localeflag", True) # TODO: temp. hack for writeI18NPackages() # Finalize the new packages and parts # - add prerequisite languages to parts; e.g. ["C", "en", "en_EN"] for partId, part in newParts.items(): if newPackages["C"] not in part.packages: package = newPackages["C"] part.packages.append(package) # all need "C" package.id |= part.bit_mask # adapt package's bit string if len(partId) > 2 and partId[2] == "_": # it's a sub-language -> include main language mainlang = partId[:2] if mainlang not in newPackages: raise RuntimeError("Locale '%s' specified, but not base locale '%s'" % (partId, mainlang)) if newPackages[mainlang] not in part.packages: part.packages.append(newPackages[mainlang]) # add main language newPackages[mainlang].id |= part.bit_mask # adapt package's bit string # finally, sort packages for part in newParts.values(): part.packagesSorted # - add to script object for partId in newParts: if partId in script.parts: raise RuntimeError("Name collison between code part and generated I18N part.") script.parts[partId] = newParts[partId] script.packages.extend(newPackages.values()) return script
def runCompiled(self, script, treeCompiler, version="build"): def getOutputFile(compileType): filePath = compConf.get("paths/file") if not filePath: filePath = os.path.join(compileType, "script", script.namespace + ".js") return filePath def getFileUri(scriptUri): appfile = os.path.basename(fileRelPath) fileUri = os.path.join(scriptUri, appfile) # make complete with file name fileUri = Path.posifyPath(fileUri) return fileUri ## # returns the Javascript code for the initial ("boot") script as a string, # using the loader.tmpl template and filling its placeholders def generateBootCode(parts, packages, boot, script, compConf, variants, settings, bootCode, globalCodes, version="source", decodeUrisFile=None, format=False): ## # create a map with part names as key and array of package id's and # return as string def partsMap(script): partData = {} packages = script.packagesSortedSimple() #print "packages: %r" % packages for part in script.parts: partData[part] = script.parts[part].packagesAsIndices(packages) #print "part '%s': %r" % (part, script.parts[part].packages) partData = json.dumpsCode(partData) return partData def fillTemplate(vals, template): # Fill the code template with various vals templ = MyTemplate(template) result = templ.safe_substitute(vals) return result def packageUrisToJS1(packages, version, namespace=None): # Translate URI data to JavaScript allUris = [] for packageId, package in enumerate(packages): packageUris = [] for fileId in package: if version == "build": # TODO: gosh, the next is an ugly hack! #namespace = self._resourceHandler._genobj._namespaces[0] # all name spaces point to the same paths in the libinfo struct, so any of them will do if not namespace: namespace = script.namespace # all name spaces point to the same paths in the libinfo struct, so any of them will do relpath = OsPath(fileId) else: namespace = self._classes[fileId]["namespace"] relpath = OsPath(self._classes[fileId]["relpath"]) shortUri = Uri(relpath.toUri()) packageUris.append("%s:%s" % (namespace, shortUri.encodedValue())) allUris.append(packageUris) return allUris ## # Translate URI data to JavaScript # using Package objects def packageUrisToJS(packages, version): allUris = [] for packageId, package in enumerate(packages): packageUris = [] if package.file: # build namespace = "__out__" fileId = package.file relpath = OsPath(fileId) shortUri = Uri(relpath.toUri()) packageUris.append("%s:%s" % (namespace, shortUri.encodedValue())) else: # "source" : for clazz in package.classes: namespace = self._classes[clazz]["namespace"] relpath = OsPath(self._classes[clazz]["relpath"]) shortUri = Uri(relpath.toUri()) packageUris.append("%s:%s" % (namespace, shortUri.encodedValue())) allUris.append(packageUris) return allUris def loadTemplate(bootCode): # try custom loader templates loaderFile = compConf.get("paths/loader-template", None) if not loaderFile: # use default templates if version=="build": #loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-build.tmpl.js") # TODO: test-wise using generic template loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js") else: #loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader-source.tmpl.js") loaderFile = os.path.join(filetool.root(), os.pardir, "data", "generator", "loader.tmpl.js") template = filetool.read(loaderFile) return template # --------------------------------------------------------------- if not parts: return "" result = "" vals = {} packages = script.packagesSortedSimple() loader_with_boot = self._job.get("packages/loader-with-boot", True) # stringify data in globalCodes for entry in globalCodes: globalCodes[entry] = json.dumpsCode(globalCodes[entry]) # undo damage done by simplejson to raw strings with escapes \\ -> \ globalCodes[entry] = globalCodes[entry].replace('\\\\\\', '\\').replace(r'\\', '\\') # " gets tripple escaped, therefore the first .replace() vals.update(globalCodes) if version=="build": vals["Resources"] = json.dumpsCode({}) # TODO: undo Resources from globalCodes!!! vals["Boot"] = '"%s"' % boot if version == "build": vals["BootPart"] = bootCode else: vals["BootPart"] = "" # fake package data for key, package in enumerate(packages): vals["BootPart"] += "qx.$$packageData['%d']={};\n" % key # Translate part information to JavaScript vals["Parts"] = partsMap(script) # Translate URI data to JavaScript #vals["Uris"] = packageUrisToJS1(packages, version) vals["Uris"] = packageUrisToJS(packages, version) vals["Uris"] = json.dumpsCode(vals["Uris"]) # Add potential extra scripts vals["UrisBefore"] = [] if self._job.get("add-script", False): additional_scripts = self._job.get("add-script",[]) for additional_script in additional_scripts: vals["UrisBefore"].append(additional_script["uri"]) vals["UrisBefore"] = json.dumpsCode(vals["UrisBefore"]) # Whether boot package is inline if version == "source": vals["BootIsInline"] = json.dumpsCode(False) else: vals["BootIsInline"] = json.dumpsCode(loader_with_boot) # Closure package information cParts = {} if version == "build": for part in script.parts: if not loader_with_boot or part != "boot": cParts[part] = True vals["ClosureParts"] = json.dumpsCode(cParts) # Package Hashes vals["PackageHashes"] = {} for key, package in enumerate(packages): if package.hash: vals["PackageHashes"][key] = package.hash else: vals["PackageHashes"][key] = "%d" % key # fake code package hashes in source ver. vals["PackageHashes"] = json.dumpsCode(vals["PackageHashes"]) # Script hook for qx.$$loader.decodeUris() function vals["DecodeUrisPlug"] = "" if decodeUrisFile: plugCode = filetool.read(self._config.absPath(decodeUrisFile)) # let it bomb if file can't be read vals["DecodeUrisPlug"] = plugCode.strip() # Enable "?nocache=...." for script loading? vals["NoCacheParam"] = "true" if self._job.get("compile-options/uris/add-nocache-param", True) else "false" # Locate and load loader basic script template = loadTemplate(bootCode) # Fill template gives result result = fillTemplate(vals, template) return result ## # shallow layer above generateBootCode(), and its only client def generateBootScript(globalCodes, script, bootPackage="", compileType="build"): self._console.info("Generating boot script...") if not self._job.get("packages/i18n-with-boot", True): # remove I18N info from globalCodes, so they don't go into the loader globalCodes["Translations"] = {} globalCodes["Locales"] = {} else: if compileType == "build": # also remove them here, as this info is now with the packages globalCodes["Translations"] = {} globalCodes["Locales"] = {} plugCodeFile = compConf.get("code/decode-uris-plug", False) if compileType == "build": filepackages = [(x.file,) for x in packages] bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants, settings, bootPackage, globalCodes, compileType, plugCodeFile, format) else: filepackages = [x.classes for x in packages] bootContent = generateBootCode(parts, filepackages, boot, script, compConf, variants={}, settings={}, bootCode=None, globalCodes=globalCodes, version=compileType, decodeUrisFile=plugCodeFile, format=format) return bootContent def getPackageData(package): data = {} data["resources"] = package.data.resources data["translations"] = package.data.translations data["locales"] = package.data.locales data = json.dumpsCode(data) data += ';\n' return data def compilePackage(packageIndex, package): self._console.info("Compiling package #%s:" % packageIndex, False) self._console.indent() # Compile file content pkgCode = self._treeCompiler.compileClasses(package.classes, variants, optimize, format) pkgData = getPackageData(package) hash = sha.getHash(pkgData + pkgCode)[:12] # first 12 chars should be enough isBootPackage = packageIndex == 0 if isBootPackage: compiledContent = ("qx.$$packageData['%s']=" % hash) + pkgData + pkgCode else: compiledContent = u'''qx.$$packageData['%s']=%s\n''' % (hash, pkgData) compiledContent += u'''qx.Part.$$notifyLoad("%s", function() {\n%s\n});''' % (hash, pkgCode) # package.hash = hash # to fill qx.$$loader.packageHashes in generateBootScript() self._console.debug("Done: %s" % self._computeContentSize(compiledContent)) self._console.outdent() return compiledContent ## # takes an array of (po-data, locale-data) dict pairs # merge all po data and all cldr data in a single dict each def mergeTranslationMaps(transMaps): poData = {} cldrData = {} for pac_dat, loc_dat in transMaps: for loc in pac_dat: if loc not in poData: poData[loc] = {} poData[loc].update(pac_dat[loc]) for loc in loc_dat: if loc not in cldrData: cldrData[loc] = {} cldrData[loc].update(loc_dat[loc]) return (poData, cldrData) # -- Main - runCompiled ------------------------------------------------ # Early return compileType = self._job.get("compile/type", "") if compileType not in ("build", "source"): return packages = script.packagesSortedSimple() parts = script.parts boot = script.boot variants = script.variants libraries = script.libraries self._treeCompiler = treeCompiler self._variants = variants self._script = script self._console.info("Generate %s version..." % compileType) self._console.indent() # - Evaluate job config --------------------- # Compile config compConf = self._job.get("compile-options") compConf = ExtMap(compConf) # Whether the code should be formatted format = compConf.get("code/format", False) script.scriptCompress = compConf.get("paths/gzip", False) # Read in settings settings = self.getSettings() script.settings = settings # Read libraries libs = self._job.get("library", []) # Get translation maps locales = compConf.get("code/locales", []) translationMaps = self.getTranslationMaps(packages, variants, locales) # Read in base file name fileRelPath = getOutputFile(compileType) filePath = self._config.absPath(fileRelPath) script.baseScriptPath = filePath if compileType == "build": # read in uri prefixes scriptUri = compConf.get('uris/script', 'script') scriptUri = Path.posifyPath(scriptUri) fileUri = getFileUri(scriptUri) # for resource list resourceUri = compConf.get('uris/resource', 'resource') resourceUri = Path.posifyPath(resourceUri) else: # source version needs place where the app HTML ("index.html") lives self.approot = self._config.absPath(compConf.get("paths/app-root", "")) resourceUri = None scriptUri = None # Get global script data (like qxlibraries, qxresources,...) globalCodes = {} globalCodes["Settings"] = settings globalCodes["Variants"] = self.generateVariantsCode(variants) globalCodes["Libinfo"] = self.generateLibInfoCode(libs, format, resourceUri, scriptUri) # add synthetic output lib if scriptUri: out_sourceUri= scriptUri else: out_sourceUri = self._computeResourceUri({'class': ".", 'path': os.path.dirname(script.baseScriptPath)}, OsPath(""), rType="class", appRoot=self.approot) out_sourceUri = os.path.normpath(out_sourceUri.encodedValue()) globalCodes["Libinfo"]['__out__'] = { 'sourceUri': out_sourceUri } globalCodes["Resources"] = self.generateResourceInfoCode(script, settings, libraries, format) globalCodes["Translations"],\ globalCodes["Locales"] = mergeTranslationMaps(translationMaps) # Potentally create dedicated I18N packages i18n_as_parts = not self._job.get("packages/i18n-with-boot", True) if i18n_as_parts: script = self.generateI18NParts(script, globalCodes) self.writePackages([p for p in script.packages if getattr(p, "__localeflag", False)], script) if compileType == "build": # - Specific job config --------------------- # read in compiler options optimize = compConf.get("code/optimize", []) self._treeCompiler.setOptimize(optimize) # - Generating packages --------------------- self._console.info("Generating packages...") self._console.indent() bootPackage = "" for packageIndex, package in enumerate(packages): package.compiled = compilePackage(packageIndex, package) self._console.outdent() if not len(packages): raise RuntimeError("No valid boot package generated.") # - Put loader and packages together ------- loader_with_boot = self._job.get("packages/loader-with-boot", True) # handle loader and boot package if not loader_with_boot: loadPackage = Package(0) # make a dummy Package for the loader packages.insert(0, loadPackage) # attach file names (do this before calling generateBootScript) for package, fileName in zip(packages, self.packagesFileNames(script.baseScriptPath, len(packages))): package.file = os.path.basename(fileName) if self._job.get("compile-options/paths/scripts-add-hash", False): package.file = self._fileNameWithHash(package.file, package.hash) # generate and integrate boot code if loader_with_boot: # merge loader code with first package bootCode = generateBootScript(globalCodes, script, packages[0].compiled) packages[0].compiled = bootCode else: loaderCode = generateBootScript(globalCodes, script) packages[0].compiled = loaderCode # write packages self.writePackages(packages, script) # ---- 'source' version ------------------------------------------------ else: sourceContent = generateBootScript(globalCodes, script, bootPackage="", compileType=compileType) # Construct file name resolvedFilePath = self._resolveFileName(filePath, variants, settings) # Save result file filetool.save(resolvedFilePath, sourceContent) if compConf.get("paths/gzip"): filetool.gzip(resolvedFilePath, sourceContent) self._console.outdent() self._console.debug("Done: %s" % self._computeContentSize(sourceContent)) self._console.outdent() self._console.outdent() return # runCompiled()