def _handleResources(script, generator, filtered=True): def createResourceInfo(res, resval): resinfo = [ { "target": "resource", "data": { res : resval }} ] #filetool.save(approot+"/data/resource/" + res + ".json", json.dumpsCode(resinfo)) return resinfo def copyResource(res, library): sourcepath = os.path.join(library._resourcePath, res) targetpath = approot + "/resource/" + res filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) return # ---------------------------------------------------------------------- context.console.info("Processing resources: ", False) approot = context.jobconf.get("provider/app-root", "./provider") filetool.directory(approot+"/data") filetool.directory(approot+"/resource") # quick copy of runLogResources, for fast results packages = script.packagesSorted() parts = script.parts variants = script.variants allresources = {} if filtered: # -- the next call is fake, just to populate package.data.resources! _ = generator._codeGenerator.generateResourceInfoCode(script, generator._settings, context.jobconf.get("library",[])) for packageId, package in enumerate(packages): allresources.update(package.data.resources) else: # get the main library mainlib = [x for x in script.libraries if x.namespace == script.namespace][0] reslist = mainlib.getResources() allresources = Script.createResourceStruct(reslist, updateOnlyExistingSprites = False) # get resource info resinfos = {} numResources = len(allresources) for num,res in enumerate(allresources): context.console.progress(num+1, numResources) # fake a classId-like resourceId ("a.b.c"), for filter matching resId = os.path.splitext(res)[0] resId = resId.replace("/", ".") if filtered and not passesOutputfilter(resId): continue resinfos[res] = createResourceInfo(res, allresources[res]) # extract library name space if isinstance(allresources[res], types.ListType): # it's an image = [14, 14, u'png', u'qx' [, u'qx/decoration/Modern/checkradio-combined.png', 0, 0]] library_ns = allresources[res][3] else: # html page etc. = "qx" library_ns = allresources[res] if library_ns: # library_ns == '' means embedded image -> no copying library = libraries[library_ns] copyResource(res, library) filetool.save(approot+"/data/resource/resources.json", json.dumpsCode(resinfos)) return
def generateResourceInfoCode(self, script, settings, libraries, format=False): def addResourceInfoToPackages(script): for package in script.packages: package_resources = [] # TODO: the next is a hack, since package.classes are still id's package_classes = [x for x in script.classesObj if x.id in package.classes] for clazz in package_classes: package_resources.extend(clazz.resources) package.data.resources = Script.createResourceStruct(package_resources, formatAsTree=resources_tree, updateOnlyExistingSprites=True) return # -- main -------------------------------------------------------------- compConf = self._job.get ("compile-options") compConf = ExtMap (compConf) resources_tree = compConf.get ("code/resources-tree", False) classes = Class.mapResourcesToClasses (libraries, script.classesObj, self._job.get("asset-let", {})) filteredResources = [] for clazz in classes: filteredResources.extend(clazz.resources) resdata = Script.createResourceStruct (filteredResources, formatAsTree=resources_tree, updateOnlyExistingSprites=True) # add resource info to packages addResourceInfoToPackages(script) return resdata # end: generateResourceInfoCode()
def addResourceInfoToPackages(script): for package in script.packages: package_resources = [] # TODO: the next is a hack, since package.classes are still id's package_classes = [x for x in script.classesObj if x.id in package.classes] for clazz in package_classes: package_resources.extend(clazz.resources) package.data.resources = Script.createResourceStruct(package_resources, formatAsTree=resources_tree, updateOnlyExistingSprites=True) return
def packagesResourceInfo(self, script): classes = Class.mapResourcesToClasses (script.libraries, script.classesObj, self._job.get("asset-let", {})) for package in script.packages: package_resources = [] package_classes = package.classes for clazz in package_classes: package_resources.extend(clazz.resources) package.data.resources = Script.createResourceStruct(package_resources, formatAsTree=False, updateOnlyExistingSprites=True) return script
def _handleResources(script, generator, filtered=True): def createResourceInfo(res, resval): resinfo = [{"target": "resource", "data": {res: resval}}] #filetool.save(approot+"/data/resource/" + res + ".json", json.dumpsCode(resinfo)) return resinfo skip_expression = re.compile( r'%s' % '|'.join(filetool.VERSIONCONTROL_DIR_PATTS), re.I) def copyResource(res, library): if skip_expression.search(os.path.basename(res)): return sourcepath = os.path.join(library.resourcePath, res) targetpath = approot + "/resource/" + res filetool.directory(os.path.dirname(targetpath)) shutil.copy(sourcepath, targetpath) #copier = copytool.CopyTool(context.console) #args = ['-x', ','.join(filetool.VERSIONCONTROL_DIR_PATTS), sourcepath, targetpath] #copier.parse_args(args) #copier.do_work() return # ---------------------------------------------------------------------- context.console.info("Processing resources: ", False) approot = context.jobconf.get("provider/app-root", "./provider") filetool.directory(approot + "/data") filetool.directory(approot + "/resource") # quick copy of runLogResources, for fast results packages = script.packagesSorted() parts = script.parts variants = script.variants allresources = {} if filtered: generator._codeGenerator.packagesResourceInfo(script) for packageId, package in enumerate(packages): allresources.update(package.data.resources) else: # get the main library mainlib = [ x for x in script.libraries if x.namespace == script.namespace ][0] reslist = mainlib.getResources() allresources = Script.createResourceStruct( reslist, updateOnlyExistingSprites=False) # get resource info resinfos = {} numResources = len(allresources) for num, res in enumerate(allresources): context.console.progress(num + 1, numResources) # fake a classId-like resourceId ("a.b.c"), for filter matching resId = os.path.splitext(res)[0] resId = resId.replace("/", ".") if filtered and not passesOutputfilter(resId): continue resinfos[res] = createResourceInfo(res, allresources[res]) # extract library name space if isinstance( allresources[res], types.ListType ): # it's an image = [14, 14, u'png', u'qx' [, u'qx/decoration/Modern/checkradio-combined.png', 0, 0]] library_ns = allresources[res][3] else: # html page etc. = "qx" library_ns = allresources[res] if library_ns: # library_ns == '' means embedded image -> no copying library = libraries[library_ns] copyResource(res, library) filetool.save(approot + "/data/resource/resources.json", json.dumpsCode(resinfos)) return
class Generator(object): def __init__(self, context): global console, interruptRegistry interruptRegistry = context['interruptRegistry'] self._context = context self._config = context['config'] #config self._job = context['jobconf'] #config.getJob(job) self._console = context['console'] #console_ self._variants = {} self._settings = {} self.approot = None self._classesObj = {} # {'cid':generator.code.Class} if 'cache' in context: # in case the Generator want to use a common cache object self._cache = context['cache'] else: cache_path = self._job.get("cache/compile", "cache") cache_path = self._config.absPath(cache_path) self._cache = Cache( cache_path, **{ 'interruptRegistry': context['interruptRegistry'], 'console': context['console'], 'cache/downloads': self._job.get("cache/downloads", cache_path + "/downloads"), 'cache/invalidate-on-tool-change': self._job.get('cache/invalidate-on-tool-change', False), }) context['cache'] = self._cache console = self._console console.resetFilter() # reset potential filters from a previous job Context.cache = self._cache return ## # This is the main dispatch method to run a single job. It uses the top- # level keys of the job description to run all necessary methods. In order # to do so, it also sets up a lot of tool chain infrastructure. def run(self): # -- Helpers ---------------------------------------------------------- def listJobTriggers(): return { "api": { "type": "JClassDepJob" }, "collect-environment-info": { "type": "JSimpleJob" }, "copy-files": { "type": "JSimpleJob" }, "combine-images": { "type": "JSimpleJob" }, "clean-files": { "type": "JSimpleJob" }, "copy-resources": { #"type" : "JClassDepJob" "type": "JCompileJob", }, "compile": { "type": "JCompileJob", }, "compile-source": { "type": "JCompileJob", }, "compile-dist": { "type": "JCompileJob", }, "fix-files": { "type": "JClassDepJob", }, "lint-check": { "type": "JClassDepJob", }, "log": { "type": "JCompileJob", }, "manifest-validate": { "type": "JSimpleJob" }, "migrate-files": { "type": "JSimpleJob", # this might change once we stop to shell exit to an external script }, "pretty-print": { "type": "JClassDepJob", }, "provider": { #"type" : "JCompileJob", "type": "JClassDepJob", }, "shell": { "type": "JSimpleJob" }, "slice-images": { "type": "JSimpleJob" }, "translate": { "type": "JClassDepJob" }, "simulate": { "type": "JSimpleJob" }, "watch-files": { "type": "JSimpleJob" }, "web-server": { "type": "JSimpleJob" }, "web-server-config": { "type": "JSimpleJob" }, } ## # Invoke the DependencyLoader to calculate the list of required classes # from include/exclude settings def computeClassList(includeWithDeps, excludeWithDeps, includeNoDeps, excludeWithDepsHard, script, verifyDeps=False): self._console.info("Collecting classes ", feed=False) self._console.indent() classList = self._depLoader.getClassList(includeWithDeps, excludeWithDeps, includeNoDeps, excludeWithDepsHard, script, verifyDeps) # with generator.code.ClassList(): #classList = ClassList(self._libraries, includeWithDeps, includeNoDeps, excludeWithDeps, variants, buildType) #classList = classList.calculate(verifyDeps) self._console.outdent() return classList ## # Invoke the PartBuilder to compute the packages for the configured # parts. def partsConfigFromClassList(includeWithDeps, excludeWithDeps, script): def evalPackagesConfig(excludeWithDeps, classList, variants): # Reading configuration partsCfg = self._job.get("packages/parts", {}) # Expanding expressions self._console.debug("Expanding include expressions...") partIncludes = {} for partId in partsCfg: partIncludes[partId] = textutil.expandGlobs( partsCfg[partId]['include'], self._classesObj) # Computing packages #boot, partPackages, packageClasses = self._partBuilder.getPackages(partIncludes, excludeWithDeps, self._context, script) partPackages, _ = self._partBuilder.getPackages( partIncludes, excludeWithDeps, self._context, script) packageClasses = script.packagesSorted() #return boot, partPackages, packageClasses return script.boot, script.parts, packageClasses # ----------------------------------------------------------- classList = script.classes variants = script.variants self._partBuilder = PartBuilder(self._console, self._depLoader) # Check for a 'packages' configuration in the job if 0: # this branch should work, but doesn't; # create a synthetic job key and let evalPackagesConfig do the rest if not self._job.get("packages"): package_config = { "parts": { "boot": { "include": includeWithDeps } }, "init": "boot" } self._job.setFeature("packages", package_config) ( boot, partPackages, # partPackages[partId]=[0,1,3] packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...] ) = evalPackagesConfig(excludeWithDeps, classList, variants) else: if self._job.get("packages"): ( boot, partPackages, # partPackages[partId]=[0,1,3] packageClasses # packageClasses[0]=['qx.Class','qx.bom.Stylesheet',...] ) = evalPackagesConfig(excludeWithDeps, classList, variants) else: # Emulate a 'boot' part boot = "boot" partPackages = {"boot": [0]} packageClasses = [classList] # patch script object script.boot = boot packageObj = Package(0) packageObj.classes = script.classesObj script.packages.append(packageObj) partObj = Part("boot") partObj.packages.append(packageObj) initial_deps = list( set(includeWithDeps).difference(script.excludes) ) # defining classes from config minus expanded excludes partObj.initial_deps = initial_deps partObj.deps = initial_deps[:] script.parts = {"boot": partObj} return boot, partPackages, packageClasses ## # Get the variants from the config def getVariants(confkey): variants = {} variantsConfig = self._job.get(confkey, {}) variantsRuntime = self._variants for key in variantsConfig: variants[key] = variantsConfig[key] for key in variantsRuntime: variants[key] = [variantsRuntime[key]] # sanity check variants for key, val in variants.items(): if not isinstance(val, types.ListType): #raise ValueError("Config error: Variant values must be lists: \"%s\":\"%r\"" % (key,val)) # allow scalar values variants[key] = [val] return variants ## # Get the exclude definition from the config def getExcludes(excludeCfg): #excludeCfg = self._job.get("exclude", []) excludeWithDeps = [] excludeWithDepsHard = [] if len(excludeCfg) == 0: return [], [] else: ignore_excludes = self._job.get("config-warnings/exclude", []) if '*' not in ignore_excludes: # check individually complain_excludes = [ x for x in excludeCfg if not x in ignore_excludes ] if complain_excludes: self._console.warn("Excludes may break code (%r)" % complain_excludes) # Splitting lists self._console.debug("Preparing exclude configuration...") excludeWithDeps, excludeWithDepsHard = textutil.splitPrefixedStrings( excludeCfg) # Configuration feedback self._console.indent() if len(excludeWithDepsHard) > 0: #if self._job.get("config-warnings/exclude", True): # self._console.warn("Excluding without dependencies is not supported, treating them as normal excludes: %r" % excludeWithDepsHard) #excludeWithDeps.extend(excludeWithDepsHard) #excludeWithDepsHard = [] pass self._console.debug( "Excluding %s items smart, %s items explicit" % (len(excludeWithDeps), len(excludeWithDepsHard))) self._console.outdent() # Resolve regexps self._console.indent() self._console.debug("Expanding expressions...") for list_ in (excludeWithDeps, excludeWithDepsHard): lst = list_[:] list_[:] = [] for elem in lst: try: expanded = textutil.expandGlob(elem, self._classesObj) except RuntimeError, ex: self._console.warn("Invalid exclude block: %s\n%s" % (excludeCfg, ex)) else: list_.extend(expanded) self._console.outdent() return excludeWithDeps, excludeWithDepsHard ## # Get the include definition from the config # # @param includeCfg [] self._job.get("include", []) # def getIncludes(includeCfg): # Splitting lists self._console.debug("Preparing include configuration...") includeWithDeps, includeNoDeps = textutil.splitPrefixedStrings( includeCfg) self._console.indent() if len(includeWithDeps) > 0 or len(includeNoDeps) > 0: # Configuration feedback self._console.debug( "Including %s items smart, %s items explicit" % (len(includeWithDeps), len(includeNoDeps))) if len(includeNoDeps) > 0: if self._job.get("config-warnings/include", True): self._console.warn( "Explicitly included classes may not work") # ?! # Resolve regexps self._console.debug("Expanding expressions...") try: includeWithDeps = textutil.expandGlobs( includeWithDeps, self._classesObj) includeNoDeps = textutil.expandGlobs( includeNoDeps, self._classesObj) except RuntimeError: self._console.error("Invalid include block: %s" % includeCfg) raise elif self._job.get("packages"): # Special part include handling self._console.info("Including part classes...") partsCfg = partsCfg = self._job.get("packages/parts", {}) includeWithDeps = [] for partId in partsCfg: includeWithDeps.extend(partsCfg[partId]) # Configuration feedback self._console.debug( "Including %s items smart, %s items explicit" % (len(includeWithDeps), len(includeNoDeps))) # Resolve regexps self._console.debug("Expanding expressions...") includeWithDeps = textutil.expandGlobs(includeWithDeps, self._classesObj) self._console.outdent() return includeWithDeps, includeNoDeps ## # Console output about variant being generated def printVariantInfo(variantSetNum, variants, variantSets, variantData): if len(variantSets) < 2: # only log when more than 1 set return variantStr = json.dumps(variants, ensure_ascii=False) self._console.head("Processing variant set %s/%s" % (variantSetNum + 1, len(variantSets))) # Debug variant combination hasVariants = False for key in variants: if len(variantData[key]) > 1: hasVariants = True if hasVariants: self._console.info("Switched variants:") self._console.indent() for key in variants: if len(variantData[key]) > 1: self._console.info("%s = %s" % (key, variants[key])) self._console.outdent() return def prepareGenerator(): # scanning given library paths (self._namespaces, self._classesObj, self._docs, self._translations, self._libraries) = self.scanLibrary(config.get("library", [])) # create tool chain instances self._locale = LocaleCls( self._context, self._classesObj, self._translations, self._cache, self._console, ) self._depLoader = DependencyLoader(self._classesObj, self._cache, self._console, require, use, self._context) self._codeGenerator = CodeGenerator(self._cache, self._console, self._config, self._job, self._settings, self._locale, self._classesObj) # distribute environment checks map # TODO : this could also be passed as a parameter to Class.dependencies() if "qx.core.Environment" in self._classesObj: envChecksMap = self._classesObj[ "qx.core.Environment"].extractChecksMap() for clazz in self._classesObj.values(): clazz.context['envchecksmap'] = envChecksMap ## # Safely take out a member from a set. Returns the member if it could # be removed, None otherwise. def takeout(s, m): try: s.remove(m) except KeyError: return None return m # -- Main -------------------------------------------------------------- starttime = time.time() config = self._job job = self._job require = config.get("require", {}) use = config.get("use", {}) # Apply output log filter, if any self._console.setFilter(config.get("log/filter/debug", [])) # This job's triggers triggersSet = listJobTriggers() jobKeySet = set(job.getData().keys()) jobTriggers = jobKeySet.intersection(triggersSet) # Create tool chain instances self._actionLib = ActionLib(self._config, self._console) # process simple triggers if takeout(jobTriggers, "collect-environment-info"): Logging.runCollectEnvironmentInfo(self._job, self._config) if takeout(jobTriggers, "copy-files"): FileSystem.runCopyFiles(self._job, self._config) if takeout(jobTriggers, "combine-images"): Resources.runImageCombining(self._job, self._config) if takeout(jobTriggers, "clean-files"): FileSystem.runClean(self._job, self._config, self._cache) if takeout(jobTriggers, "manifest-validate"): JsonValidation.validateManifest(self._job, self._config) if takeout(jobTriggers, "migrate-files"): CodeMaintenance.runMigration(self._job, config.get("library")) if takeout(jobTriggers, "shell"): self._actionLib.runShellCommands(self._job) if takeout(jobTriggers, "simulate"): Testing.runSimulation(self._job) if takeout(jobTriggers, "slice-images"): Resources.runImageSlicing(self._job, self._config) if takeout(jobTriggers, "watch-files"): self._actionLib.watch(self._job, self._config) if takeout(jobTriggers, "web-server"): MiniWebServer.runWebServer(self._job, self._config) if takeout(jobTriggers, "web-server-config"): MiniWebServer.generateHttpdConfig(self._job, self._config) if jobTriggers: # -- Process job triggers that require a class list (and some) prepareGenerator() # Preprocess include/exclude lists includeWithDeps, includeNoDeps = getIncludes( self._job.get("include", [])) excludeWithDeps, excludeWithDepsHard = getExcludes( self._job.get("exclude", [])) # process classdep triggers if takeout(jobTriggers, "fix-files"): CodeMaintenance.runFix(self._job, self._classesObj) if takeout(jobTriggers, "lint-check"): CodeMaintenance.runLint(self._job, self._classesObj) if takeout(jobTriggers, "translate"): Locale.runUpdateTranslation(self._job, self._classesObj, self._libraries, self._translations) if takeout(jobTriggers, "pretty-print"): self._codeGenerator.runPrettyPrinting(self._classesObj) if takeout(jobTriggers, "provider"): script = Script() script.classesObj = self._classesObj.values() environData = getVariants("environment") variantSets = util.computeCombinations(environData) script.variants = variantSets[0] script.optimize = config.get("compile-options/code/optimize", []) script.libraries = self._libraries script.namespace = self.getAppName() script.locales = config.get("compile-options/code/locales", []) CodeProvider.runProvider(script, self) if jobTriggers: # -- Process job triggers that require the full tool chain # Processing all combinations of variants environData = getVariants( "environment" ) # e.g. {'qx.debug':false, 'qx.aspects':[true,false]} variantSets = util.computeCombinations( environData) # e.g. [{'qx.debug':'on','qx.aspects':'on'},...] for variantSetNum, variantset in enumerate(variantSets): # some console output printVariantInfo(variantSetNum, variantset, variantSets, environData) script = Script( ) # a new Script object represents the target code script.classesAll = self._classesObj # for deps. analysis script.namespace = self.getAppName() script.variants = variantset script.environment = variantset script.optimize = config.get("compile-options/code/optimize", []) script.locales = config.get("compile-options/code/locales", []) script.libraries = self._libraries script.jobconfig = self._job # set source/build version if "compile" in jobTriggers: script.buildType = config.get("compile/type", "") if script.buildType not in ("source", "build", "hybrid"): raise ValueError("Unknown compile type '%s'" % script.buildType) if (script.buildType == "source" # TODO: source processing could be placed outside the variant loop or "variants" not in script. optimize # TODO: script.variants is used both declaratively (config's environment map) *and* to signal variants optimization (e.g. in Class.dependencies()) ): script.variants = {} # get current class list script.classes = computeClassList(includeWithDeps, excludeWithDeps, includeNoDeps, excludeWithDepsHard, script, verifyDeps=True) # keep the list of class objects in sync script.classesObj = [ self._classesObj[id] for id in script.classes ] if "statics" in script.optimize: featureMap = self._depLoader.registerDependeeFeatures( script.classesObj, script.variants, script.buildType) script._featureMap = featureMap else: script._featureMap = {} # set the complete exclude list for classes excludes = set(excludeWithDeps[:]) excludes.update( self._depLoader.expand_hard_excludes( excludeWithDepsHard, script)) script.excludes = list(excludes) # prepare 'script' object if set(("compile", "log")).intersection(jobTriggers): partsConfigFromClassList(includeWithDeps, excludeWithDeps, script) # Execute real tasks if "api" in jobTriggers: ApiLoader.runApiData(self._job, self._config, script, self._docs) if "copy-resources" in jobTriggers: FileSystem.runResources(self._config, script) if "compile" in jobTriggers: self._codeGenerator.runCompiled(script) if "log" in jobTriggers: Logging.runLogDependencies(self._job, script) Logging.runPrivateDebug(self._job) #Logging.runClassOrderingDebug(self._job, script) Logging.runLogUnusedClasses(self._job, script) Logging.runLogResources(self._job, script) elapsedsecs = time.time() - starttime self._console.info("Done (%dm%05.2f)" % (int(elapsedsecs / 60), elapsedsecs % 60)) return