def sourcesLoaded(self, locale, moduleName, modulePath): self.rebuildNeeded = False module = self.currentBuild.files[locale][moduleName] templates = buildutil.dirEntries(modulePath + "/tmpl") for path in templates: if not path.endswith(".moustache.html"): continue path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl") contents = self.projectBuilder.modifiedFiles.read(locale, path) if contents: module[path] = contents self.rebuildNeeded = True if not self.rebuildNeeded: return print " Loading Hogan templates..." module["__templates__"] = u"Modules.%s.templates={};" % moduleName templates = buildutil.dirEntries(modulePath + "/tmpl") for path in templates: if not path.endswith(".moustache.html"): continue path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl") template = None templateId = None for line in module[path].splitlines(): if line.startswith("<!-- template"): try: templateId = re.findall(r"id=\"(.*)\"", line)[0] except IndexError: raise BuildError("Template is missing an ID in file %s" % os.path.basename(path)) template = u"" elif line.startswith("<!-- /template"): template = template.replace(" href=\"#\"", " href=\"javascript:void(0)\"") pipes = subprocess.Popen(self.shermanDir + "/features/hogan/precompile.js", shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE) compiledTemplate = "" while pipes.poll() == None: (stdoutdata, stderrdata) = pipes.communicate(input = template) if stderrdata != None or pipes.returncode != 0: raise BuildError("Error compiling Moustache template %s: %s" % (os.path.basename(path), stderrdata)) compiledTemplate += stdoutdata module["__templates__"] += "Modules.%s.templates[\"%s\"] = new Hogan.Template(%s);\n" % (moduleName, templateId, compiledTemplate) template = None else: if template is not None: template += line
def manifestLoaded(self, moduleName, modulePath, manifest): try: paths = [] for source in manifest["sources"]: if not "runJsLint" in source or source["runJsLint"] == True: path = self.projectBuilder.resolveFile( source["path"], modulePath + "/js") paths.append(path) errors = [] if JSLINT_MP: pool = multiprocessing.Pool(JSLINT_MP_PROCESSES) results = [ pool.apply_async(JSLint(), ( self.shermanDir, path, )) for path in paths ] for result in results: r = result.get() if not r[0]: errors.append(r[1]) else: for path in paths: jsLint = JSLint() if not jsLint(self.shermanDir, path): errors.append(path) for error in errors: print " jslint error in file %s" % error except Exception, exception: raise BuildError("Could not run jslint for module %s" % moduleName, exception)
def sourcesConcatenated(self, locale, moduleName, modulePath): module = self.currentBuild.files[locale][moduleName] if "__styles__" in module: if not "bundled" in self.options or self.options["bundled"]: styles = module["__styles__"].replace("\"", "\\\"").replace( "\n", "\\n") module["__concat__"] += "Modules.%s.css = \"%s\";\n" % ( moduleName, styles) else: try: styles = module["__styles__"] fileName = buildutil.getDestinationFileName( moduleName, None, styles, None, "css") if not os.path.exists( self.buildDir + "/" + fileName ): # avoid multiple locales writing the same file with codecs.open(self.buildDir + "/" + fileName, "w", "utf-8") as f: f.write(styles) module["__output__"].append(fileName) except Exception, exception: raise BuildError( "Could not write CSS output file for module %s" % moduleName, exception)
def sourcesConcatenated(self, locale, moduleName, modulePath): print " Minifying JavaScript..." module = self.currentBuild.files[locale][moduleName] js = module["__concat__"] js = self.preMinifyTricks(js) tempPath = self.projectBuilder.buildDir + "/" + moduleName + ".js" with codecs.open(tempPath, "w", "utf-8") as f: f.write(js) p = subprocess.Popen("java -jar %s/other/closure-compiler/compiler.jar --js %s --jscomp_off nonStandardJsDocs" % (self.shermanDir, tempPath), shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) js = "" err = "" while p.poll() == None: (stdoutdata, stderrdata) = p.communicate() js += stdoutdata err += stderrdata if p.returncode == 0: os.remove(tempPath) else: raise BuildError("Minification of module %s failed: %s" % (moduleName, err)) module["__concat__"] = js
def sourcesConcatenated(self, locale, moduleName, modulePath): print " Applying translations..." module = self.currentBuild.files[locale][moduleName] if not "__allTranslations__" in module: return module["__concat__"] = self.applyTranslations( module["__allTranslations__"], locale, moduleName, module["__concat__"], escaping="javascript") if "__templates__" in module: module["__templates__"] = self.applyTranslations( module["__allTranslations__"], locale, moduleName, module["__templates__"], escaping="html") localeTranslations = {} for key, values in sorted(module["__translations__"].items()): if locale in values: value = values[locale] else: raise BuildError( "No translation possible for key %s and locale %s" % (key, locale)) localeTranslations[key] = value module["__concat__"] += "Modules.%s.translations = %s;\n" % ( moduleName, json.dumps(localeTranslations))
def sourcesLoaded(self, locale, moduleName, modulePath): self.rebuildNeeded = False path = modulePath + "/i18n/translations.json" if not os.path.exists(path): return module = self.currentBuild.files[locale][moduleName] try: contents = self.projectBuilder.modifiedFiles.read(locale, path) if contents: print " Loading translations..." translations = json.loads(contents) module["__translations__"] = translations module["__allTranslations__"] = {} for prerequisite in module["__manifest__"]["dependencies"]: prereqModule = self.currentBuild.files[locale][ prerequisite] if "__translations__" in prereqModule: module["__allTranslations__"].update( prereqModule["__translations__"]) module["__allTranslations__"].update(translations) self.rebuildNeeded = True except Exception, exception: raise BuildError( "Could not load translations for module %s" % moduleName, exception)
def sourcesLoaded(self, locale, moduleName, modulePath): module = self.currentBuild.files[locale][moduleName] if not "__styles__" in module: return if not self.projectBuilder.features["css"].isRebuildNeeded( locale, moduleName, modulePath): return print " Compiling LESS..." PIPE = subprocess.PIPE pipes = subprocess.Popen(self.shermanDir + "/features/less/compile.js", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) css = "" while pipes.poll() == None: (stdoutdata, stderrdata) = pipes.communicate(input=module["__styles__"]) if stderrdata != None or pipes.returncode != 0: raise BuildError("Error compiling LESS styles: %s" % stderrdata) css += stdoutdata module["__styles__"] = css
def sourcesLoaded(self, locale, moduleName, modulePath): self.rebuildNeeded = False module = self.currentBuild.files[locale][moduleName] templates = buildutil.dirEntries(modulePath + "/tmpl") for path in templates: if not path.endswith(".tmpl.html"): continue path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl") contents = self.projectBuilder.modifiedFiles.read(locale, path) if contents: module[path] = contents self.rebuildNeeded = True if not self.rebuildNeeded: return print " Loading jQuery templates..." module["__templates__"] = u"" wsReplacer = re.compile(r"[ \t\n\r\f\v]+", flags=re.MULTILINE) spaceReplacer = re.compile(r"> <", flags=re.MULTILINE) templates = buildutil.dirEntries(modulePath + "/tmpl") for path in templates: if not path.endswith(".tmpl.html"): continue path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl") template = None templateId = None for line in module[path].splitlines(): if line.startswith("<!-- template"): try: templateId = re.findall(r"id=\"(.*)\"", line)[0] except IndexError: raise BuildError( "Template is missing an ID in file %s" % os.path.basename(path)) template = u"" elif line.startswith("<!-- /template"): template = template.replace( " href=\"#\"", " href=\"javascript:void(0)\"") template = wsReplacer.sub(r" ", template).strip() template = spaceReplacer.sub(r"><", template) module[ "__templates__"] += u"$.template(\"%s.%s\", '%s');\n" % ( moduleName, templateId, buildutil.jsStringEscape(template)) template = None else: if template is not None: template += line
def loadModuleManifest(self, moduleName, modulePath): try: contents = self.modifiedFiles.read("*", modulePath + "/manifest.json") if contents: manifest = json.loads(contents) for locale in self.locales: self.currentBuild.files[locale][moduleName][ "__manifest__"] = manifest if not "dependencies" in manifest: raise BuildError( "No dependencies specified for module %s" % moduleName) except Exception, exception: raise BuildError( "Could not load manifest for module %s" % moduleName, exception)
def mimeTypeForExtension(extension): if extension[0] == ".": extension = extension[1:] extension = extension.lower() if extension in extensionToMimeTypeMap: return extensionToMimeTypeMap[extension] else: raise BuildError("Cannot determine MIME type for extension %s" % extension)
def loadProjectManifest(self): try: with open(self.config.projectManifest, "r") as f: contents = f.read() self.projectManifest = json.loads(contents) except Exception, exception: raise BuildError( "Could not load manifest file %s" % self.config.projectManifest, exception)
def resolveFile(self, path, directory=""): if path[0] == "/": if os.path.exists(self.projectDir + path): return self.projectDir + path elif os.path.exists(self.shermanDir + path): return self.shermanDir + path else: if os.path.exists(directory + "/" + path): return directory + "/" + path raise BuildError("Missing resource: %s" % path)
def isRebuildNeeded(self, locale, moduleName, modulePath): if self.rebuildNeeded: return True for featureName in self.features: try: if self.features[featureName].isRebuildNeeded( locale, moduleName, modulePath): return True except Exception, exception: raise BuildError("Exception in feature %s" % featureName, exception)
def do_GET(self): try: search = "" path = self.path qi = path.find("?") if qi > -1: search = path[qi:] path = path[0:qi] if path == "/": self.path = "/cgi-bin/index.py" + search builder.loadProjectManifest() shutil.copy(builder.config.projectManifest, builder.buildDir) builder.build() if path.startswith("/") and path.endswith(".js"): moduleName = path[1:-3] if moduleName == "boot": builder.loadProjectManifest() shutil.copy(builder.config.projectManifest, builder.buildDir) builder.build() localeFiles = builder.currentBuild.files[ builder.projectManifest["defaultLocale"]] if moduleName in localeFiles: module = localeFiles[moduleName] for fileName in module["__output__"]: if fileName.endswith(".js"): with open( builder.buildDir + "/" + fileName, "r") as f: self.wfile.write(f.read()) return raise BuildError( "Module %s did not generate a JavaScript output file" % moduleName) if builder.config.simulateHighLatency: time.sleep(0.2 + 2 * random.random()) CGIHTTPServer.CGIHTTPRequestHandler.do_GET(self) except BuildError, error: error.printMessage() self.wfile.write("<html>") self.wfile.write("<body>") self.wfile.write("<h1>Build Error</h1>") self.wfile.write("<pre>%s</pre>" % str(error)) self.wfile.write( "<p>(check console output for more info)</p>") self.wfile.write("</body>") self.wfile.write("</html>")
def modulesWritten(self): self.buildFullBootModule() for locale in self.projectBuilder.locales: resources = {} bootScripts = [] for module in self.projectBuilder.modules: moduleName = module["name"] if moduleName == "inline": continue module = self.currentBuild.files[locale][moduleName] jsFileName = None for fileName in module["__output__"]: if fileName.endswith(".js"): jsFileName = fileName if not jsFileName: raise BuildError("Module %s did not generate a JavaScript output file" % moduleName) resources[moduleName] = {} resources[moduleName][locale] = jsFileName resources[moduleName]["dependencies"] = module["__manifest__"]["dependencies"] if moduleName == "boot": bootScripts.append("[static_base]/" + jsFileName) if "essential" in module["__manifest__"] and module["__manifest__"]["essential"]: resources[moduleName]["essential"] = True bootJson = { "bootscripts": bootScripts, "resources": resources, "locale": locale, "baseurl": "[static_base]/", "config": "[config]" } if "tiles" in self.projectBuilder.features: bootJson["tileModuleDependencies"] = self.projectBuilder.features["tiles"].tileModuleDependencies bootJson = json.dumps(bootJson) fileName = buildutil.getDestinationFileName("boot", None, bootJson, locale, "json") with codecs.open(self.buildDir + "/" + fileName, "w", "utf-8") as f: f.write(bootJson) bootHash = buildutil.getContentHash(bootJson) self.writeVersionFile("__versionjson__", locale, bootHash) self.reinstateBootModule()
def applyTranslations(self, allTranslations, locale, moduleName, content, escaping): for match in re.finditer(r"\[\[([a-zA-Z0-9_]+)\]\]", content): key = match.group(1) if not key in allTranslations: raise BuildError("Undefined text key %s in module %s" % (key, moduleName)) translations = allTranslations[key] if not locale in translations: raise BuildError( "Translation not provided for text key %s and locale %s" % (key, locale)) value = translations[locale] if escaping == "html": value = buildutil.htmlEscape(value) elif escaping == "javascript": value = json.dumps(value) if len(value) > 1: value = value[1:-1] # strip of the quotes on both sides else: raise BuildError("Unrecognized escaping type %s" % escaping) content = content.replace("[[%s]]" % key, value) return content
def loadSources(self, locale, moduleName, modulePath): module = self.currentBuild.files[locale][moduleName] try: self.rebuildNeeded = False for source in module["__manifest__"]["sources"]: path = self.resolveFile(source["path"], modulePath + "/js") contents = self.modifiedFiles.read(locale, path) if contents: module[path] = contents self.rebuildNeeded = True except Exception, exception: raise BuildError( "Could not load sources for module %s" % moduleName, exception)
def writeFiles(self, locale, moduleName, modulePath): print " Writing output file..." module = self.currentBuild.files[locale][moduleName] try: contents = module["__concat__"] filename = buildutil.getDestinationFileName( moduleName, None, contents, locale, "js") with codecs.open(self.buildDir + "/" + filename, "w", "utf-8") as f: f.write(contents) module["__output__"].append(filename) except Exception, exception: raise BuildError( "Could not write output file for module %s" % moduleName, exception)
def concatenateSources(self, locale, moduleName, modulePath): module = self.currentBuild.files[locale][moduleName] try: print " Concatenating sources..." concat = "" for source in module["__manifest__"]["sources"]: path = self.resolveFile(source["path"], modulePath + "/js") content = module[path].strip() if len(content) > 0: content += ("\n" if content[-1] == ";" else ";\n") concat += content module["__concat__"] = concat except Exception, exception: raise BuildError( "Could not concatenate sources for module %s" % moduleName, exception)
def sourcesLoaded(self, locale, moduleName, modulePath): self.rebuildNeeded = False module = self.currentBuild.files[locale][moduleName] if not "styles" in module["__manifest__"]: return try: for style in module["__manifest__"]["styles"]: path = self.projectBuilder.resolveFile(style["path"], modulePath + "/css") contents = self.projectBuilder.modifiedFiles.read(locale, path) if contents: module[path] = contents self.rebuildNeeded = True except Exception, exception: raise BuildError( "Could not load styles for module %s" % moduleName, exception)
class Feature(ShermanFeature): @ShermanFeature.priority(10) def manifestLoaded(self, moduleName, modulePath, manifest): try: paths = [] for source in manifest["sources"]: if not "runJsLint" in source or source["runJsLint"] == True: path = self.projectBuilder.resolveFile( source["path"], modulePath + "/js") paths.append(path) errors = [] if JSLINT_MP: pool = multiprocessing.Pool(JSLINT_MP_PROCESSES) results = [ pool.apply_async(JSLint(), ( self.shermanDir, path, )) for path in paths ] for result in results: r = result.get() if not r[0]: errors.append(r[1]) else: for path in paths: jsLint = JSLint() if not jsLint(self.shermanDir, path): errors.append(path) for error in errors: print " jslint error in file %s" % error except Exception, exception: raise BuildError("Could not run jslint for module %s" % moduleName, exception) if len(errors) > 0: shutil.move(self.shermanDir + "/report.html", self.projectDir + "/report.html") raise BuildError( "jslint errors detected, see report.html for details.")
def buildModule(self, moduleName): defaultLocale = self.projectManifest["defaultLocale"] if self.currentBuild.files[defaultLocale][moduleName]["__built__"]: return # module already built self.currentBuild.files[defaultLocale][moduleName]["__built__"] = True if os.path.exists(self.projectDir + "/modules/" + moduleName): modulePath = self.projectDir + "/modules/" + moduleName elif os.path.exists(self.shermanDir + "/modules/" + moduleName): modulePath = self.shermanDir + "/modules/" + moduleName else: raise BuildError("Could not find module %s" % moduleName) self.loadModuleManifest(moduleName, modulePath) # make sure dependencies are built before the module itself for prerequisite in self.currentBuild.files[defaultLocale][moduleName][ "__manifest__"]["dependencies"]: self.buildModule(prerequisite) print "Building module %s..." % moduleName defaultLocale = self.projectManifest["defaultLocale"] manifest = self.currentBuild.files[defaultLocale][moduleName][ "__manifest__"] self.invokeFeatures("manifestLoaded", moduleName, modulePath, manifest) for locale in self.locales: print " Processing locale %s..." % locale self.loadSources(locale, moduleName, modulePath) if self.isRebuildNeeded(locale, moduleName, modulePath): self.removeOldFiles(locale, moduleName, modulePath) self.concatenateSources(locale, moduleName, modulePath) self.writeFiles(locale, moduleName, modulePath)
def loadFeatures(self): if not "features" in self.target: raise BuildError( "No features defined in manifest file %s for target %s" % (self.config.projectManifest, self.config.target)) paths = [self.projectDir + "/features", self.shermanDir + "/features"] for feature in self.target["features"]: featureName = feature["name"] if featureName in self.features: continue # already loaded try: f = None (f, path, description) = imp.find_module(featureName, paths) sys.path.append(self.shermanDir) module = imp.load_module(featureName, f, path, description) self.features[featureName] = module.Feature( shermanfeature.Options(projectDir=self.projectDir, shermanDir=self.shermanDir, buildDir=self.buildDir, projectBuilder=self, featureOptions=feature["options"] if "options" in feature else {})) #except ImportError, error: # raise BuildError("Could not load feature %s" % featureName, error) #except Exception, exception: # raise BuildError("Exception while loading feature %s" % featureName, exception) finally: if f: f.close() print "Enabled feature: %s" % featureName
def invokeFeatures(self, hookName, *args): hooks = [] for featureName in self.features: feature = self.features[featureName] if hookName in feature.__class__.__dict__: function = feature.__class__.__dict__[hookName] else: for base in feature.__class__.__bases__: if hookName in base.__dict__: function = base.__dict__[hookName] break hooks.append((function.priority if "priority" in function.func_dict else shermanfeature.DEFAULT_PRIORITY, featureName, (feature, function))) hooks = sorted(hooks, key=lambda hook: hook[0]) for hook in hooks: (priority, featureName, (feature, function)) = hook try: function(feature, *args) except Exception, exception: raise BuildError("Exception in feature %s" % featureName, exception)
class ProjectBuilder(object): def __init__(self, config): self.shermanDir = os.path.abspath(os.path.dirname(__file__)) self.projectDir = os.path.dirname(config.projectManifest) self.buildDir = config.buildDir or tempfile.mkdtemp( ".build", "sherman.") self.config = config self.projectManifest = None self.locales = [] self.target = None self.modules = [] self.rebuildNeeded = False self.features = {} class ModifiedFiles(object): def __init__(self): self.timestamps = {} def read(self, locale, path): path = os.path.abspath(path) key = locale + ":" + path mtime = os.stat(path).st_mtime if key in self.timestamps and mtime == self.timestamps[key]: return False self.timestamps[key] = mtime with codecs.open(path, "r", "utf-8") as f: return f.read() self.modifiedFiles = ModifiedFiles() class Build(object): # locale => { # module => { # filename => content, # "__concat__" => content, # "__manifest__" => object, # "__output__" => [ filename, ... ] # } # } files = {} self.currentBuild = Build() self.loadProjectManifest() def resolveFile(self, path, directory=""): if path[0] == "/": if os.path.exists(self.projectDir + path): return self.projectDir + path elif os.path.exists(self.shermanDir + path): return self.shermanDir + path else: if os.path.exists(directory + "/" + path): return directory + "/" + path raise BuildError("Missing resource: %s" % path) def loadProjectManifest(self): try: with open(self.config.projectManifest, "r") as f: contents = f.read() self.projectManifest = json.loads(contents) except Exception, exception: raise BuildError( "Could not load manifest file %s" % self.config.projectManifest, exception) if not "locales" in self.projectManifest or len( self.projectManifest["locales"]) == 0: raise BuildError("No locales defined in manifest file %s" % self.config.projectManifest) if not "defaultLocale" in self.projectManifest: raise BuildError("No default locale defined in manifest file %s" % self.config.projectManifest) if not self.projectManifest["defaultLocale"] in self.projectManifest[ "locales"]: raise BuildError( "Default locale not defined among locales in manifest file %s" % self.config.projectManifest) self.locales = self.projectManifest["locales"] if not self.config.target in self.projectManifest["targets"]: raise BuildError("No target %s defined in manifest file %s" % (self.config.target, self.config.projectManifest)) self.target = self.projectManifest["targets"][self.config.target] if "modules" in self.target: self.modules = self.target["modules"] else: if not "modules" in self.projectManifest: raise BuildError( "No modules defined in manifest file %s for target %s" % (self.config.projectManifest, self.config.target)) self.modules = self.projectManifest["modules"] if "options" in self.target: if "fileNamePattern" in self.target["options"]: buildutil.fileNamePattern = self.target["options"][ "fileNamePattern"]