def sourcesLoaded(self, locale, moduleName, modulePath):
self.rebuildNeeded = False
module = self.currentBuild.files[locale][moduleName]
templates = buildutil.dirEntries(modulePath + "/tmpl")
for path in templates:
if not path.endswith(".moustache.html"):
continue
path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl")
contents = self.projectBuilder.modifiedFiles.read(locale, path)
if contents:
module[path] = contents
self.rebuildNeeded = True
if not self.rebuildNeeded:
return
print " Loading Hogan templates..."
module["__templates__"] = u"Modules.%s.templates={};" % moduleName
templates = buildutil.dirEntries(modulePath + "/tmpl")
for path in templates:
if not path.endswith(".moustache.html"):
continue
path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl")
template = None
templateId = None
for line in module[path].splitlines():
if line.startswith("<!-- template"):
try:
templateId = re.findall(r"id=\"(.*)\"", line)[0]
except IndexError:
raise BuildError("Template is missing an ID in file %s" % os.path.basename(path))
template = u""
elif line.startswith("<!-- /template"):
template = template.replace(" href=\"#\"", " href=\"javascript:void(0)\"")
pipes = subprocess.Popen(self.shermanDir + "/features/hogan/precompile.js", shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
compiledTemplate = ""
while pipes.poll() == None:
(stdoutdata, stderrdata) = pipes.communicate(input = template)
if stderrdata != None or pipes.returncode != 0:
raise BuildError("Error compiling Moustache template %s: %s" % (os.path.basename(path), stderrdata))
compiledTemplate += stdoutdata
module["__templates__"] += "Modules.%s.templates[\"%s\"] = new Hogan.Template(%s);\n" % (moduleName, templateId, compiledTemplate)
template = None
else:
if template is not None:
template += line
def manifestLoaded(self, moduleName, modulePath, manifest):
try:
paths = []
for source in manifest["sources"]:
if not "runJsLint" in source or source["runJsLint"] == True:
path = self.projectBuilder.resolveFile(
source["path"], modulePath + "/js")
paths.append(path)
errors = []
if JSLINT_MP:
pool = multiprocessing.Pool(JSLINT_MP_PROCESSES)
results = [
pool.apply_async(JSLint(), (
self.shermanDir,
path,
)) for path in paths
]
for result in results:
r = result.get()
if not r[0]:
errors.append(r[1])
else:
for path in paths:
jsLint = JSLint()
if not jsLint(self.shermanDir, path):
errors.append(path)
for error in errors:
print " jslint error in file %s" % error
except Exception, exception:
raise BuildError("Could not run jslint for module %s" % moduleName,
exception)
def sourcesConcatenated(self, locale, moduleName, modulePath):
module = self.currentBuild.files[locale][moduleName]
if "__styles__" in module:
if not "bundled" in self.options or self.options["bundled"]:
styles = module["__styles__"].replace("\"", "\\\"").replace(
"\n", "\\n")
module["__concat__"] += "Modules.%s.css = \"%s\";\n" % (
moduleName, styles)
else:
try:
styles = module["__styles__"]
fileName = buildutil.getDestinationFileName(
moduleName, None, styles, None, "css")
if not os.path.exists(
self.buildDir + "/" + fileName
): # avoid multiple locales writing the same file
with codecs.open(self.buildDir + "/" + fileName, "w",
"utf-8") as f:
f.write(styles)
module["__output__"].append(fileName)
except Exception, exception:
raise BuildError(
"Could not write CSS output file for module %s" %
moduleName, exception)
def sourcesConcatenated(self, locale, moduleName, modulePath):
print " Minifying JavaScript..."
module = self.currentBuild.files[locale][moduleName]
js = module["__concat__"]
js = self.preMinifyTricks(js)
tempPath = self.projectBuilder.buildDir + "/" + moduleName + ".js"
with codecs.open(tempPath, "w", "utf-8") as f:
f.write(js)
p = subprocess.Popen("java -jar %s/other/closure-compiler/compiler.jar --js %s --jscomp_off nonStandardJsDocs" %
(self.shermanDir, tempPath), shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
js = ""
err = ""
while p.poll() == None:
(stdoutdata, stderrdata) = p.communicate()
js += stdoutdata
err += stderrdata
if p.returncode == 0:
os.remove(tempPath)
else:
raise BuildError("Minification of module %s failed: %s" % (moduleName, err))
module["__concat__"] = js
def sourcesConcatenated(self, locale, moduleName, modulePath):
print " Applying translations..."
module = self.currentBuild.files[locale][moduleName]
if not "__allTranslations__" in module:
return
module["__concat__"] = self.applyTranslations(
module["__allTranslations__"],
locale,
moduleName,
module["__concat__"],
escaping="javascript")
if "__templates__" in module:
module["__templates__"] = self.applyTranslations(
module["__allTranslations__"],
locale,
moduleName,
module["__templates__"],
escaping="html")
localeTranslations = {}
for key, values in sorted(module["__translations__"].items()):
if locale in values:
value = values[locale]
else:
raise BuildError(
"No translation possible for key %s and locale %s" %
(key, locale))
localeTranslations[key] = value
module["__concat__"] += "Modules.%s.translations = %s;\n" % (
moduleName, json.dumps(localeTranslations))
def sourcesLoaded(self, locale, moduleName, modulePath):
self.rebuildNeeded = False
path = modulePath + "/i18n/translations.json"
if not os.path.exists(path):
return
module = self.currentBuild.files[locale][moduleName]
try:
contents = self.projectBuilder.modifiedFiles.read(locale, path)
if contents:
print " Loading translations..."
translations = json.loads(contents)
module["__translations__"] = translations
module["__allTranslations__"] = {}
for prerequisite in module["__manifest__"]["dependencies"]:
prereqModule = self.currentBuild.files[locale][
prerequisite]
if "__translations__" in prereqModule:
module["__allTranslations__"].update(
prereqModule["__translations__"])
module["__allTranslations__"].update(translations)
self.rebuildNeeded = True
except Exception, exception:
raise BuildError(
"Could not load translations for module %s" % moduleName,
exception)
def sourcesLoaded(self, locale, moduleName, modulePath):
module = self.currentBuild.files[locale][moduleName]
if not "__styles__" in module:
return
if not self.projectBuilder.features["css"].isRebuildNeeded(
locale, moduleName, modulePath):
return
print " Compiling LESS..."
PIPE = subprocess.PIPE
pipes = subprocess.Popen(self.shermanDir + "/features/less/compile.js",
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
css = ""
while pipes.poll() == None:
(stdoutdata,
stderrdata) = pipes.communicate(input=module["__styles__"])
if stderrdata != None or pipes.returncode != 0:
raise BuildError("Error compiling LESS styles: %s" %
stderrdata)
css += stdoutdata
module["__styles__"] = css
def sourcesLoaded(self, locale, moduleName, modulePath):
self.rebuildNeeded = False
module = self.currentBuild.files[locale][moduleName]
templates = buildutil.dirEntries(modulePath + "/tmpl")
for path in templates:
if not path.endswith(".tmpl.html"):
continue
path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl")
contents = self.projectBuilder.modifiedFiles.read(locale, path)
if contents:
module[path] = contents
self.rebuildNeeded = True
if not self.rebuildNeeded:
return
print " Loading jQuery templates..."
module["__templates__"] = u""
wsReplacer = re.compile(r"[ \t\n\r\f\v]+", flags=re.MULTILINE)
spaceReplacer = re.compile(r"> <", flags=re.MULTILINE)
templates = buildutil.dirEntries(modulePath + "/tmpl")
for path in templates:
if not path.endswith(".tmpl.html"):
continue
path = self.projectBuilder.resolveFile(path, modulePath + "/tmpl")
template = None
templateId = None
for line in module[path].splitlines():
if line.startswith("<!-- template"):
try:
templateId = re.findall(r"id=\"(.*)\"", line)[0]
except IndexError:
raise BuildError(
"Template is missing an ID in file %s" %
os.path.basename(path))
template = u""
elif line.startswith("<!-- /template"):
template = template.replace(
" href=\"#\"", " href=\"javascript:void(0)\"")
template = wsReplacer.sub(r" ", template).strip()
template = spaceReplacer.sub(r"><", template)
module[
"__templates__"] += u"$.template(\"%s.%s\", '%s');\n" % (
moduleName, templateId,
buildutil.jsStringEscape(template))
template = None
else:
if template is not None:
template += line
def loadModuleManifest(self, moduleName, modulePath):
try:
contents = self.modifiedFiles.read("*",
modulePath + "/manifest.json")
if contents:
manifest = json.loads(contents)
for locale in self.locales:
self.currentBuild.files[locale][moduleName][
"__manifest__"] = manifest
if not "dependencies" in manifest:
raise BuildError(
"No dependencies specified for module %s" % moduleName)
except Exception, exception:
raise BuildError(
"Could not load manifest for module %s" % moduleName,
exception)
def mimeTypeForExtension(extension):
if extension[0] == ".":
extension = extension[1:]
extension = extension.lower()
if extension in extensionToMimeTypeMap:
return extensionToMimeTypeMap[extension]
else:
raise BuildError("Cannot determine MIME type for extension %s" %
extension)
def loadProjectManifest(self):
try:
with open(self.config.projectManifest, "r") as f:
contents = f.read()
self.projectManifest = json.loads(contents)
except Exception, exception:
raise BuildError(
"Could not load manifest file %s" %
self.config.projectManifest, exception)
def resolveFile(self, path, directory=""):
if path[0] == "/":
if os.path.exists(self.projectDir + path):
return self.projectDir + path
elif os.path.exists(self.shermanDir + path):
return self.shermanDir + path
else:
if os.path.exists(directory + "/" + path):
return directory + "/" + path
raise BuildError("Missing resource: %s" % path)
def isRebuildNeeded(self, locale, moduleName, modulePath):
if self.rebuildNeeded:
return True
for featureName in self.features:
try:
if self.features[featureName].isRebuildNeeded(
locale, moduleName, modulePath):
return True
except Exception, exception:
raise BuildError("Exception in feature %s" % featureName,
exception)
def do_GET(self):
try:
search = ""
path = self.path
qi = path.find("?")
if qi > -1:
search = path[qi:]
path = path[0:qi]
if path == "/":
self.path = "/cgi-bin/index.py" + search
builder.loadProjectManifest()
shutil.copy(builder.config.projectManifest,
builder.buildDir)
builder.build()
if path.startswith("/") and path.endswith(".js"):
moduleName = path[1:-3]
if moduleName == "boot":
builder.loadProjectManifest()
shutil.copy(builder.config.projectManifest,
builder.buildDir)
builder.build()
localeFiles = builder.currentBuild.files[
builder.projectManifest["defaultLocale"]]
if moduleName in localeFiles:
module = localeFiles[moduleName]
for fileName in module["__output__"]:
if fileName.endswith(".js"):
with open(
builder.buildDir + "/" + fileName,
"r") as f:
self.wfile.write(f.read())
return
raise BuildError(
"Module %s did not generate a JavaScript output file"
% moduleName)
if builder.config.simulateHighLatency:
time.sleep(0.2 + 2 * random.random())
CGIHTTPServer.CGIHTTPRequestHandler.do_GET(self)
except BuildError, error:
error.printMessage()
self.wfile.write("<html>")
self.wfile.write("<body>")
self.wfile.write("<h1>Build Error</h1>")
self.wfile.write("<pre>%s</pre>" % str(error))
self.wfile.write(
"<p>(check console output for more info)</p>")
self.wfile.write("</body>")
self.wfile.write("</html>")
def modulesWritten(self):
self.buildFullBootModule()
for locale in self.projectBuilder.locales:
resources = {}
bootScripts = []
for module in self.projectBuilder.modules:
moduleName = module["name"]
if moduleName == "inline":
continue
module = self.currentBuild.files[locale][moduleName]
jsFileName = None
for fileName in module["__output__"]:
if fileName.endswith(".js"):
jsFileName = fileName
if not jsFileName:
raise BuildError("Module %s did not generate a JavaScript output file" % moduleName)
resources[moduleName] = {}
resources[moduleName][locale] = jsFileName
resources[moduleName]["dependencies"] = module["__manifest__"]["dependencies"]
if moduleName == "boot":
bootScripts.append("[static_base]/" + jsFileName)
if "essential" in module["__manifest__"] and module["__manifest__"]["essential"]:
resources[moduleName]["essential"] = True
bootJson = {
"bootscripts": bootScripts,
"resources": resources,
"locale": locale,
"baseurl": "[static_base]/",
"config": "[config]"
}
if "tiles" in self.projectBuilder.features:
bootJson["tileModuleDependencies"] = self.projectBuilder.features["tiles"].tileModuleDependencies
bootJson = json.dumps(bootJson)
fileName = buildutil.getDestinationFileName("boot", None, bootJson, locale, "json")
with codecs.open(self.buildDir + "/" + fileName, "w", "utf-8") as f:
f.write(bootJson)
bootHash = buildutil.getContentHash(bootJson)
self.writeVersionFile("__versionjson__", locale, bootHash)
self.reinstateBootModule()
def applyTranslations(self, allTranslations, locale, moduleName, content,
escaping):
for match in re.finditer(r"\[\[([a-zA-Z0-9_]+)\]\]", content):
key = match.group(1)
if not key in allTranslations:
raise BuildError("Undefined text key %s in module %s" %
(key, moduleName))
translations = allTranslations[key]
if not locale in translations:
raise BuildError(
"Translation not provided for text key %s and locale %s" %
(key, locale))
value = translations[locale]
if escaping == "html":
value = buildutil.htmlEscape(value)
elif escaping == "javascript":
value = json.dumps(value)
if len(value) > 1:
value = value[1:-1] # strip of the quotes on both sides
else:
raise BuildError("Unrecognized escaping type %s" % escaping)
content = content.replace("[[%s]]" % key, value)
return content
def loadSources(self, locale, moduleName, modulePath):
module = self.currentBuild.files[locale][moduleName]
try:
self.rebuildNeeded = False
for source in module["__manifest__"]["sources"]:
path = self.resolveFile(source["path"], modulePath + "/js")
contents = self.modifiedFiles.read(locale, path)
if contents:
module[path] = contents
self.rebuildNeeded = True
except Exception, exception:
raise BuildError(
"Could not load sources for module %s" % moduleName, exception)
def writeFiles(self, locale, moduleName, modulePath):
print " Writing output file..."
module = self.currentBuild.files[locale][moduleName]
try:
contents = module["__concat__"]
filename = buildutil.getDestinationFileName(
moduleName, None, contents, locale, "js")
with codecs.open(self.buildDir + "/" + filename, "w",
"utf-8") as f:
f.write(contents)
module["__output__"].append(filename)
except Exception, exception:
raise BuildError(
"Could not write output file for module %s" % moduleName,
exception)
def concatenateSources(self, locale, moduleName, modulePath):
module = self.currentBuild.files[locale][moduleName]
try:
print " Concatenating sources..."
concat = ""
for source in module["__manifest__"]["sources"]:
path = self.resolveFile(source["path"], modulePath + "/js")
content = module[path].strip()
if len(content) > 0:
content += ("\n" if content[-1] == ";" else ";\n")
concat += content
module["__concat__"] = concat
except Exception, exception:
raise BuildError(
"Could not concatenate sources for module %s" % moduleName,
exception)
def sourcesLoaded(self, locale, moduleName, modulePath):
self.rebuildNeeded = False
module = self.currentBuild.files[locale][moduleName]
if not "styles" in module["__manifest__"]:
return
try:
for style in module["__manifest__"]["styles"]:
path = self.projectBuilder.resolveFile(style["path"],
modulePath + "/css")
contents = self.projectBuilder.modifiedFiles.read(locale, path)
if contents:
module[path] = contents
self.rebuildNeeded = True
except Exception, exception:
raise BuildError(
"Could not load styles for module %s" % moduleName, exception)
class Feature(ShermanFeature):
@ShermanFeature.priority(10)
def manifestLoaded(self, moduleName, modulePath, manifest):
try:
paths = []
for source in manifest["sources"]:
if not "runJsLint" in source or source["runJsLint"] == True:
path = self.projectBuilder.resolveFile(
source["path"], modulePath + "/js")
paths.append(path)
errors = []
if JSLINT_MP:
pool = multiprocessing.Pool(JSLINT_MP_PROCESSES)
results = [
pool.apply_async(JSLint(), (
self.shermanDir,
path,
)) for path in paths
]
for result in results:
r = result.get()
if not r[0]:
errors.append(r[1])
else:
for path in paths:
jsLint = JSLint()
if not jsLint(self.shermanDir, path):
errors.append(path)
for error in errors:
print " jslint error in file %s" % error
except Exception, exception:
raise BuildError("Could not run jslint for module %s" % moduleName,
exception)
if len(errors) > 0:
shutil.move(self.shermanDir + "/report.html",
self.projectDir + "/report.html")
raise BuildError(
"jslint errors detected, see report.html for details.")
def buildModule(self, moduleName):
defaultLocale = self.projectManifest["defaultLocale"]
if self.currentBuild.files[defaultLocale][moduleName]["__built__"]:
return # module already built
self.currentBuild.files[defaultLocale][moduleName]["__built__"] = True
if os.path.exists(self.projectDir + "/modules/" + moduleName):
modulePath = self.projectDir + "/modules/" + moduleName
elif os.path.exists(self.shermanDir + "/modules/" + moduleName):
modulePath = self.shermanDir + "/modules/" + moduleName
else:
raise BuildError("Could not find module %s" % moduleName)
self.loadModuleManifest(moduleName, modulePath)
# make sure dependencies are built before the module itself
for prerequisite in self.currentBuild.files[defaultLocale][moduleName][
"__manifest__"]["dependencies"]:
self.buildModule(prerequisite)
print "Building module %s..." % moduleName
defaultLocale = self.projectManifest["defaultLocale"]
manifest = self.currentBuild.files[defaultLocale][moduleName][
"__manifest__"]
self.invokeFeatures("manifestLoaded", moduleName, modulePath, manifest)
for locale in self.locales:
print " Processing locale %s..." % locale
self.loadSources(locale, moduleName, modulePath)
if self.isRebuildNeeded(locale, moduleName, modulePath):
self.removeOldFiles(locale, moduleName, modulePath)
self.concatenateSources(locale, moduleName, modulePath)
self.writeFiles(locale, moduleName, modulePath)
def loadFeatures(self):
if not "features" in self.target:
raise BuildError(
"No features defined in manifest file %s for target %s" %
(self.config.projectManifest, self.config.target))
paths = [self.projectDir + "/features", self.shermanDir + "/features"]
for feature in self.target["features"]:
featureName = feature["name"]
if featureName in self.features:
continue # already loaded
try:
f = None
(f, path, description) = imp.find_module(featureName, paths)
sys.path.append(self.shermanDir)
module = imp.load_module(featureName, f, path, description)
self.features[featureName] = module.Feature(
shermanfeature.Options(projectDir=self.projectDir,
shermanDir=self.shermanDir,
buildDir=self.buildDir,
projectBuilder=self,
featureOptions=feature["options"]
if "options" in feature else {}))
#except ImportError, error:
# raise BuildError("Could not load feature %s" % featureName, error)
#except Exception, exception:
# raise BuildError("Exception while loading feature %s" % featureName, exception)
finally:
if f:
f.close()
print "Enabled feature: %s" % featureName
def invokeFeatures(self, hookName, *args):
hooks = []
for featureName in self.features:
feature = self.features[featureName]
if hookName in feature.__class__.__dict__:
function = feature.__class__.__dict__[hookName]
else:
for base in feature.__class__.__bases__:
if hookName in base.__dict__:
function = base.__dict__[hookName]
break
hooks.append((function.priority if "priority" in function.func_dict
else shermanfeature.DEFAULT_PRIORITY, featureName,
(feature, function)))
hooks = sorted(hooks, key=lambda hook: hook[0])
for hook in hooks:
(priority, featureName, (feature, function)) = hook
try:
function(feature, *args)
except Exception, exception:
raise BuildError("Exception in feature %s" % featureName,
exception)
class ProjectBuilder(object):
def __init__(self, config):
self.shermanDir = os.path.abspath(os.path.dirname(__file__))
self.projectDir = os.path.dirname(config.projectManifest)
self.buildDir = config.buildDir or tempfile.mkdtemp(
".build", "sherman.")
self.config = config
self.projectManifest = None
self.locales = []
self.target = None
self.modules = []
self.rebuildNeeded = False
self.features = {}
class ModifiedFiles(object):
def __init__(self):
self.timestamps = {}
def read(self, locale, path):
path = os.path.abspath(path)
key = locale + ":" + path
mtime = os.stat(path).st_mtime
if key in self.timestamps and mtime == self.timestamps[key]:
return False
self.timestamps[key] = mtime
with codecs.open(path, "r", "utf-8") as f:
return f.read()
self.modifiedFiles = ModifiedFiles()
class Build(object):
# locale => {
# module => {
# filename => content,
# "__concat__" => content,
# "__manifest__" => object,
# "__output__" => [ filename, ... ]
# }
# }
files = {}
self.currentBuild = Build()
self.loadProjectManifest()
def resolveFile(self, path, directory=""):
if path[0] == "/":
if os.path.exists(self.projectDir + path):
return self.projectDir + path
elif os.path.exists(self.shermanDir + path):
return self.shermanDir + path
else:
if os.path.exists(directory + "/" + path):
return directory + "/" + path
raise BuildError("Missing resource: %s" % path)
def loadProjectManifest(self):
try:
with open(self.config.projectManifest, "r") as f:
contents = f.read()
self.projectManifest = json.loads(contents)
except Exception, exception:
raise BuildError(
"Could not load manifest file %s" %
self.config.projectManifest, exception)
if not "locales" in self.projectManifest or len(
self.projectManifest["locales"]) == 0:
raise BuildError("No locales defined in manifest file %s" %
self.config.projectManifest)
if not "defaultLocale" in self.projectManifest:
raise BuildError("No default locale defined in manifest file %s" %
self.config.projectManifest)
if not self.projectManifest["defaultLocale"] in self.projectManifest[
"locales"]:
raise BuildError(
"Default locale not defined among locales in manifest file %s"
% self.config.projectManifest)
self.locales = self.projectManifest["locales"]
if not self.config.target in self.projectManifest["targets"]:
raise BuildError("No target %s defined in manifest file %s" %
(self.config.target, self.config.projectManifest))
self.target = self.projectManifest["targets"][self.config.target]
if "modules" in self.target:
self.modules = self.target["modules"]
else:
if not "modules" in self.projectManifest:
raise BuildError(
"No modules defined in manifest file %s for target %s" %
(self.config.projectManifest, self.config.target))
self.modules = self.projectManifest["modules"]
if "options" in self.target:
if "fileNamePattern" in self.target["options"]:
buildutil.fileNamePattern = self.target["options"][
"fileNamePattern"]
