def removeFile(self, filename):
"""Removes the given file"""
filename = self.__session.expandFileName(filename)
if os.path.exists(filename):
Console.info("Deleting file %s" % filename)
os.remove(filename)
def build(self):
"""Build static website."""
Console.info("Intializing Konstrukteur...")
Console.indent()
# Path configuration
# TODO: Use Jasy configuration instead
self.__templatePath = os.path.join("source", "template")
self.__contentPath = os.path.join("source", "content")
self.__sourcePath = os.path.join("source")
self.__pagePath = os.path.join(self.__contentPath, "page")
self.__postPath = os.path.join(self.__contentPath, "post")
if not os.path.exists(self.__templatePath):
raise RuntimeError("Path to templates not found : %s" % self.__templatePath)
if not os.path.exists(self.__contentPath):
raise RuntimeError("Path to content not found : %s" % self.__contentPath)
# A theme could be any project registered in the current session
if self.__theme:
themeProject = session.getProjectByName(self.__theme)
if not themeProject:
raise RuntimeError("Theme '%s' not found" % self.__theme)
self.__initializeTemplates()
self.__generateOutput()
def getCompressed(self, profile):
"""Returns the compressed CSS code of this item."""
field = "style:compressed[%s]-%s" % (self.id, profile.getId())
mtime = self.getModificationTime(profile)
compressed = self.project.getCache().read(field, mtime)
if compressed is None:
Console.info("Compressing tree %s...", Console.colorize(self.id, "bold"))
# Start with the merged tree (includes resolved)
tree = self.getMergedTree(profile)
# Reduce tree
Engine.reduceTree(tree, profile)
# Compress tree
compressed = Engine.compressTree(tree, profile.getCompressionLevel(), profile.getFormattingLevel())
# Store in cache
self.project.getCache().store(field, compressed, mtime)
return compressed
def write(self, filename, debug=False):
if Image is None:
raise UserError("Missing Python PIL which is required to create sprite sheets!")
img = Image.new('RGBA', (self.width, self.height))
draw = ImageDraw.Draw(img)
#draw.rectangle((0, 0, self.width, self.height), fill=(255, 255, 0, 255))
# Load images and pack them in
for block in self.blocks:
res = Image.open(block.image.src)
x, y = block.fit.x, block.fit.y
if block.rotated:
Console.debug('%s is rotated' % block.image.src)
res = res.rotate(90)
img.paste(res, (x, y))
del res
if debug:
x, y, w, h = block.fit.x, block.fit.y, block.w, block.h
draw.rectangle((x, y , x + w , y + h), outline=(0, 0, 255, 255) if block.rotated else (255, 0, 0, 255))
if debug:
for i, block in enumerate(self.packer.getUnused()):
x, y, w, h = block.x, block.y, block.w, block.h
draw.rectangle((x, y , x + w , y + h), fill=(255, 255, 0, 255))
img.save(filename)
def generate(self, path='', debug=False):
"""Generate sheets/variants."""
Console.info('Generating sprite sheet variants...')
Console.indent()
sheets, count = self.packBest()
# Write PNG files
data = {}
for pos, sheet in enumerate(sheets):
Console.info('Writing image (%dx%dpx) with %d images' % (sheet.width, sheet.height, len(sheet)))
name = 'jasysprite_%d.png' % pos
# Export
sheet.write(os.path.join(self.base, path, name), debug)
data[name] = sheet.export()
Console.outdent()
# Generate config file
Console.info('Exporting data...')
script = os.path.join(self.base, path, 'jasysprite.%s' % self.dataFormat)
writeConfig(data, script)
def __call__(self, **kwargs):
merged = {}
merged.update(self.curry)
merged.update(kwargs)
#
# SUPPORT SOME DEFAULT FEATURES CONTROLLED BY TASK PARAMETERS
#
# Allow overriding of prefix via task or cmdline parameter.
# By default use name of the task (no prefix for cleanup tasks)
if "prefix" in merged:
session.setCurrentPrefix(merged["prefix"])
del merged["prefix"]
elif "clean" in self.name:
session.setCurrentPrefix(None)
else:
session.setCurrentPrefix(self.name)
#
# EXECUTE ATTACHED FUNCTION
#
Console.header(self.__name__)
# Execute internal function
return self.func(**merged)
def __extendContent(node, call, targetBlock, stopCombineAt):
"""
Builds up a list of selector/mediaqueries to insert after the extend to produce
the @content sections on the intended selectors.
"""
for child in reversed(node):
if child:
__extendContent(child, call, targetBlock, stopCombineAt)
if node.type == "content" and hasattr(call, "rules"):
# Extends support @content as well. In this case we produce a new selector
# which matches the position of the content section and append it after
# the original extended mixin on return
Console.debug("Inserting content section into new virtual selector")
selector, media = Util.combineSelector(node, stop=stopCombineAt)
selectorNode = Node.Node(type="selector")
selectorNode.name = selector
selectorNode.append(copy.deepcopy(call.rules), "rules")
# Support media queries, too
if media:
mediaNode = Node.Node(type="media")
mediaNode.name = media
mediaNode.append(selectorNode)
targetBlock.append(mediaNode)
else:
targetBlock.append(selectorNode)
def __clean(node):
"""
Removes all empty rules.
Starting from inside out for a deep cleanup. This is a required step for the next one where we combine media
queries and selectors and need to have an easy reference point to the previous node.
"""
# Process children first
for child in reversed(node):
if child is not None:
__clean(child)
if hasattr(node, "rules") and len(node.rules) == 0:
Console.debug("Cleaning up empty selector/mixin/@media/@supports at line %s" % node.line)
node.parent.remove(node)
elif node.type == "content":
Console.debug("Cleaning up left over @content at line %s" % node.line)
node.parent.remove(node)
elif node.type == "meta":
Console.debug("Cleaning up left over @meta at line %s" % node.line)
node.parent.remove(node)
elif node.type == "block" and node.parent.type in ("sheet", "block"):
Console.debug("Inlining content of unnecessary block node at line %s" % node.line)
node.parent.insertAllReplace(node, node)
elif node.type == "root" and len(node) == 0:
Console.debug("Cleaning up left over @root at line %s" % node.line)
node.parent.remove(node)
def patch(node, permutation):
""" Replaces all occourences with incoming values """
modified = False
if node.type == "identifier" or node.type == "dot":
if node.type == "dot":
name = assembleDot(node)
else:
name = node.value
replacement = __translateToValue(permutation.get(name))
if replacement:
Console.debug("Found permutation query (%s => %s) in line %s", name, replacement, node.line)
replacementNode = Parser.parseExpression(replacement)
node.parent.replace(node, replacementNode)
modified = True
# Process children
for child in reversed(node):
if child != None:
if patch(child, permutation):
modified = True
return modified
def loadCommands(self, objectName, fileName, encoding="utf-8"):
"""Loads new commands into the session wide command registry."""
counter = 0
commands = self.__commands
# Method for being used as a decorator to share methods to the outside
def share(func):
name = "%s.%s" % (objectName, func.__name__)
if name in commands:
raise Exception("Command %s already exists!" % name)
commands[name] = func
nonlocal counter
counter += 1
return func
# Execute given file. Using clean new global environment
# but add additional decorator for allowing to define shared methods
# and the session object (self).
code = open(fileName, "r", encoding=encoding).read()
exec(compile(code, os.path.abspath(fileName), "exec"), {"share" : share, "session" : self})
# Export destination name as global
Console.info("Imported %s.", Console.colorize("%s commands" % counter, "magenta"))
return counter
def printTasks(indent=16):
"""Prints out a list of all avaible tasks and their descriptions."""
for name in sorted(__taskRegistry):
obj = __taskRegistry[name]
formattedName = name
if obj.__doc__:
space = (indent - len(name)) * " "
print(" %s: %s%s" % (formattedName, space, Console.colorize(obj.__doc__, "magenta")))
else:
print(" %s" % formattedName)
if obj.availableArgs or obj.hasFlexArgs:
text = ""
if obj.availableArgs:
text += Util.hyphenate("--%s <var>" % " <var> --".join(obj.availableArgs))
if obj.hasFlexArgs:
if text:
text += " ..."
else:
text += "--<name> <var>"
print(" %s" % (Console.colorize(text, "grey")))
def store(self, key, value, timestamp=None, transient=False, inMemory=True):
"""
Stores the given value.
Default timestamp goes to the current time. Can be modified
to the time of an other files modification date etc.
Transient enables in-memory cache for the given value
"""
if self.__hashkeys:
key = hashlib.sha1(key.encode("ascii")).hexdigest()
if inMemory:
self.__transient[key] = value
if transient:
return
if not timestamp:
timestamp = time.time()
try:
self.__shelve[key + "-timestamp"] = timestamp
self.__shelve[key] = value
except pickle.PicklingError as err:
Console.error("Failed to store enty: %s" % key)
def resume(self):
"""Resumes the session after it has been paused."""
Console.info("Resuming session...")
for project in self.__projects:
project.resume()
def loadLibrary(self, objectName, fileName, encoding="utf-8", doc=None):
"""
Creates a new object inside the user API (jasyscript.py) with the given name
containing all @share'd functions and fields loaded from the given file.
"""
if objectName in self.__scriptEnvironment:
raise UserError("Could not import library %s as the object name %s is already used." % (fileName, objectName))
# Create internal class object for storing shared methods
class Shared(object): pass
exportedModule = Shared()
exportedModule.__doc__ = doc or "Imported from %s" % os.path.relpath(fileName, os.getcwd())
counter = 0
# Method for being used as a decorator to share methods to the outside
def share(func):
nonlocal counter
setattr(exportedModule, func.__name__, func)
counter += 1
return func
# Execute given file. Using clean new global environment
# but add additional decorator for allowing to define shared methods
# and the session object (self).
code = open(fileName, "r", encoding=encoding).read()
exec(compile(code, os.path.abspath(fileName), "exec"), {"share" : share, "session" : self})
# Export destination name as global
Console.debug("Importing %s shared methods under %s...", counter, objectName)
self.__scriptEnvironment[objectName] = exportedModule
return counter
def __generateTranslationBundle(self):
"""
Returns a translation object for the given language containing
all relevant translation files for the current project set.
"""
language = self.getCurrentPermutation().get("locale")
if language is None:
return None
if language in self.__translationBundles:
return self.__translationBundles[language]
Console.info("Creating translation bundle: %s", language)
Console.indent()
# Initialize new Translation object with no project assigned
# This object is used to merge all seperate translation instances later on.
combined = jasy.item.Translation.TranslationItem(None, id=language)
relevantLanguages = self.__expandLanguage(language)
# Loop structure is build to prefer finer language matching over project priority
for currentLanguage in reversed(relevantLanguages):
for project in self.__projects:
for translation in project.getTranslations().values():
if translation.getLanguage() == currentLanguage:
Console.debug("Adding %s entries from %s @ %s...", len(translation.getTable()), currentLanguage, project.getName())
combined += translation
Console.debug("Combined number of translations: %s", len(combined.getTable()))
Console.outdent()
self.__translationBundles[language] = combined
return combined
def __build(self):
""" Build static website """
self.__parseContent()
self.__outputContent()
Console.info("Done processing website")
def __init__(self, session, assetManager=None, compressionLevel=1, formattingLevel=0):
Console.info("Initializing OutputManager...")
Console.indent()
Console.info("Formatting Level: %s", formattingLevel)
Console.info("Compression Level: %s", compressionLevel)
self.__session = session
self.__assetManager = assetManager
self.__fileManager = FileManager(session)
self.__scriptOptimization = Optimization()
self.__compressGeneratedCode = False
self.__kernelClasses = []
if compressionLevel > 0:
self.__scriptOptimization.enable("variables")
self.__scriptOptimization.enable("declarations")
self.__compressGeneratedCode = True
if compressionLevel > 1:
self.__scriptOptimization.enable("blocks")
self.__scriptOptimization.enable("privates")
self.__scriptFormatting = Formatting()
if formattingLevel > 0:
self.__scriptFormatting.enable("semicolon")
self.__scriptFormatting.enable("comma")
Console.outdent()
def __combineSiblings(node):
"""Backwards processing and insertion into previous sibling if both are declarations"""
length = len(node)
pos = length - 1
while pos > 0:
child = node[pos]
prevChild = node[pos - 1]
# Special FOR loop optimization, emulate faked VAR
if child.type == "for" and prevChild.type == "var":
setup = getattr(child, "setup", None)
if setup and setup.type == "var":
Console.debug("Removing for-loop setup section at line %s" % setup.line)
child.remove(setup)
child = setup
# Combine declarations of VAR statements
if child.type == "var" and prevChild.type == "var":
# debug("Combining var statement at line %s" % child.line)
# Fix loop through casting node to list()
for variable in list(child):
prevChild.append(variable)
if child in node:
node.remove(child)
pos -= 1
def deploy(self, classes, assetFolder=None):
"""
Deploys all asset files to the destination asset folder. This merges
assets from different projects into one destination folder.
"""
# Sometimes it's called with explicit None - we want to fill the default
# in that case as well.
if assetFolder is None:
assetFolder = "$prefix/asset"
assets = self.__assets
projects = self.__session.getProjects()
copyAssetFolder = self.__session.expandFileName(assetFolder)
filterExpr = self.__compileFilterExpr(classes)
Console.info("Deploying assets...")
counter = 0
length = len(assets)
for fileId in assets:
if not filterExpr.match(fileId):
length -= 1
continue
srcFile = assets[fileId].getPath()
dstFile = os.path.join(copyAssetFolder, fileId.replace("/", os.sep))
if jasy.core.File.syncfile(srcFile, dstFile):
counter += 1
Console.info("Updated %s/%s files" % (counter, length))
def removeDir(self, dirname):
"""Removes the given directory"""
dirname = self.__session.expandFileName(dirname)
if os.path.exists(dirname):
Console.info("Deleting folder %s" % dirname)
shutil.rmtree(dirname)
def __structurize(self, data):
"""
This method structurizes the incoming data into a cascaded structure representing the
file system location (aka file IDs) as a tree. It further extracts the extensions and
merges files with the same name (but different extensions) into the same entry. This is
especially useful for alternative formats like audio files, videos and fonts. It only
respects the data of the first entry! So it is not a good idea to have different files
with different content stored with the same name e.g. content.css and content.png.
"""
root = {}
# Easier to debug and understand when sorted
for fileId in sorted(data):
current = root
splits = fileId.split("/")
# Extract the last item aka the filename itself
basename = splits.pop()
# Find the current node to store info on
for split in splits:
if not split in current:
current[split] = {}
elif type(current[split]) != dict:
raise UserError("Invalid asset structure. Folder names must not be identical to any filename without extension: \"%s\" in %s" % (split, fileId))
current = current[split]
# Create entry
Console.debug("Adding %s..." % fileId)
current[basename] = data[fileId]
return root
def attach(self, path):
# Call AbstractItem's attach method first
super().attach(path)
Console.debug("Loading translation file: %s", path)
Console.indent()
# Flat data strucuture where the keys are unique
table = {}
path = self.getPath()
# Decide infrastructure/parser to use based on file name
po = polib.pofile(path)
Console.debug("Translated messages: %s=%s%%", self.language, po.percent_translated())
for entry in po.translated_entries():
entryId = generateMessageId(entry.msgid, entry.msgid_plural, entry.msgctxt)
if entryId not in table:
if entry.msgstr != "":
table[entryId] = entry.msgstr
elif entry.msgstr_plural:
# This field contains all different plural cases (type=dict)
table[entryId] = entry.msgstr_plural
Console.debug("Translation of %s entries ready" % len(table))
Console.outdent()
self.table = table
return self
def __init__(self, id, config, mimeTypes=None):
self.id = id
self.config = config
self.mimeTypes = mimeTypes
self.root = getKey(config, "root", ".")
self.enableDebug = getKey(config, "debug", False)
Console.info('Static "%s" => "%s" [debug:%s]', self.id, self.root, self.enableDebug)
def exportToJson(self, items=None):
"""
Exports asset data for usage at the client side.
Utilizes JavaScript class jasy.Asset to inject data into the client at runtime.
"""
# Processing assets
assets = self.__assets
# Destination folder for assets
assetPath = os.path.join(self.__profile.getDestinationPath(), self.__profile.getAssetOutputFolder())
result = {}
filterExpr = self.__compileFilterExpr(items) if items else None
for fileId in assets:
if filterExpr and not filterExpr.match(fileId):
continue
entry = {}
# t = file type
# u = full file url
# h = file hash (based on content)
# d = asset data (image size, etc.)
assetItem = assets[fileId]
self.__copylist.add(assetItem)
if self.__profile.getUseSource():
# Compute relative folder from asset location to even external
# locations (e.g. auto cloned remote projects)
entry["u"] = os.path.relpath(assetItem.getPath(), assetPath)
elif self.__profile.getHashAssets():
# Export checksum (SHA1 encoded as URL-safe Base62)
entry["h"] = assetItem.getChecksum()
# Store file type as analyzed by asset item
entry["t"] = assetItem.getType(short=True)
# Store additional data figured out by asset item e.g.
# image dimensions, video format, play duration, ...
assetData = assetItem.exportData()
if assetData:
entry["d"] = assetData
result[fileId] = entry
# Ignore empty result
if not result:
return None
Console.debug("Exported %s assets", len(result))
return json.dumps({
"assets" : self.__structurize(result)
}, indent=2, sort_keys=True)
def cacheManifest(profile):
PREFIX = "{{prefix}}"
HASH = "{{id}}"
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
appcache = """CACHE MANIFEST
# Jasy AppCache Manifest file
# Version: {version}
CACHE:
{htmlfile}
{scripts}
{assets}
NETWORK:
*"""
htmlcache = '<!DOCTYPE html><html manifest="{manifestfile}"></html>'
destinationPath = profile.getDestinationPath()
fileManager = FileManager(profile)
session = profile.getSession()
parts = profile.getParts()
assetBuilder = AssetBuilder.AssetBuilder(profile)
scriptBuilder = ScriptBuilder.ScriptBuilder(profile)
styleBuilder = StyleBuilder.StyleBuilder(profile)
assetManager = profile.getAssetManager()
htmlfile = "index.html"
for permutation in profile.permutate():
scripts = []
assets = []
if KERNEL_NAME in parts:
scripts.append("js/kernel.js")
for part in parts:
if part != KERNEL_NAME:
scripts.append(profile.expandFileName("js/%s-{{id}}.js" % part))
assets.append(profile.expandFileName("css/%s-{{id}}.css" % part))
# TODO: How to get permutated asset list?
for (srcFile, dstFile) in assetManager.getAssetList():
assets.append(os.path.relpath(dstFile, profile.getDestinationPath()))
appcacheFilename = "appcache-{{id}}.manifest"
fileManager.writeFile(
"{{destination}}/" + appcacheFilename,
appcache.format(version=timestamp, htmlfile=htmlfile, scripts="\n".join(scripts), assets="\n".join(assets))
)
fileManager.writeFile("{{destination}}/index-{{id}}.html", htmlcache.format(manifestfile=profile.expandFileName(appcacheFilename)))
Console.info("Generate manifest file...")
def addTask(task):
"""Registers the given task with its name."""
if task.name in __taskRegistry:
Console.debug("Overriding task: %s" % task.name)
else:
Console.debug("Registering task: %s" % task.name)
__taskRegistry[task.name] = task
def setCurrentPrefix(self, path):
"""Interface for Task class to configure the current prefix to use"""
if path is None:
self.__currentPrefix = None
Console.debug("Resetting prefix to working directory")
else:
self.__currentPrefix = os.path.normpath(os.path.abspath(os.path.expanduser(path)))
Console.debug("Setting prefix to: %s" % self.__currentPrefix)
def pause(self):
"""
Pauses the session. This release cache files etc. and makes
it possible to call other jasy processes on the same projects.
"""
Console.info("Pausing session...")
for project in self.__projects:
project.pause()
def export(self, path):
Console.info("Writing result...")
Console.info("Target directory: %s", path)
Console.indent()
jasy.core.File.write(os.path.join(path, "jasyproject.yaml"), 'name: locale\npackage: ""\n')
count = self.__exportRecurser(self.__data, "locale", path)
Console.info("Created %s classes", count)
Console.outdent()
def about():
"""Print outs the Jasy about page."""
import jasy
jasy.info()
from jasy.env.Task import getCommand
Console.info("Command: %s", getCommand())
Console.info("Version: %s", jasy.__version__)
def __recurser(node, scope, values, profile):
# Replace variable with actual value
if node.type == "variable" and not (node.parent.type == "assign"
and node.parent[0] is node):
name = node.name
if name not in values:
raise ExecuterError(
"Could not resolve variable %s! Missing value!" % name, node)
value = values[name]
if value is None:
raise ExecuterError(
"Could not resolve variable %s! Value is none!" % name, node)
Console.debug("Resolving variable: %s at line %s with %s from %s",
name, node.line, values[name].type, values[name].line)
node.parent.replace(node, copy.deepcopy(values[name]))
# Decide which sub tree of an if-condition is relevant based on current variable situation
elif node.type == "if":
Console.debug("Processing if-condition at %s", node.line)
# Pre-process condition
# We manually process each child in for if-types
__recurser(node.condition, scope, values, profile)
# Cast condition to Python boolean type
resultValue = Operation.castToBool(node.condition)
# Process relevant part of the sub tree
if resultValue is True:
# Fix missing processing of result node
__recurser(node.thenPart, scope, values, profile)
# Finally replace if-node with result node
node.parent.replace(node, node.thenPart)
elif resultValue is False and hasattr(node, "elsePart"):
# Fix missing processing of result node
__recurser(node.elsePart, scope, values, profile)
# Finally replace if-node with result node
node.parent.replace(node, node.elsePart)
else:
# Cleanup original if-node
node.parent.remove(node)
# Nothing to do here as content is already processed
return
# Update scope of new block starts
if hasattr(node, "scope"):
relation = getattr(node, "rel", None)
# Conditional blocks are not exactly blocks in this variable resolution engine
if not relation in ("thenPart", "elsePart"):
scope = node.scope
values = copy.copy(values)
node.values = values
# Reset all local variables to None
# which enforces not to keep values from outer scope
for name in scope.modified:
values[name] = None
# Process children / content
for child in list(node):
# Ignore non-children... through possible interactive structure changes
if child and child.parent is node:
__recurser(child, scope, values, profile)
# Update values of variables
# This happens after processing children to possibly reduce child structure to an easy to assign (aka preprocessed value)
if (node.type == "declaration"
and hasattr(node, "initializer")) or node.type == "assign":
if node.type == "declaration":
name = node.name
init = node.initializer
Console.debug("Found declaration of %s at line %s", name,
node.line)
else:
name = node[0].name
init = node[1]
Console.debug("Found assignment of %s at line %s", name, node.line)
# Modify value instead of replace when assign operator is set
if hasattr(node, "assignOp") and node.assignOp is not None:
if name not in values:
raise ExecuterError(
"Assign operator is not supported as left hand variable is missing: %s"
% name, node)
repl = Operation.compute(node, values[name], init, node.assignOp)
if repl is not None:
values[name] = repl
else:
# Update internal variable mapping
# Console.debug("Update value of %s to %s" % (name, init))
values[name] = init
# Remove declaration node from tree
node.parent.remove(node)
# Support for variables inside property names or selectors
elif node.type in ("property", "selector") and getattr(
node, "dynamic", False):
def replacer(matchObj):
name = matchObj.group(1)
if name not in values:
raise ExecuterError(
"Could not resolve variable %s! Missing value!" % name,
node)
value = values[name]
if value is None:
raise ExecuterError(
"Could not resolve variable %s! Value is none!" % name,
node)
if value.type == "identifier":
return value.value
elif value.type == "string":
return value.value
elif value.type == "number":
return "%s%s" % (value.value, getattr(value, "unit", ""))
else:
raise ExecuterError(
"Could not replace property inline variable with value of type: %s"
% value.type, node)
# Fix all selectors
if node.type == "selector":
selectors = node.name
for pos, selector in enumerate(selectors):
selectors[pos] = RE_INLINE_VARIABLE.sub(replacer, selector)
else:
node.name = RE_INLINE_VARIABLE.sub(replacer, node.name)
# Execute system commands
elif node.type == "command":
repl = Util.executeCommand(node, profile)
if not repl is node:
node.parent.replace(node, repl)
# Support typical operators
elif node.type in Util.ALL_OPERATORS:
repl = Operation.compute(node)
if repl is not None:
node.parent.replace(node, repl)
def addImageSpriteData(self, id, left, top):
Console.debug("Registering sprite location for %s: %s@%sx%s", self.id, id, left, top)
self.__imageSpriteData = [id, left, top]
def checkLinksInItem(item):
# Process types
if "type" in item:
if item["type"] == "Function":
# Check param types
if "params" in item:
for paramName in item["params"]:
paramEntry = item["params"][paramName]
if "type" in paramEntry:
for paramTypeEntry in paramEntry["type"]:
if not paramTypeEntry[
"name"] in knownClasses and not paramTypeEntry[
"name"] in additionalTypes and not (
"builtin" in paramTypeEntry
or "pseudo"
in paramTypeEntry):
item["errornous"] = True
Console.error(
'Invalid param type "%s" in %s' %
(paramTypeEntry["name"],
className))
if not "pseudo" in paramTypeEntry and paramTypeEntry[
"name"] in knownClasses:
paramTypeEntry["linkable"] = True
# Check return types
if "returns" in item:
for returnTypeEntry in item["returns"]:
if not returnTypeEntry[
"name"] in knownClasses and not returnTypeEntry[
"name"] in additionalTypes and not (
"builtin" in returnTypeEntry
or "pseudo" in returnTypeEntry):
item["errornous"] = True
Console.error(
'Invalid return type "%s" in %s' %
(returnTypeEntry["name"], className))
if not "pseudo" in returnTypeEntry and returnTypeEntry[
"name"] in knownClasses:
returnTypeEntry["linkable"] = True
elif not item["type"] in knownClasses and not item[
"type"] in builtinTypes and not item[
"type"] in pseudoTypes and not item[
"type"] in additionalTypes:
item["errornous"] = True
Console.error('Invalid type "%s" in %s' %
(item["type"], className))
# Process doc
if "doc" in item:
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
Console.error(
"%s in %s:%s~%s" %
(linkCheck, sectionName, className, name))
else:
Console.error("%s in %s" %
(linkCheck, className))
linkExtract.sub(processInternalLink, item["doc"])
def __recurser(node, table):
counter = 0
# Process children
for child in list(node):
if child is not None:
counter += __recurser(child, table)
# Process all method calls
if node.type == "call":
funcName = None
funcNameNode = None
# Uses global translation method (not typical)
if node[0].type == "identifier":
funcNameNode = node[0]
# Uses namespaced translation method.
# Typically core.locale.Translation.tr() or Translation.tr()
elif node[0].type == "dot" and node[0][1].type == "identifier":
funcNameNode = node[0][1]
# Gettext methods only at the moment
funcName = funcNameNode and funcNameNode.value
if funcName in translationFunctions:
Console.debug("Found translation method %s in %s", funcName,
node.line)
Console.indent()
params = node[1]
# Remove marktr() calls
if funcName == "marktr":
node.parent.remove(node)
# Verify param types
elif params[0].type is not "string":
# maybe something marktr() relevant being used, in this case we need to keep the call and inline the data
pass
# Error handling
elif (funcName == "trn"
or funcName == "trc") and params[1].type != "string":
Console.warn(
"Expecting translation string to be type string: %s at line %s"
% (params[1].type, params[1].line))
# Signature tr(msg, arg1, ...)
elif funcName == "tr":
key = params[0].value
if key in table:
params[0].value = table[key]
counter += 1
if len(params) == 1:
node.parent.replace(node, params[0])
else:
replacement = __splitTemplate(params[0].value, params[1:])
if replacement:
node.parent.replace(node, replacement)
# Signature trc(context, msg, arg1, ...)
elif funcName == "trc":
key = "%s[C:%s]" % (params[1].value, params[0].value)
if key in table:
params[1].value = table[key]
counter += 1
if len(params) == 2:
node.parent.replace(node, params[1])
else:
replacement = __splitTemplate(params[1].value, params[2:])
if replacement:
node.parent.replace(node, replacement)
# Signature trn(msgSingular, msgPlural, int, arg1, ...)
elif funcName == "trn":
key = "%s[N:%s]" % (params[0].value, params[1].value)
if not key in table:
Console.outdent()
return counter
counter += 1
# Use optimized trnc() method instead of trn()
funcNameNode.value = "trnc"
# Remove first two string parameters
params.remove(params[0])
params.remove(params[0])
# Inject new object into params
container = Node.Node(None, "object_init")
params.insert(0, container)
# Create new construction with all properties generated from the translation table
for plural in table[key]:
pluralEntry = Node.Node(None, "property_init")
pluralEntryIdentifier = Node.Node(None, "identifier")
pluralEntryIdentifier.value = plural
pluralEntryValue = Node.Node(None, "string")
pluralEntryValue.value = table[key][plural]
pluralEntry.append(pluralEntryIdentifier)
pluralEntry.append(pluralEntryValue)
container.append(pluralEntry)
# Replace strings with plus operations to omit complex client side string operation
if len(params) > 2:
for pluralEntry in container:
replacement = __splitTemplate(pluralEntry[1].value,
params[2:])
if replacement:
pluralEntry.replace(pluralEntry[1], replacement)
# When all variables have been patched in all string with placeholder
# we are able to remove the whole list of placeholder values afterwards
while len(params) > 2:
params.pop()
Console.outdent()
return counter
def skip(self):
"""Eats comments and whitespace."""
input = self.source
startLine = self.line
# Whether this is the first called as happen on start parsing a file (eat leading comments/white space)
startOfFile = self.cursor is 0
indent = ""
while True:
if len(input) > self.cursor:
ch = input[self.cursor]
else:
return
self.cursor += 1
if len(input) > self.cursor:
next = input[self.cursor]
else:
next = None
if ch == "\n" and not self.scanNewlines:
self.line += 1
indent = ""
elif ch == "/" and next == "*":
self.cursor += 1
text = "/*"
inline = startLine == self.line and startLine > 1
commentStartLine = self.line
if startLine == self.line and not startOfFile:
mode = "inline"
elif (self.line - 1) > startLine:
# distance before this comment means it is a comment block for a whole section (multiple lines of code)
mode = "section"
else:
# comment for maybe multiple following lines of code, but not that important (no visual white space divider)
mode = "block"
while True:
try:
ch = input[self.cursor]
self.cursor += 1
except IndexError:
raise ParseError("Unterminated comment", self.fileId,
self.line)
if ch == "*":
next = input[self.cursor]
if next == "/":
text += "*/"
self.cursor += 1
break
elif ch == "\n":
self.line += 1
text += ch
# Filter escaping on slash-star combinations in comment text
text = text.replace("*\/", "*/")
try:
self.comments.append(
Comment.Comment(text, mode, commentStartLine, indent,
self.fileId))
except Comment.CommentException as commentError:
Console.error("Ignoring comment in %s: %s", self.fileId,
commentError)
elif ch == "/" and next == "/":
self.cursor += 1
text = "//"
if startLine == self.line and not startOfFile:
mode = "inline"
elif (self.line - 1) > startLine:
# distance before this comment means it is a comment block for a whole section (multiple lines of code)
mode = "section"
else:
# comment for maybe multiple following lines of code, but not that important (no visual white space divider)
mode = "block"
while True:
try:
ch = input[self.cursor]
self.cursor += 1
except IndexError:
# end of file etc.
break
if ch == "\n":
self.line += 1
break
text += ch
try:
self.comments.append(
Comment.Comment(text, mode, self.line - 1, "",
self.fileId))
except Comment.CommentException:
Console.error("Ignoring comment in %s: %s", self.fileId,
commentError)
# check for whitespace, also for special cases like 0xA0
elif ch in "\xA0 \t":
indent += ch
else:
self.cursor -= 1
return
def __outdent(self, text, indent, startLineNo):
"""
Outdent multi line comment text and filtering empty lines
"""
lines = []
# First, split up the comments lines and remove the leading indentation
for lineNo, line in enumerate((indent + text).split("\n")):
if line.startswith(indent):
lines.append(line[len(indent):].rstrip())
elif line.strip() == "":
lines.append("")
else:
# Only warn for doc comments, otherwise it might just be code commented out
# which is sometimes formatted pretty crazy when commented out
if self.variant == "doc":
Console.warn(
"Could not outdent doc comment at line %s in %s",
startLineNo + lineNo, self.fileId)
return text
# Find first line with real content, then grab the one after it to get the
# characters which need
outdentString = ""
for lineNo, line in enumerate(lines):
if line != "" and line.strip() != "":
matchedDocIndent = docIndentReg.match(line)
if not matchedDocIndent:
# As soon as we find a non doc indent like line we stop
break
elif matchedDocIndent.group(2) != "":
# otherwise we look for content behind the indent to get the
# correct real indent (with spaces)
outdentString = matchedDocIndent.group(1)
break
lineNo += 1
# Process outdenting to all lines (remove the outdentString from the start of the lines)
if outdentString != "":
lineNo = 0
outdentStringLen = len(outdentString)
for lineNo, line in enumerate(lines):
if len(line) <= outdentStringLen:
lines[lineNo] = ""
else:
if not line.startswith(outdentString):
# Only warn for doc comments, otherwise it might just be code commented out
# which is sometimes formatted pretty crazy when commented out
if self.variant == "doc":
Console.warn(
"Invalid indentation in doc comment at line %s in %s",
startLineNo + lineNo, self.fileId)
else:
lines[lineNo] = line[outdentStringLen:]
# Merge final lines and remove leading and trailing new lines
return "\n".join(lines).strip("\n")
def checkSingleInstallation(keys, versions, packageName, minVersion,
installPath, updatePath):
if packageName.lower() in keys:
if StrictVersion(minVersion) > StrictVersion("0.0"):
if StrictVersion(versions[
packageName.lower()]) < StrictVersion(minVersion):
Console.info(
Console.colorize(
Console.colorize(
'Jasy requirement error: "%s"' % packageName,
"red"), "bold"))
Console.indent()
Console.info(
Console.colorize(
Console.colorize(
'Version is NOT OK (needed: %s installed: %s)'
% (minVersion, versions[packageName.lower()]),
"red"), "bold"))
Console.info(
'Update to the newest version of %s using %s' %
(packageName, updatePath))
Console.outdent()
return False
else:
Console.info(
Console.colorize(
Console.colorize(
'Jasy requirement error: "%s"' % packageName, "red"),
"bold"))
Console.indent()
Console.info(
Console.colorize(
Console.colorize('Did NOT find installation', "red"),
"bold"))
Console.info('Install the newest version of %s using %s' %
(packageName, installPath))
Console.outdent()
return False
return True
#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
#
import jasy.core.Console as Console
from distutils.version import StrictVersion
try:
import pip
except ImportError:
Console.error("pip is required to run JASY!")
sys.exit(1)
needs = [{
"packageName": "Pygments",
"minVersion": "1.5",
"installPath": "'pip install Pygments'",
"updatePath": "'pip install --upgrade pygments'"
}, {
"packageName": "polib",
"minVersion": "1.0.1",
"installPath": "'pip install polib'",
"updatePath": "'pip install --upgrade polib'"
}, {
"packageName": "requests",
"minVersion": "0.14",
"installPath": "'pip install requests'",
"updatePath": "'pip install --upgrade requests'"
}, {
"packageName": "CherryPy",
def clean(self):
"""Clears the cache of the project."""
Console.info("Clearing cache of %s..." % self.__name)
self.__cache.clear()
def scan(self):
if self.scanned:
return
updatemsg = "[updated]" if self.__modified else "[cached]"
if self.version:
Console.info("Scanning %s @ %s %s...",
Console.colorize(self.getName(), "bold"),
Console.colorize(self.version, "magenta"),
Console.colorize(updatemsg, "grey"))
else:
Console.info("Scanning %s %s...",
Console.colorize(self.getName(), "bold"),
Console.colorize(updatemsg, "grey"))
Console.indent()
# Support for pre-initialize projects...
setup = self.__setup
if setup and self.__modified:
Console.info("Running setup...")
Console.indent()
for cmd in setup:
Console.info("Executing %s...", cmd)
result = None
try:
result = None
result = Util.executeCommand(
cmd,
"Failed to execute setup command %s" % cmd,
path=self.__path)
except Exception as ex:
if result:
Console.error(result)
raise UserError("Could not scan project %s: %s" %
(self.__name, ex))
Console.outdent()
# Processing custom content section. Only supports classes and assets.
if self.__config.has("content"):
self.kind = "manual"
self.__addContent(self.__config.get("content"))
else:
# Read scan path from config
if not self.__config.has("scan"):
if self.__hasDir("source"):
self.kind = "application"
scan = self.__resolveScanConfig(structures[self.kind])
elif self.__hasDir("src"):
self.kind = "resource"
scan = self.__resolveScanConfig(structures[self.kind])
else:
self.kind = "flat"
scan = self.__resolveScanConfig(structures[self.kind])
else:
scan = self.__resolveScanConfig(self.__config.get("scan"))
for config in scan:
if isinstance(config["paths"], str):
self.__addDir(config["paths"], config["regex"],
config["type"], config["package"])
else:
for path in config["paths"]:
self.__addDir(path, config["regex"], config["type"],
config["package"])
# Generate summary
summary = []
for section in self.items.keys():
content = self.items[section]
name, constructor = self.__resolveConstructor(section)
if content:
summary.append(
Console.colorize("%s %s" % (len(content), name),
"magenta"))
# Print out
if summary:
Console.info("Content: %s" % (", ".join(summary)))
self.scanned = True
Console.outdent()
def __resolve(project):
name = project.getName()
# List of required projects
Console.info("Getting requirements of %s...",
Console.colorize(name, "bold"))
Console.indent()
requires = project.getRequires(checkoutDirectory, updateRepositories)
Console.outdent()
if not requires:
return
Console.debug("Processing %s requirements...", len(requires))
Console.indent()
# Adding all project in reverse order.
# Adding all local ones first before going down to their requirements
childProjects = []
for requiredProject in reversed(requires):
requiredName = requiredProject.getName()
if requiredName not in names:
Console.debug("Adding: %s %s (via %s)", requiredName,
requiredProject.version, project.getName())
names[requiredName] = True
result.append(requiredProject)
childProjects.append(requiredProject)
elif not requiredProject in result:
Console.debug("Blocking: %s %s (via %s)", requiredName,
requiredProject.version, project.getName())
requiredProject.pause()
# Process all requirements of added projects
for requiredProject in reversed(childProjects):
if requiredProject.hasRequires():
__resolve(requiredProject)
Console.outdent()
def __cleanup(node):
"""Reprocesses JavaScript to remove dead paths."""
optimized = False
# Process from inside to outside
for child in reversed(node):
# None children are allowed sometimes e.g. during array_init like [1,2,,,7,8]
if child is not None:
if __cleanup(child):
optimized = True
# Optimize if cases
if node.type == "if":
check = __checkCondition(node.condition)
if check is not None:
optimized = True
Console.debug("Optimizing if/else at line %s", node.line)
if check is True:
node.parent.replace(node, node.thenPart)
elif check is False:
if hasattr(node, "elsePart"):
node.parent.replace(node, node.elsePart)
else:
node.parent.remove(node)
# Optimize hook statement
if node.type == "hook":
check = __checkCondition(node[0])
if check is not None:
Console.debug("Optimizing hook at line %s", node.line)
optimized = True
if check is True:
node.parent.replace(node, node[1])
elif check is False:
node.parent.replace(node, node[2])
# Optimize switch statement
if node.type == "switch" and node.discriminant.type in ("string",
"number"):
discriminant = node.discriminant.value
fallback = None
matcher = None
allowed = ["default", "case"]
for child in node:
# Require that every case block ends with a break (no fall-throughs)
if child.type == "case":
block = child[len(child) - 1]
if len(block) == 0 or block[len(block) - 1].type != "break":
Console.warn(
"Could not optimize switch statement (at line %s) because of fallthrough break statement.",
node.line)
return False
if child.type == "default":
fallback = child.statements
elif child.type == "case" and child.label.value == discriminant:
matcher = child.statements
# Remove break statement
matcher.pop()
if matcher or fallback:
if not matcher:
matcher = fallback
node.parent.replace(node, matcher)
Console.debug("Optimizing switch at line %s", node.line)
optimized = True
return optimized
def connectInterface(className, interfaceName, classApi, interfaceApi):
Console.debug("- Connecting %s with %s", className, interfaceName)
#
# Properties
#
interfaceProperties = getattr(interfaceApi, "properties", None)
if interfaceProperties:
classProperties = getattr(classApi, "properties", {})
for name in interfaceProperties:
if name not in classProperties:
Console.warn(
"Class %s is missing implementation for property %s of interface %s!",
className, name, interfaceName)
else:
# Add reference to interface
if "interface" not in classProperties[name]:
classProperties[name]["defined"] = []
classProperties[name]["defined"].append({
"name":
interfaceName,
"link":
"property:%s~%s" % (interfaceName, name)
})
# Copy over documentation
if "doc" not in classProperties[
name] and "doc" in interfaceProperties[name]:
classProperties[name]["doc"] = interfaceProperties[name][
"doc"]
if "summary" not in classProperties[
name] and "summary" in interfaceProperties[name]:
classProperties[name]["summary"] = interfaceProperties[
name]["summary"]
if "errornous" in classProperties[
name] and "errornous" not in interfaceProperties[name]:
del classProperties[name]["errornous"]
# Update tags with data from interface
if "tags" in interfaceProperties[name]:
if "tags" not in classProperties[name]:
classProperties[name]["tags"] = {}
safeUpdate(classProperties[name]["tags"],
interfaceProperties[name]["tags"])
#
# Events
#
interfaceEvents = getattr(interfaceApi, "events", None)
if interfaceEvents:
classEvents = getattr(classApi, "events", {})
for name in interfaceEvents:
if name not in classEvents:
Console.warn(
"Class %s is missing implementation for event %s of interface %s!",
className, name, interfaceName)
else:
# Add reference to interface
if "interface" not in classEvents[name]:
classEvents[name]["defined"] = []
classEvents[name]["defined"].append({
"name":
interfaceName,
"link":
"event:%s~%s" % (interfaceName, name)
})
# Copy user event type and documentation from interface
if "doc" not in classEvents[name] and "doc" in interfaceEvents[
name]:
classEvents[name]["doc"] = interfaceEvents[name]["doc"]
if "summary" not in classEvents[
name] and "summary" in interfaceEvents[name]:
classEvents[name]["summary"] = interfaceEvents[name][
"summary"]
if "type" not in classEvents[
name] and "type" in interfaceEvents[name]:
classEvents[name]["type"] = interfaceEvents[name]["type"]
if "errornous" in classEvents[
name] and not "errornous" in interfaceEvents[name]:
del classEvents[name]["errornous"]
# Update tags with data from interface
if "tags" in interfaceEvents[name]:
if not "tags" in classEntry:
classEvents[name]["tags"] = {}
safeUpdate(classEvents[name]["tags"],
interfaceEvents[name]["tags"])
#
# Members
#
interfaceMembers = getattr(interfaceApi, "members", None)
if interfaceMembers:
classMembers = getattr(classApi, "members", {})
for name in interfaceMembers:
if name not in classMembers:
Console.warn(
"Class %s is missing implementation for member %s of interface %s!",
className, name, interfaceName)
else:
interfaceEntry = interfaceMembers[name]
classEntry = classMembers[name]
# Add reference to interface
if not "interface" in classEntry:
classEntry["defined"] = []
classEntry["defined"].append({
"name":
interfaceName,
"link":
"member:%s~%s" % (interfaceName, name)
})
# Copy over doc from interface
if not "doc" in classEntry and "doc" in interfaceEntry:
classEntry["doc"] = interfaceEntry["doc"]
if not "summary" in classEntry and "summary" in interfaceEntry:
classEntry["summary"] = interfaceEntry["summary"]
if "errornous" in classEntry and not "errornous" in interfaceEntry:
del classEntry["errornous"]
# Priorize return value from interface (it's part of the interface feature set to enforce this)
if "returns" in interfaceEntry:
classEntry["returns"] = interfaceEntry["returns"]
# Update tags with data from interface
if "tags" in interfaceEntry:
if not "tags" in classEntry:
classEntry["tags"] = {}
safeUpdate(classEntry["tags"], interfaceEntry["tags"])
# Copy over params from interface
if "params" in interfaceEntry:
# Fix completely missing parameters
if not "params" in classEntry:
classEntry["params"] = {}
for paramName in interfaceEntry["params"]:
# Prefer data from interface
if not paramName in classEntry["params"]:
classEntry["params"][paramName] = {}
classEntry["params"][paramName].update(
interfaceEntry["params"][paramName])
# Clear errournous documentation flags
if "errornous" in classEntry["params"][
paramName] and not "errornous" in interfaceEntry[
"params"][paramName]:
del classEntry["params"][paramName]["errornous"]
def __recurser(node, unused):
"""
The cleanup part which always processes one scope and cleans up params and
variable definitions which are unused
"""
retval = False
# Process children
if node.type != "function":
for child in node:
# None children are allowed sometimes e.g. during array_init like [1,2,,,7,8]
if child != None:
if __recurser(child, unused):
retval = True
if node.type == "script" and hasattr(node, "parent"):
# Remove unused parameters
params = getattr(node.parent, "params", None)
if params:
# Start from back, as we can only remove params as long
# as there is not a required one after the current one
for identifier in reversed(params):
if identifier.value in unused:
Console.debug("Removing unused parameter '%s' in line %s",
identifier.value, identifier.line)
params.remove(identifier)
retval = True
else:
break
# Remove function names which are unused
if node.parent.functionForm == "expressed_form":
funcName = getattr(node.parent, "name", None)
if funcName != None and funcName in unused:
Console.debug("Removing unused function name at line %s" %
node.line)
del node.parent.name
retval = True
elif node.type == "function":
# Remove full unused functions (when not in top-level scope)
if node.functionForm == "declared_form" and getattr(
node, "parent", None) and node.parent.type != "call":
funcName = getattr(node, "name", None)
if funcName != None and funcName in unused:
Console.debug(
"Removing unused function declaration %s at line %s" %
(funcName, node.line))
node.parent.remove(node)
retval = True
elif node.type == "var":
for decl in reversed(node):
if getattr(decl, "name", None) in unused:
if hasattr(decl, "initializer"):
init = decl.initializer
if init.type in ("null", "this", "true", "false",
"identifier", "number", "string",
"regexp"):
Console.debug(
"Removing unused primitive variable %s at line %s"
% (decl.name, decl.line))
node.remove(decl)
retval = True
elif init.type == "function" and (
not hasattr(init, "name") or init.name in unused):
Console.debug(
"Removing unused function variable %s at line %s" %
(decl.name, decl.line))
node.remove(decl)
retval = True
# If we have only one child, we replace the whole var statement with just the init block
elif len(node) == 1:
semicolon = Node.Node(init.tokenizer, "semicolon")
semicolon.append(init, "expression")
# Protect non-expressions with parens
if init.type in ("array_init", "object_init"):
init.parenthesized = True
elif init.type == "call" and init[0].type == "function":
init[0].parenthesized = True
node.parent.replace(node, semicolon)
retval = True
# If we are the last declaration, move it out of node and append after var block
elif node[-1] == decl or node[0] == decl:
isFirst = node[0] == decl
node.remove(decl)
nodePos = node.parent.index(node)
semicolon = Node.Node(init.tokenizer, "semicolon")
semicolon.append(init, "expression")
# Protect non-expressions with parens
if init.type in ("array_init", "object_init"):
init.parenthesized = True
elif init.type == "call" and init[0].type == "function":
init[0].parenthesized = True
if isFirst:
node.parent.insert(nodePos, semicolon)
else:
node.parent.insert(nodePos + 1, semicolon)
retval = True
else:
Console.debug(
"Could not automatically remove unused variable %s at line %s without possible side-effects"
% (decl.name, decl.line))
else:
node.remove(decl)
retval = True
if len(node) == 0:
Console.debug("Removing empty 'var' block at line %s" % node.line)
node.parent.remove(node)
return retval
def __getLoadDepsRecurser(self, classObj, stack):
"""
This is the main routine which tries to control over a system
of unsorted classes. It directly tries to fullfil every dependency
a class have, but has some kind of exception based loop protection
to prevent circular dependencies from breaking the build.
It respects break information given by file specific meta data, but
also adds custom hints where it found recursions. This lead to a valid
sort, but might lead to problems between exeactly the two affected classes.
Without doing an exact execution it's not possible to whether found out
which of two each-other referencing classes needs to be loaded first.
This is basically only interesting in cases where one class needs another
during the definition phase which is not the case that often.
"""
if classObj in stack:
stack.append(classObj)
msg = " >> ".join(
[x.getId() for x in stack[stack.index(classObj):]])
raise CircularDependency("Circular Dependency: %s" % msg)
stack.append(classObj)
classDeps = classObj.getDependencies(self.__permutation,
classes=self.__names,
warnings=False)
classMeta = classObj.getMetaData(self.__permutation)
result = set()
circular = set()
# Respect manually defined breaks
# Breaks are dependencies which are down-priorized to break
# circular dependencies between classes.
for breakName in classMeta.breaks:
if breakName in self.__names:
circular.add(self.__names[breakName])
# Now process the deps of the given class
loadDeps = self.__loadDeps
for depObj in classDeps:
if depObj is classObj:
continue
depName = depObj.getId()
if depName in classMeta.breaks:
Console.debug("Manual Break: %s => %s" % (classObj, depObj))
pass
elif depObj in loadDeps:
result.update(loadDeps[depObj])
result.add(depObj)
else:
current = self.__getLoadDepsRecurser(depObj, stack[:])
result.update(current)
result.add(depObj)
# Sort dependencies by number of other dependencies
# For performance reasions we access the __loadDeps
# dict directly as this data is already stored
result = sorted(result,
key=lambda depObj: len(self.__loadDeps[depObj]))
loadDeps[classObj] = result
if circular:
self.__circularDeps[classObj] = circular
return result
def doCompleteDoctor():
"""Checks for uninstalled or too old versions of requirements and gives a complete output"""
Console.header("Doctor")
dists = [dist for dist in pip.get_installed_distributions()]
keys = [dist.key for dist in pip.get_installed_distributions()]
versions = {}
for dist in dists:
versions[dist.key] = dist.version
def checkSingleInstallation(keys, versions, packageName, minVersion,
installPath, updatePath):
Console.info('%s:' % packageName)
Console.indent()
if packageName.lower() in keys:
Console.info(Console.colorize('Found installation', "green"))
if StrictVersion(minVersion) > StrictVersion("0.0"):
if StrictVersion(versions[
packageName.lower()]) >= StrictVersion(minVersion):
Console.info(
Console.colorize(
'Version is OK (needed: %s installed: %s)' %
(minVersion, versions[packageName.lower()]),
"green"))
else:
Console.info(
Console.colorize(
Console.colorize(
'- Version is NOT OK (needed: %s installed: %s)'
% (minVersion, versions[packageName.lower()]),
"red"), "bold"))
Console.info(
'Update to the newest version of %s using %s' %
(packageName, updatePath))
else:
Console.info(
Console.colorize(
Console.colorize('Did NOT find installation', "red"),
"bold"))
Console.info('Install the newest version of %s using %s' %
(packageName, installPath))
Console.outdent()
# Required packages
Console.info(Console.colorize("Required Packages:", "bold"))
Console.indent()
for entry in needs:
checkSingleInstallation(keys, versions, entry["packageName"],
entry["minVersion"], entry["installPath"],
entry["updatePath"])
Console.outdent()
# Optional packages
Console.info("")
Console.info(Console.colorize("Optional Packages:", "bold"))
Console.indent()
for entry in optionals:
checkSingleInstallation(keys, versions, entry["packageName"],
entry["minVersion"], entry["installPath"],
entry["updatePath"])
Console.outdent()
def processSprites(self):
"""Processes jasysprite files to merge sprite data into asset registry."""
assets = self.__assets
configs = [
fileId for fileId in assets
if assets[fileId].isImageSpriteConfig()
]
if configs:
Console.info(
"Processing %s...",
Console.colorize("%s sprites", "magenta") % len(configs))
sprites = []
Console.indent()
for fileId in configs:
Console.debug("Processing %s...", fileId)
asset = assets[fileId]
spriteBase = os.path.dirname(fileId)
try:
spriteConfig = asset.getParsedObject()
except ValueError as err:
raise UserError("Could not parse jasysprite at %s: %s" %
(fileId, err))
Console.indent()
for spriteImage in spriteConfig:
spriteImageId = "%s/%s" % (spriteBase, spriteImage)
singleRelPaths = spriteConfig[spriteImage]
Console.debug("Image %s combines %s images", spriteImageId,
len(singleRelPaths))
for singleRelPath in singleRelPaths:
singleId = "%s/%s" % (spriteBase, singleRelPath)
singleData = singleRelPaths[singleRelPath]
singleItem = assets[singleId]
# Verify that sprite sheet is up-to-date
fileChecksum = singleItem.getChecksum()
storedChecksum = singleData["checksum"]
Console.debug("Checksum Compare: %s <=> %s", fileChecksum,
storedChecksum)
if storedChecksum != fileChecksum:
raise UserError(
"Sprite Sheet is not up-to-date. Checksum of %s differs."
% singleId)
if spriteImageId not in sprites:
spriteImageIndex = len(sprites)
sprites.append(spriteImageId)
else:
spriteImageIndex = sprites.index(spriteImageId)
# Add relevant data to find image on sprite sheet
singleItem.addImageSpriteData(spriteImageIndex,
singleData["left"],
singleData["top"])
Console.outdent()
# The config file does not make any sense on the client side
Console.debug("Deleting sprite config from assets: %s", fileId)
del assets[fileId]
Console.outdent()
self.__sprites = sprites
def __init__(self, locale):
Console.info("Parsing CLDR files for %s..." % locale)
Console.indent()
splits = locale.split("_")
# Store for internal usage
self.__locale = locale
self.__language = splits[0]
self.__territory = splits[1] if len(splits) > 1 else None
# This will hold all data extracted data
self.__data = {}
# Add info section
self.__data["info"] = {
"LOCALE": self.__locale,
"LANGUAGE": self.__language,
"TERRITORY": self.__territory
}
# Add keys (fallback to C-default locale)
path = "%s.xml" % os.path.join(CLDR_DIR, "keys", self.__language)
try:
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
except IOError:
path = "%s.xml" % os.path.join(CLDR_DIR, "keys", "C")
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
self.__data["key"] = {
"Short": {
key.get("type"): key.text
for key in tree.findall("./keys/short/key")
},
"Full": {
key.get("type"): key.text
for key in tree.findall("./keys/full/key")
}
}
# Add main CLDR data: Fallback chain for locales
main = os.path.join(CLDR_DIR, "main")
files = []
while True:
files.append("%s.xml" % os.path.join(main, locale))
if "_" in locale:
locale = locale[:locale.rindex("_")]
else:
break
# Extend data with root data
files.append(os.path.join(main, "root.xml"))
# Finally import all these files in order
for path in reversed(files):
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
self.__addDisplayNames(tree)
self.__addDelimiters(tree)
self.__addCalendars(tree)
self.__addNumbers(tree)
# Add supplemental CLDR data
self.__addSupplementals(self.__territory)
Console.outdent()
def __process(self,
apiData,
classFilter=None,
internals=False,
privates=False,
printErrors=True,
highlightCode=True):
knownClasses = set(list(apiData))
#
# Attaching Links to Source Code (Lines)
# Building Documentation Summaries
#
Console.info("Adding Source Links...")
for className in apiData:
classApi = apiData[className]
constructData = getattr(classApi, "construct", None)
if constructData is not None:
if "line" in constructData:
constructData["sourceLink"] = "source:%s~%s" % (
className, constructData["line"])
for section in ("properties", "events", "statics", "members"):
sectionData = getattr(classApi, section, None)
if sectionData is not None:
for name in sectionData:
if "line" in sectionData[name]:
sectionData[name][
"sourceLink"] = "source:%s~%s" % (
className, sectionData[name]["line"])
#
# Including Mixins / IncludedBy
#
Console.info("Resolving Mixins...")
Console.indent()
# Just used temporary to keep track of which classes are merged
mergedClasses = set()
def getApi(className):
classApi = apiData[className]
if className in mergedClasses:
return classApi
classIncludes = getattr(classApi, "includes", None)
if classIncludes:
for mixinName in classIncludes:
if mixinName not in apiData:
Console.error("Invalid mixin %s in class %s",
className, mixinName)
continue
mixinApi = apiData[mixinName]
if not hasattr(mixinApi, "includedBy"):
mixinApi.includedBy = set()
mixinApi.includedBy.add(className)
mergeMixin(className, mixinName, classApi,
getApi(mixinName))
mergedClasses.add(className)
return classApi
for className in apiData:
apiData[className] = getApi(className)
Console.outdent()
#
# Checking links
#
Console.info("Checking Links...")
additionalTypes = ("Call", "Identifier", "Map", "Integer", "Node",
"Element", "Class", "Exception", "Uri")
def checkInternalLink(link, className):
match = internalLinkParse.match(link)
if not match:
return 'Invalid link "#%s"' % link
if match.group(3) is not None:
className = match.group(3)
if not className in knownClasses and not className in apiData:
return 'Invalid class in link "#%s"' % link
# Accept all section/item values for named classes,
# as it might be pretty complicated to verify this here.
if className not in apiData:
return True
classApi = apiData[className]
sectionName = match.group(2)
itemName = match.group(5)
if itemName is None:
return True
if sectionName is not None:
if sectionName not in linkMap:
return 'Invalid section in link "#%s"' % link
section = getattr(classApi, linkMap[sectionName], None)
if section is None:
return 'Invalid section in link "#%s"' % link
else:
if itemName in section:
return True
return 'Invalid item in link "#%s"' % link
for sectionName in ("statics", "members", "properties", "events"):
section = getattr(classApi, sectionName, None)
if section and itemName in section:
return True
return 'Invalid item link "#%s"' % link
def checkLinksInItem(item):
# Process types
if "type" in item:
if item["type"] == "Function":
# Check param types
if "params" in item:
for paramName in item["params"]:
paramEntry = item["params"][paramName]
if "type" in paramEntry:
for paramTypeEntry in paramEntry["type"]:
if not paramTypeEntry[
"name"] in knownClasses and not paramTypeEntry[
"name"] in additionalTypes and not (
"builtin" in paramTypeEntry
or "pseudo"
in paramTypeEntry):
item["errornous"] = True
Console.error(
'Invalid param type "%s" in %s' %
(paramTypeEntry["name"],
className))
if not "pseudo" in paramTypeEntry and paramTypeEntry[
"name"] in knownClasses:
paramTypeEntry["linkable"] = True
# Check return types
if "returns" in item:
for returnTypeEntry in item["returns"]:
if not returnTypeEntry[
"name"] in knownClasses and not returnTypeEntry[
"name"] in additionalTypes and not (
"builtin" in returnTypeEntry
or "pseudo" in returnTypeEntry):
item["errornous"] = True
Console.error(
'Invalid return type "%s" in %s' %
(returnTypeEntry["name"], className))
if not "pseudo" in returnTypeEntry and returnTypeEntry[
"name"] in knownClasses:
returnTypeEntry["linkable"] = True
elif not item["type"] in knownClasses and not item[
"type"] in builtinTypes and not item[
"type"] in pseudoTypes and not item[
"type"] in additionalTypes:
item["errornous"] = True
Console.error('Invalid type "%s" in %s' %
(item["type"], className))
# Process doc
if "doc" in item:
def processInternalLink(match):
linkUrl = match.group(2)
if linkUrl.startswith("#"):
linkCheck = checkInternalLink(linkUrl[1:], className)
if linkCheck is not True:
item["errornous"] = True
if sectionName:
Console.error(
"%s in %s:%s~%s" %
(linkCheck, sectionName, className, name))
else:
Console.error("%s in %s" %
(linkCheck, className))
linkExtract.sub(processInternalLink, item["doc"])
Console.indent()
# Process APIs
for className in apiData:
classApi = apiData[className]
sectionName = None
constructData = getattr(classApi, "construct", None)
if constructData is not None:
checkLinksInItem(constructData)
for sectionName in ("properties", "events", "statics", "members"):
section = getattr(classApi, sectionName, None)
if section is not None:
for name in section:
checkLinksInItem(section[name])
Console.outdent()
#
# Filter Internals/Privates
#
Console.info("Filtering Items...")
def isVisible(entry):
if "visibility" in entry:
visibility = entry["visibility"]
if visibility == "private" and not privates:
return False
if visibility == "internal" and not internals:
return False
return True
def filterInternalsPrivates(classApi, field):
data = getattr(classApi, field, None)
if data:
for name in list(data):
if not isVisible(data[name]):
del data[name]
for className in apiData:
filterInternalsPrivates(apiData[className], "statics")
filterInternalsPrivates(apiData[className], "members")
#
# Connection Interfaces / ImplementedBy
#
Console.info("Connecting Interfaces...")
Console.indent()
for className in apiData:
classApi = getApi(className)
if not hasattr(classApi, "main"):
continue
classType = classApi.main["type"]
if classType == "core.Class":
classImplements = getattr(classApi, "implements", None)
if classImplements:
for interfaceName in classImplements:
interfaceApi = apiData[interfaceName]
implementedBy = getattr(interfaceApi, "implementedBy",
None)
if not implementedBy:
implementedBy = interfaceApi.implementedBy = []
implementedBy.append(className)
connectInterface(className, interfaceName, classApi,
interfaceApi)
Console.outdent()
#
# Merging Named Classes
#
Console.info("Merging Named Classes...")
Console.indent()
for className in list(apiData):
classApi = apiData[className]
destName = classApi.main["name"]
if destName is not None and destName != className:
Console.debug("Extending class %s with %s", destName,
className)
if destName in apiData:
destApi = apiData[destName]
destApi.main["from"].append(className)
else:
destApi = apiData[destName] = Data.ApiData(
destName, highlight=highlightCode)
destApi.main = {
"type": "Extend",
"name": destName,
"from": [className]
}
# If there is a "main" tag found in the class use its API description
if "tags" in classApi.main and classApi.main[
"tags"] is not None and "main" in classApi.main["tags"]:
if "doc" in classApi.main:
destApi.main["doc"] = classApi.main["doc"]
classApi.main["extension"] = True
# Read existing data
construct = getattr(classApi, "construct", None)
statics = getattr(classApi, "statics", None)
members = getattr(classApi, "members", None)
if construct is not None:
if hasattr(destApi, "construct"):
Console.warn(
"Overriding constructor in extension %s by %s",
destName, className)
destApi.construct = copy.copy(construct)
if statics is not None:
if not hasattr(destApi, "statics"):
destApi.statics = {}
for staticName in statics:
destApi.statics[staticName] = copy.copy(
statics[staticName])
destApi.statics[staticName]["from"] = className
destApi.statics[staticName][
"fromLink"] = "static:%s~%s" % (className,
staticName)
if members is not None:
if not hasattr(destApi, "members"):
destApi.members = {}
for memberName in members:
destApi.members[memberName] = copy.copy(
members[memberName])
destApi.members[memberName]["from"] = className
destApi.members[memberName][
"fromLink"] = "member:%s~%s" % (className,
memberName)
Console.outdent()
#
# Connecting Uses / UsedBy
#
Console.info("Collecting Use Patterns...")
# This matches all uses with the known classes and only keeps them if matched
allClasses = set(list(apiData))
for className in apiData:
uses = apiData[className].uses
# Rebuild use list
cleanUses = set()
for use in uses:
if use != className and use in allClasses:
cleanUses.add(use)
useEntry = apiData[use]
if not hasattr(useEntry, "usedBy"):
useEntry.usedBy = set()
useEntry.usedBy.add(className)
apiData[className].uses = cleanUses
#
# Building Search Index
#
Console.info("Building Search Index...")
search = {}
def addSearch(classApi, field):
data = getattr(classApi, field, None)
if data:
for name in data:
if name not in search:
search[name] = set()
search[name].add(className)
for className in apiData:
classApi = apiData[className]
addSearch(classApi, "statics")
addSearch(classApi, "members")
addSearch(classApi, "properties")
addSearch(classApi, "events")
#
# Post Process (dict to sorted list)
#
Console.info("Post Processing Data...")
for className in sorted(apiData):
classApi = apiData[className]
convertTags(classApi.main)
construct = getattr(classApi, "construct", None)
if construct:
convertFunction(construct)
convertTags(construct)
for section in ("statics", "members", "properties", "events"):
items = getattr(classApi, section, None)
if items:
sortedList = []
for itemName in sorted(items):
item = items[itemName]
item["name"] = itemName
if "type" in item and item["type"] == "Function":
convertFunction(item)
convertTags(item)
sortedList.append(item)
setattr(classApi, section, sortedList)
#
# Collecting Package Docs
#
Console.info("Collecting Package Docs...")
Console.indent()
# Inject existing package docs into api data
for project in self.__session.getProjects():
docs = project.getDocs()
for packageName in docs:
if self.__isIncluded(packageName, classFilter):
Console.debug("Creating package documentation %s",
packageName)
apiData[packageName] = docs[packageName].getApi()
# Fill missing package docs
for className in sorted(apiData):
splits = className.split(".")
packageName = splits[0]
for split in splits[1:]:
if packageName not in apiData:
Console.warn("Missing package documentation %s",
packageName)
apiData[packageName] = Data.ApiData(
packageName, highlight=highlightCode)
apiData[packageName].main = {
"type": "Package",
"name": packageName
}
packageName = "%s.%s" % (packageName, split)
# Now register all classes in their parent namespace/package
for className in sorted(apiData):
splits = className.split(".")
packageName = ".".join(splits[:-1])
if packageName:
package = apiData[packageName]
# debug("- Registering class %s in parent %s", className, packageName)
entry = {
"name": splits[-1],
"link": className,
}
classMain = apiData[className].main
if "doc" in classMain and classMain["doc"]:
summary = Text.extractSummary(classMain["doc"])
if summary:
entry["summary"] = summary
if "type" in classMain and classMain["type"]:
entry["type"] = classMain["type"]
if not hasattr(package, "content"):
package.content = [entry]
else:
package.content.append(entry)
Console.outdent()
#
# Writing API Index
#
Console.debug("Building Index...")
index = {}
for className in sorted(apiData):
classApi = apiData[className]
mainInfo = classApi.main
# Create structure for className
current = index
for split in className.split("."):
if split not in current:
current[split] = {}
current = current[split]
# Store current type
current["$type"] = mainInfo["type"]
# Keep information if
if hasattr(classApi, "content"):
current["$content"] = True
#
# Return
#
return apiData, index, search
#
# Jasy - Web Tooling Framework
# Copyright 2010-2012 Zynga Inc.
# Copyright 2013-2014 Sebastian Werner
#
from jasy import UserError
import jasy.core.Console as Console
# Make PIL (native module) optional
try:
from PIL import Image
from PIL import ImageDraw
Console.debug("Using Pillow (PIL)")
except ImportError as err1:
Image = None
ImageDraw = None
Console.debug("No support for Pillow (PIL)!")
class SpriteSheet():
def __init__(self, packer, blocks):
self.packer = packer
self.width = packer.root.w
self.height = packer.root.h
self.blocks = blocks
self.area = self.width * self.height
def __processSprites(self):
"""Processes jasysprite.json files to merge sprite data into asset registry"""
assets = self.__assets
configs = [
fileId for fileId in assets
if assets[fileId].isImageSpriteConfig()
]
if configs:
Console.info("Processing %s image sprite configs...", len(configs))
sprites = []
Console.indent()
for fileId in configs:
Console.debug("Processing %s...", fileId)
asset = assets[fileId]
spriteBase = os.path.dirname(fileId)
try:
spriteConfig = asset.getParsedObject()
except ValueError as err:
raise UserError("Could not parse jasysprite.json at %s: %s" %
(fileId, err))
Console.indent()
for spriteImage in spriteConfig:
spriteImageId = "%s/%s" % (spriteBase, spriteImage)
singleRelPaths = spriteConfig[spriteImage]
Console.debug("Image %s combines %s images", spriteImageId,
len(singleRelPaths))
for singleRelPath in singleRelPaths:
singleId = "%s/%s" % (spriteBase, singleRelPath)
singleData = singleRelPaths[singleRelPath]
if singleId in assets:
singleAsset = assets[singleId]
else:
Console.info("Creating new asset: %s", singleId)
singleAsset = jasy.item.Asset.AssetItem(None)
assets[singleId] = singleAsset
if not spriteImageId in sprites:
spriteImageIndex = len(sprites)
sprites.append(spriteImageId)
else:
spriteImageIndex = sprites.index(spriteImageId)
singleAsset.addImageSpriteData(spriteImageIndex,
singleData["left"],
singleData["top"])
if "width" in singleData and "height" in singleData:
singleAsset.addImageDimensionData(
singleData["width"], singleData["height"])
# Verify that sprite sheet is up-to-date
if "checksum" in singleData:
fileChecksum = singleAsset.getChecksum()
storedChecksum = singleData["checksum"]
Console.debug("Checksum Compare: %s <=> %s",
fileChecksum[0:6], storedChecksum[0:6])
if storedChecksum != fileChecksum:
raise UserError(
"Sprite Sheet is not up-to-date. Checksum of %s differs.",
singleId)
Console.outdent()
Console.debug("Deleting sprite config from assets: %s", fileId)
del assets[fileId]
Console.outdent()
self.__sprites = sprites
def write(self,
distFolder,
classFilter=None,
callback="apiload",
showInternals=False,
showPrivates=False,
printErrors=True,
highlightCode=True):
"""
Writes API data generated from JavaScript into distFolder.
:param distFolder: Where to store the API data
:param classFilter: Tuple of classes or method to use for filtering
:param callback: Name of callback to use for loading or None if pure JSON should be used
:param showInternals: Include internal methods inside API data
:param showPrivates: Include private methods inside API data
:param printErrors: Whether errors should be printed to the console
:param highlightCode: Whether to enable code highlighting using Pygments
:type distFolder: str
:type classFilter: tuple or function
:type callback: function
:type showInternals: bool
:type showPrivates: bool
:type printErrors: bool
:type highlightCode: bool
"""
#
# Collecting
#
Console.info("Collecting API Data...")
Console.indent()
apiData = {}
highlightedCode = {}
for project in self.__session.getProjects():
classes = project.getScripts()
Console.info("Loading API of project %s: %s...",
Console.colorize(project.getName(), "bold"),
Console.colorize("%s classes" % len(classes), "cyan"))
Console.indent()
for className in classes:
if self.__isIncluded(className, classFilter):
data = classes[className].getApi(highlightCode)
if not data.isEmpty:
apiData[className] = data
highlightedCode[className] = classes[
className].getHighlightedCode()
else:
Console.info("Skipping %s, class is empty." %
className)
Console.outdent()
Console.outdent()
#
# Processing
#
Console.info("Processing API Data...")
Console.indent()
data, index, search = self.__process(apiData,
classFilter=classFilter,
internals=showInternals,
privates=showPrivates,
printErrors=printErrors,
highlightCode=highlightCode)
Console.outdent()
#
# Writing
#
Console.info("Storing API data...")
Console.indent()
writeCounter = 0
extension = "js" if callback else "json"
compress = True
class JsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, set):
return list(obj)
return json.JSONEncoder.default(self, obj)
def encode(content, name):
if compress:
jsonContent = json.dumps(content,
sort_keys=True,
cls=JsonEncoder,
separators=(',', ':'))
else:
jsonContent = json.dumps(content,
sort_keys=True,
cls=JsonEncoder,
indent=2)
if callback:
return "%s(%s,'%s');" % (callback, jsonContent, name)
else:
return jsonContent
Console.info("Saving class data (%s files)...", len(data))
Console.indent()
for className in data:
try:
classData = data[className]
if isinstance(classData, dict):
classExport = classData
else:
classExport = classData.export()
File.write(
self.__profile.expandFileName(
os.path.join(distFolder,
"%s.%s" % (className, extension))),
encode(classExport, className))
except TypeError as writeError:
Console.error("Could not write API data of: %s: %s", className,
writeError)
continue
Console.outdent()
if highlightCode:
Console.info("Saving highlighted code (%s files)...",
len(highlightedCode))
Console.indent()
for className in highlightedCode:
try:
File.write(
self.__profile.expandFileName(
os.path.join(distFolder, "%s.html" % className)),
highlightedCode[className])
except TypeError as writeError:
Console.error(
"Could not write highlighted code of: %s: %s",
className, writeError)
continue
Console.outdent()
Console.info("Writing index...")
Console.indent()
File.write(
self.__profile.expandFileName(
os.path.join(distFolder, "meta-index.%s" % extension)),
encode(index, "meta-index"))
File.write(
self.__profile.expandFileName(
os.path.join(distFolder, "meta-search.%s" % extension)),
encode(search, "meta-search"))
Console.outdent()
Console.outdent()
def __addSupplementals(self, territory):
""" Converts data from supplemental folder """
supplemental = os.path.join(CLDR_DIR, "supplemental")
# Plurals
path = os.path.join(supplemental, "plurals.xml")
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
self.__data["Plural"] = {}
for item in tree.findall("plurals/pluralRules"):
attr = item.get("locales")
if attr != None:
if self.__language in attr.split(" "):
for rule in item.findall("pluralRule"):
jsPlural = pluralToJavaScript(rule.text)
self.__data["Plural"][rule.get(
"count").upper()] = jsPlural
# Telephone Codes
path = os.path.join(supplemental, "telephoneCodeData.xml")
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
for item in tree.findall("telephoneCodeData/codesByTerritory"):
territoryId = item.get("territory")
if territoryId == territory:
for rule in item.findall("telephoneCountryCode"):
self.__data["PhoneCode"] = {"CODE": int(rule.get("code"))}
# Respect first only
break
# Postal Codes
path = os.path.join(supplemental, "postalCodeData.xml")
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
for item in tree.findall("postalCodeData/postCodeRegex"):
territoryId = item.get("territoryId")
if territory == territoryId:
self.__data["PostalCode"] = {"CODE": item.text}
break
# Supplemental Data
path = os.path.join(supplemental, "supplementalData.xml")
Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR))
tree = xml.etree.ElementTree.parse(path)
# :: Calendar Preference
ordering = None
for item in tree.findall("calendarPreferenceData/calendarPreference"):
if item.get("territories") == "001" and ordering == None:
ordering = item.get("ordering")
elif territory in item.get("territories").split(" "):
ordering = item.get("ordering")
break
self.__data["CalendarPref"] = {"ORDERING": ordering.split(" ")}
# :: Week Data
self.__data["Week"] = {}
weekData = tree.find("weekData")
for key in ["firstDay", "weekendStart", "weekendEnd"]:
day = None
for item in weekData.findall(key):
if item.get("territories") == "001" and day == None:
day = item.get("day")
elif territory in item.get("territories").split(" "):
day = item.get("day")
break
self.__data["Week"][camelCaseToUpper(key)] = day
# :: Measurement System
self.__data["Measurement"] = {}
measurementData = tree.find("measurementData")
for key in ["measurementSystem", "paperSize"]:
mtype = None
for item in measurementData.findall(key):
if item.get("territories") == "001" and mtype == None:
mtype = item.get("type")
elif territory in item.get("territories").split(" "):
mtype = item.get("type")
break
self.__data["Measurement"][camelCaseToUpper(key)] = mtype
def __processAnimations(self):
"""Processes jasyanimation.json files to merge animation data into asset registry"""
assets = self.__assets
configs = [
fileId for fileId in assets
if assets[fileId].isImageAnimationConfig()
]
if configs:
Console.info("Processing %s image animation configs...",
len(configs))
Console.indent()
for fileId in configs:
Console.debug("Processing %s...", fileId)
asset = assets[fileId]
base = os.path.dirname(fileId)
try:
config = json.loads(asset.getText())
except ValueError as err:
raise UserError(
"Could not parse jasyanimation.json at %s: %s" %
(fileId, err))
for relPath in config:
imageId = "%s/%s" % (base, relPath)
data = config[relPath]
if not imageId in assets:
raise UserError("Unknown asset %s in %s" %
(imageId, fileId))
animationAsset = assets[imageId]
if "rows" in data or "columns" in data:
rows = Util.getKey(data, "rows", 1)
columns = Util.getKey(data, "columns", 1)
frames = Util.getKey(data, "frames")
animationAsset.addImageAnimationData(columns, rows, frames)
if frames is None:
frames = rows * columns
elif "layout" in data:
layout = data["layout"]
animationAsset.addImageAnimationData(None,
None,
layout=layout)
frames = len(layout)
else:
raise UserError("Invalid image frame data for: %s" %
imageId)
Console.debug(" - Animation %s has %s frames", imageId,
frames)
Console.debug(" - Deleting animation config from assets: %s",
fileId)
del assets[fileId]
Console.outdent()
def checkSingleInstallation(keys, versions, packageName, minVersion,
installPath, updatePath):
Console.info('%s:' % packageName)
Console.indent()
if packageName.lower() in keys:
if LooseVersion(minVersion) > LooseVersion("0.0"):
if LooseVersion(versions[packageName.lower()]) >= LooseVersion(
minVersion):
Console.info(
Console.colorize(
'Version is OK (required: %s installed: %s)' %
(minVersion, versions[packageName.lower()]),
"green"))
else:
Console.info(
Console.colorize(
Console.colorize(
'Version installed is too old (required: %s installed: %s)'
% (minVersion, versions[packageName.lower()]),
"red"), "bold"))
Console.info(
'Update to the newest version of %s using %s' %
(packageName, updatePath))
else:
Console.info(Console.colorize('Found installation', "green"))
else:
Console.info(
Console.colorize(
Console.colorize('Did NOT find installation', "red"),
"bold"))
Console.info('Install the newest version of %s using %s' %
(packageName, installPath))
Console.outdent()
def addImageDimensionData(self, width, height):
Console.debug("Adding dimension data for %s: %sx%s", self.id, width, height)
self.__imageDimensionData = [width, height]
def optimize(node, contextId=""):
Console.debug("Crypting private fields...")
Console.indent()
coll = __search(node)
repl = {}
for name in coll:
repl[name] = "__%s" % __encode("%s.%s" % (contextId, name[2:]))
Console.debug("Replacing private field %s with %s (context: %s)", name,
repl[name], contextId)
Console.debug("Found %s private fields" % len(repl))
modified, reduction = __replace(node, repl)
Console.debug("Reduced size by %s bytes" % reduction)
Console.outdent()
return modified
def __init__(self, filename):
try:
self.fp = open(filename, "rb")
except IOError as err:
Console.error("Could not open file: %s" % filename)
raise err
def requestUrl(url,
content_type="text/plain",
headers=None,
method="GET",
port=None,
body="",
user=None,
password=None):
"""Generic HTTP request wrapper with support for basic authentification and automatic parsing of response content"""
Console.info("Opening %s request to %s..." % (method, url))
parsed = urllib.parse.urlparse(url)
if parsed.scheme == "http":
request = http.client.HTTPConnection(parsed.netloc)
elif parsed.scheme == "https":
request = http.client.HTTPSConnection(parsed.netloc)
else:
raise Exception("Unsupported url: %s" % url)
if parsed.query:
request.putrequest(method, parsed.path + "?" + parsed.query)
else:
request.putrequest(method, parsed.path)
request.putheader("Content-Type", content_type)
request.putheader("Content-Length", str(len(body)))
if user is not None and password is not None:
auth = "Basic %s" % base64.b64encode(
("%s:%s" % (user, password)).encode("utf-8")).decode("utf-8")
request.putheader("Authorization", auth)
request.endheaders()
if body:
Console.info("Sending data (%s bytes)..." % len(body))
else:
Console.info("Sending request...")
Console.indent()
request.send(body)
response = request.getresponse()
res_code = int(response.getcode())
res_headers = dict(response.getheaders())
res_content = response.read()
res_success = False
if res_code >= 200 and res_code <= 300:
Console.debug("HTTP Success!")
res_success = True
else:
Console.error("HTTP Failure Code: %s!", res_code)
if "Content-Type" in res_headers:
res_type = res_headers["Content-Type"]
if ";" in res_type:
res_type = res_type.split(";")[0]
if res_type in ("application/json", "text/html", "text/plain"):
res_content = res_content.decode("utf-8")
if res_type == "application/json":
res_content = json.loads(res_content)
if "error" in res_content:
Console.error("Error %s: %s", res_content["error"],
res_content["reason"])
elif "reason" in res_content:
Console.info("Success: %s" % res_content["reason"])
Console.outdent()
return res_success, res_headers, res_content
def __extend(node):
"""
Finds extend requests for mixins aka.
- mixins calls without params
- simple variables in a block
For all found extend requests it detects the flattened selector and appends
the selector section of the extendable mixin accordingly. After that it
removes the original mixin request.
"""
modified = 0
for child in reversed(list(node)):
# Ignore all mixin declarations. Can't operate inside them.
# For these things to work we have to wait for the include mechanics to resolve them first
# (which actually just remove these mixin declarations though)
if child is not None:
modified += __extend(child)
if isExtendCall(node):
name = node.name
Console.debug("Extend request to mixin %s at: %s", name, node.line)
Console.indent()
mixin = __findMixin(node.parent, name)
if not mixin:
raise Exception(
"Could not find mixin %s as required by extend request at line %s"
% (node.name, node.line))
Console.debug("Found matching mixin declaration at line: %s",
mixin.line)
selector, media, supports = Util.combineSelector(node.parent,
stop=mixin.parent)
# There is no possibility to handle this in a series of CSS selectors. This is why
# we have to use an include like approach instead of extend to correctly deal
# with the situation. This should work well, but is not as efficient regarding
# output file size.
if media or supports:
Console.warn(
"Extending inside a @media/@support structure behaves like including (larger result size): %s %s + %s",
media, supports, ", ".join(selector))
replacements = __resolveMixin(mixin, None)
Console.debug(
"Replacing call %s at line %s with mixin from line %s" %
(name, node.line, replacements.line))
# Reverse inject all children of that block
# at the same position as the original call
parent = node.parent
pos = parent.index(node)
parent.insertAll(pos, replacements)
elif selector:
Console.debug("Extending selector of mixin by: %s",
", ".join(selector))
if hasattr(mixin, "selector"):
# We iterate from in inverse mode, so add new selectors to the front
mixin.selector[0:0] = selector
else:
mixin.selector = selector
virtualBlock = Node.Node(type="block")
__extendContent(mixin.rules, node, virtualBlock, mixin)
if len(virtualBlock) > 0:
callSelector, callMedia, callSupports = Util.combineSelector(
node)
if callSelector:
virtualSelector = Node.Node(type="selector")
virtualSelector.name = callSelector
if callMedia:
virtualMedia = Node.Node(type="media")
virtualMedia.name = callMedia
if callSupports:
virtualSupports = Node.Node(type="supports")
virtualSupports.name = callSupports
if callSelector:
virtualSelector.append(virtualBlock, "rules")
elif callMedia:
virtualMedia.append(virtualBlock, "rules")
elif callSupports:
virtualSupports.append(virtualBlock, "rules")
if callSupports:
virtualTop = virtualSupports
elif callMedia:
virtualTop = virtualMedia
elif callSelector:
virtualTop = virtualSelector
pos = mixin.parent.index(mixin)
mixin.parent.insert(pos + 1, virtualTop)
node.parent.remove(node)
Console.outdent()
modified += 1
return modified
