def removeFile(self, filename): """Removes the given file""" filename = self.__session.expandFileName(filename) if os.path.exists(filename): Console.info("Deleting file %s" % filename) os.remove(filename)
def build(self): """Build static website.""" Console.info("Intializing Konstrukteur...") Console.indent() # Path configuration # TODO: Use Jasy configuration instead self.__templatePath = os.path.join("source", "template") self.__contentPath = os.path.join("source", "content") self.__sourcePath = os.path.join("source") self.__pagePath = os.path.join(self.__contentPath, "page") self.__postPath = os.path.join(self.__contentPath, "post") if not os.path.exists(self.__templatePath): raise RuntimeError("Path to templates not found : %s" % self.__templatePath) if not os.path.exists(self.__contentPath): raise RuntimeError("Path to content not found : %s" % self.__contentPath) # A theme could be any project registered in the current session if self.__theme: themeProject = session.getProjectByName(self.__theme) if not themeProject: raise RuntimeError("Theme '%s' not found" % self.__theme) self.__initializeTemplates() self.__generateOutput()
def getCompressed(self, profile): """Returns the compressed CSS code of this item.""" field = "style:compressed[%s]-%s" % (self.id, profile.getId()) mtime = self.getModificationTime(profile) compressed = self.project.getCache().read(field, mtime) if compressed is None: Console.info("Compressing tree %s...", Console.colorize(self.id, "bold")) # Start with the merged tree (includes resolved) tree = self.getMergedTree(profile) # Reduce tree Engine.reduceTree(tree, profile) # Compress tree compressed = Engine.compressTree(tree, profile.getCompressionLevel(), profile.getFormattingLevel()) # Store in cache self.project.getCache().store(field, compressed, mtime) return compressed
def write(self, filename, debug=False): if Image is None: raise UserError("Missing Python PIL which is required to create sprite sheets!") img = Image.new('RGBA', (self.width, self.height)) draw = ImageDraw.Draw(img) #draw.rectangle((0, 0, self.width, self.height), fill=(255, 255, 0, 255)) # Load images and pack them in for block in self.blocks: res = Image.open(block.image.src) x, y = block.fit.x, block.fit.y if block.rotated: Console.debug('%s is rotated' % block.image.src) res = res.rotate(90) img.paste(res, (x, y)) del res if debug: x, y, w, h = block.fit.x, block.fit.y, block.w, block.h draw.rectangle((x, y , x + w , y + h), outline=(0, 0, 255, 255) if block.rotated else (255, 0, 0, 255)) if debug: for i, block in enumerate(self.packer.getUnused()): x, y, w, h = block.x, block.y, block.w, block.h draw.rectangle((x, y , x + w , y + h), fill=(255, 255, 0, 255)) img.save(filename)
def generate(self, path='', debug=False): """Generate sheets/variants.""" Console.info('Generating sprite sheet variants...') Console.indent() sheets, count = self.packBest() # Write PNG files data = {} for pos, sheet in enumerate(sheets): Console.info('Writing image (%dx%dpx) with %d images' % (sheet.width, sheet.height, len(sheet))) name = 'jasysprite_%d.png' % pos # Export sheet.write(os.path.join(self.base, path, name), debug) data[name] = sheet.export() Console.outdent() # Generate config file Console.info('Exporting data...') script = os.path.join(self.base, path, 'jasysprite.%s' % self.dataFormat) writeConfig(data, script)
def __call__(self, **kwargs): merged = {} merged.update(self.curry) merged.update(kwargs) # # SUPPORT SOME DEFAULT FEATURES CONTROLLED BY TASK PARAMETERS # # Allow overriding of prefix via task or cmdline parameter. # By default use name of the task (no prefix for cleanup tasks) if "prefix" in merged: session.setCurrentPrefix(merged["prefix"]) del merged["prefix"] elif "clean" in self.name: session.setCurrentPrefix(None) else: session.setCurrentPrefix(self.name) # # EXECUTE ATTACHED FUNCTION # Console.header(self.__name__) # Execute internal function return self.func(**merged)
def __extendContent(node, call, targetBlock, stopCombineAt): """ Builds up a list of selector/mediaqueries to insert after the extend to produce the @content sections on the intended selectors. """ for child in reversed(node): if child: __extendContent(child, call, targetBlock, stopCombineAt) if node.type == "content" and hasattr(call, "rules"): # Extends support @content as well. In this case we produce a new selector # which matches the position of the content section and append it after # the original extended mixin on return Console.debug("Inserting content section into new virtual selector") selector, media = Util.combineSelector(node, stop=stopCombineAt) selectorNode = Node.Node(type="selector") selectorNode.name = selector selectorNode.append(copy.deepcopy(call.rules), "rules") # Support media queries, too if media: mediaNode = Node.Node(type="media") mediaNode.name = media mediaNode.append(selectorNode) targetBlock.append(mediaNode) else: targetBlock.append(selectorNode)
def __clean(node): """ Removes all empty rules. Starting from inside out for a deep cleanup. This is a required step for the next one where we combine media queries and selectors and need to have an easy reference point to the previous node. """ # Process children first for child in reversed(node): if child is not None: __clean(child) if hasattr(node, "rules") and len(node.rules) == 0: Console.debug("Cleaning up empty selector/mixin/@media/@supports at line %s" % node.line) node.parent.remove(node) elif node.type == "content": Console.debug("Cleaning up left over @content at line %s" % node.line) node.parent.remove(node) elif node.type == "meta": Console.debug("Cleaning up left over @meta at line %s" % node.line) node.parent.remove(node) elif node.type == "block" and node.parent.type in ("sheet", "block"): Console.debug("Inlining content of unnecessary block node at line %s" % node.line) node.parent.insertAllReplace(node, node) elif node.type == "root" and len(node) == 0: Console.debug("Cleaning up left over @root at line %s" % node.line) node.parent.remove(node)
def patch(node, permutation): """ Replaces all occourences with incoming values """ modified = False if node.type == "identifier" or node.type == "dot": if node.type == "dot": name = assembleDot(node) else: name = node.value replacement = __translateToValue(permutation.get(name)) if replacement: Console.debug("Found permutation query (%s => %s) in line %s", name, replacement, node.line) replacementNode = Parser.parseExpression(replacement) node.parent.replace(node, replacementNode) modified = True # Process children for child in reversed(node): if child != None: if patch(child, permutation): modified = True return modified
def loadCommands(self, objectName, fileName, encoding="utf-8"): """Loads new commands into the session wide command registry.""" counter = 0 commands = self.__commands # Method for being used as a decorator to share methods to the outside def share(func): name = "%s.%s" % (objectName, func.__name__) if name in commands: raise Exception("Command %s already exists!" % name) commands[name] = func nonlocal counter counter += 1 return func # Execute given file. Using clean new global environment # but add additional decorator for allowing to define shared methods # and the session object (self). code = open(fileName, "r", encoding=encoding).read() exec(compile(code, os.path.abspath(fileName), "exec"), {"share" : share, "session" : self}) # Export destination name as global Console.info("Imported %s.", Console.colorize("%s commands" % counter, "magenta")) return counter
def printTasks(indent=16): """Prints out a list of all avaible tasks and their descriptions.""" for name in sorted(__taskRegistry): obj = __taskRegistry[name] formattedName = name if obj.__doc__: space = (indent - len(name)) * " " print(" %s: %s%s" % (formattedName, space, Console.colorize(obj.__doc__, "magenta"))) else: print(" %s" % formattedName) if obj.availableArgs or obj.hasFlexArgs: text = "" if obj.availableArgs: text += Util.hyphenate("--%s <var>" % " <var> --".join(obj.availableArgs)) if obj.hasFlexArgs: if text: text += " ..." else: text += "--<name> <var>" print(" %s" % (Console.colorize(text, "grey")))
def store(self, key, value, timestamp=None, transient=False, inMemory=True): """ Stores the given value. Default timestamp goes to the current time. Can be modified to the time of an other files modification date etc. Transient enables in-memory cache for the given value """ if self.__hashkeys: key = hashlib.sha1(key.encode("ascii")).hexdigest() if inMemory: self.__transient[key] = value if transient: return if not timestamp: timestamp = time.time() try: self.__shelve[key + "-timestamp"] = timestamp self.__shelve[key] = value except pickle.PicklingError as err: Console.error("Failed to store enty: %s" % key)
def resume(self): """Resumes the session after it has been paused.""" Console.info("Resuming session...") for project in self.__projects: project.resume()
def loadLibrary(self, objectName, fileName, encoding="utf-8", doc=None): """ Creates a new object inside the user API (jasyscript.py) with the given name containing all @share'd functions and fields loaded from the given file. """ if objectName in self.__scriptEnvironment: raise UserError("Could not import library %s as the object name %s is already used." % (fileName, objectName)) # Create internal class object for storing shared methods class Shared(object): pass exportedModule = Shared() exportedModule.__doc__ = doc or "Imported from %s" % os.path.relpath(fileName, os.getcwd()) counter = 0 # Method for being used as a decorator to share methods to the outside def share(func): nonlocal counter setattr(exportedModule, func.__name__, func) counter += 1 return func # Execute given file. Using clean new global environment # but add additional decorator for allowing to define shared methods # and the session object (self). code = open(fileName, "r", encoding=encoding).read() exec(compile(code, os.path.abspath(fileName), "exec"), {"share" : share, "session" : self}) # Export destination name as global Console.debug("Importing %s shared methods under %s...", counter, objectName) self.__scriptEnvironment[objectName] = exportedModule return counter
def __generateTranslationBundle(self): """ Returns a translation object for the given language containing all relevant translation files for the current project set. """ language = self.getCurrentPermutation().get("locale") if language is None: return None if language in self.__translationBundles: return self.__translationBundles[language] Console.info("Creating translation bundle: %s", language) Console.indent() # Initialize new Translation object with no project assigned # This object is used to merge all seperate translation instances later on. combined = jasy.item.Translation.TranslationItem(None, id=language) relevantLanguages = self.__expandLanguage(language) # Loop structure is build to prefer finer language matching over project priority for currentLanguage in reversed(relevantLanguages): for project in self.__projects: for translation in project.getTranslations().values(): if translation.getLanguage() == currentLanguage: Console.debug("Adding %s entries from %s @ %s...", len(translation.getTable()), currentLanguage, project.getName()) combined += translation Console.debug("Combined number of translations: %s", len(combined.getTable())) Console.outdent() self.__translationBundles[language] = combined return combined
def __build(self): """ Build static website """ self.__parseContent() self.__outputContent() Console.info("Done processing website")
def __init__(self, session, assetManager=None, compressionLevel=1, formattingLevel=0): Console.info("Initializing OutputManager...") Console.indent() Console.info("Formatting Level: %s", formattingLevel) Console.info("Compression Level: %s", compressionLevel) self.__session = session self.__assetManager = assetManager self.__fileManager = FileManager(session) self.__scriptOptimization = Optimization() self.__compressGeneratedCode = False self.__kernelClasses = [] if compressionLevel > 0: self.__scriptOptimization.enable("variables") self.__scriptOptimization.enable("declarations") self.__compressGeneratedCode = True if compressionLevel > 1: self.__scriptOptimization.enable("blocks") self.__scriptOptimization.enable("privates") self.__scriptFormatting = Formatting() if formattingLevel > 0: self.__scriptFormatting.enable("semicolon") self.__scriptFormatting.enable("comma") Console.outdent()
def __combineSiblings(node): """Backwards processing and insertion into previous sibling if both are declarations""" length = len(node) pos = length - 1 while pos > 0: child = node[pos] prevChild = node[pos - 1] # Special FOR loop optimization, emulate faked VAR if child.type == "for" and prevChild.type == "var": setup = getattr(child, "setup", None) if setup and setup.type == "var": Console.debug("Removing for-loop setup section at line %s" % setup.line) child.remove(setup) child = setup # Combine declarations of VAR statements if child.type == "var" and prevChild.type == "var": # debug("Combining var statement at line %s" % child.line) # Fix loop through casting node to list() for variable in list(child): prevChild.append(variable) if child in node: node.remove(child) pos -= 1
def deploy(self, classes, assetFolder=None): """ Deploys all asset files to the destination asset folder. This merges assets from different projects into one destination folder. """ # Sometimes it's called with explicit None - we want to fill the default # in that case as well. if assetFolder is None: assetFolder = "$prefix/asset" assets = self.__assets projects = self.__session.getProjects() copyAssetFolder = self.__session.expandFileName(assetFolder) filterExpr = self.__compileFilterExpr(classes) Console.info("Deploying assets...") counter = 0 length = len(assets) for fileId in assets: if not filterExpr.match(fileId): length -= 1 continue srcFile = assets[fileId].getPath() dstFile = os.path.join(copyAssetFolder, fileId.replace("/", os.sep)) if jasy.core.File.syncfile(srcFile, dstFile): counter += 1 Console.info("Updated %s/%s files" % (counter, length))
def removeDir(self, dirname): """Removes the given directory""" dirname = self.__session.expandFileName(dirname) if os.path.exists(dirname): Console.info("Deleting folder %s" % dirname) shutil.rmtree(dirname)
def __structurize(self, data): """ This method structurizes the incoming data into a cascaded structure representing the file system location (aka file IDs) as a tree. It further extracts the extensions and merges files with the same name (but different extensions) into the same entry. This is especially useful for alternative formats like audio files, videos and fonts. It only respects the data of the first entry! So it is not a good idea to have different files with different content stored with the same name e.g. content.css and content.png. """ root = {} # Easier to debug and understand when sorted for fileId in sorted(data): current = root splits = fileId.split("/") # Extract the last item aka the filename itself basename = splits.pop() # Find the current node to store info on for split in splits: if not split in current: current[split] = {} elif type(current[split]) != dict: raise UserError("Invalid asset structure. Folder names must not be identical to any filename without extension: \"%s\" in %s" % (split, fileId)) current = current[split] # Create entry Console.debug("Adding %s..." % fileId) current[basename] = data[fileId] return root
def attach(self, path): # Call AbstractItem's attach method first super().attach(path) Console.debug("Loading translation file: %s", path) Console.indent() # Flat data strucuture where the keys are unique table = {} path = self.getPath() # Decide infrastructure/parser to use based on file name po = polib.pofile(path) Console.debug("Translated messages: %s=%s%%", self.language, po.percent_translated()) for entry in po.translated_entries(): entryId = generateMessageId(entry.msgid, entry.msgid_plural, entry.msgctxt) if entryId not in table: if entry.msgstr != "": table[entryId] = entry.msgstr elif entry.msgstr_plural: # This field contains all different plural cases (type=dict) table[entryId] = entry.msgstr_plural Console.debug("Translation of %s entries ready" % len(table)) Console.outdent() self.table = table return self
def __init__(self, id, config, mimeTypes=None): self.id = id self.config = config self.mimeTypes = mimeTypes self.root = getKey(config, "root", ".") self.enableDebug = getKey(config, "debug", False) Console.info('Static "%s" => "%s" [debug:%s]', self.id, self.root, self.enableDebug)
def exportToJson(self, items=None): """ Exports asset data for usage at the client side. Utilizes JavaScript class jasy.Asset to inject data into the client at runtime. """ # Processing assets assets = self.__assets # Destination folder for assets assetPath = os.path.join(self.__profile.getDestinationPath(), self.__profile.getAssetOutputFolder()) result = {} filterExpr = self.__compileFilterExpr(items) if items else None for fileId in assets: if filterExpr and not filterExpr.match(fileId): continue entry = {} # t = file type # u = full file url # h = file hash (based on content) # d = asset data (image size, etc.) assetItem = assets[fileId] self.__copylist.add(assetItem) if self.__profile.getUseSource(): # Compute relative folder from asset location to even external # locations (e.g. auto cloned remote projects) entry["u"] = os.path.relpath(assetItem.getPath(), assetPath) elif self.__profile.getHashAssets(): # Export checksum (SHA1 encoded as URL-safe Base62) entry["h"] = assetItem.getChecksum() # Store file type as analyzed by asset item entry["t"] = assetItem.getType(short=True) # Store additional data figured out by asset item e.g. # image dimensions, video format, play duration, ... assetData = assetItem.exportData() if assetData: entry["d"] = assetData result[fileId] = entry # Ignore empty result if not result: return None Console.debug("Exported %s assets", len(result)) return json.dumps({ "assets" : self.__structurize(result) }, indent=2, sort_keys=True)
def cacheManifest(profile): PREFIX = "{{prefix}}" HASH = "{{id}}" timestamp = time.strftime("%Y-%m-%d %H:%M:%S") appcache = """CACHE MANIFEST # Jasy AppCache Manifest file # Version: {version} CACHE: {htmlfile} {scripts} {assets} NETWORK: *""" htmlcache = '<!DOCTYPE html><html manifest="{manifestfile}"></html>' destinationPath = profile.getDestinationPath() fileManager = FileManager(profile) session = profile.getSession() parts = profile.getParts() assetBuilder = AssetBuilder.AssetBuilder(profile) scriptBuilder = ScriptBuilder.ScriptBuilder(profile) styleBuilder = StyleBuilder.StyleBuilder(profile) assetManager = profile.getAssetManager() htmlfile = "index.html" for permutation in profile.permutate(): scripts = [] assets = [] if KERNEL_NAME in parts: scripts.append("js/kernel.js") for part in parts: if part != KERNEL_NAME: scripts.append(profile.expandFileName("js/%s-{{id}}.js" % part)) assets.append(profile.expandFileName("css/%s-{{id}}.css" % part)) # TODO: How to get permutated asset list? for (srcFile, dstFile) in assetManager.getAssetList(): assets.append(os.path.relpath(dstFile, profile.getDestinationPath())) appcacheFilename = "appcache-{{id}}.manifest" fileManager.writeFile( "{{destination}}/" + appcacheFilename, appcache.format(version=timestamp, htmlfile=htmlfile, scripts="\n".join(scripts), assets="\n".join(assets)) ) fileManager.writeFile("{{destination}}/index-{{id}}.html", htmlcache.format(manifestfile=profile.expandFileName(appcacheFilename))) Console.info("Generate manifest file...")
def addTask(task): """Registers the given task with its name.""" if task.name in __taskRegistry: Console.debug("Overriding task: %s" % task.name) else: Console.debug("Registering task: %s" % task.name) __taskRegistry[task.name] = task
def setCurrentPrefix(self, path): """Interface for Task class to configure the current prefix to use""" if path is None: self.__currentPrefix = None Console.debug("Resetting prefix to working directory") else: self.__currentPrefix = os.path.normpath(os.path.abspath(os.path.expanduser(path))) Console.debug("Setting prefix to: %s" % self.__currentPrefix)
def pause(self): """ Pauses the session. This release cache files etc. and makes it possible to call other jasy processes on the same projects. """ Console.info("Pausing session...") for project in self.__projects: project.pause()
def export(self, path): Console.info("Writing result...") Console.info("Target directory: %s", path) Console.indent() jasy.core.File.write(os.path.join(path, "jasyproject.yaml"), 'name: locale\npackage: ""\n') count = self.__exportRecurser(self.__data, "locale", path) Console.info("Created %s classes", count) Console.outdent()
def about(): """Print outs the Jasy about page.""" import jasy jasy.info() from jasy.env.Task import getCommand Console.info("Command: %s", getCommand()) Console.info("Version: %s", jasy.__version__)
def __recurser(node, scope, values, profile): # Replace variable with actual value if node.type == "variable" and not (node.parent.type == "assign" and node.parent[0] is node): name = node.name if name not in values: raise ExecuterError( "Could not resolve variable %s! Missing value!" % name, node) value = values[name] if value is None: raise ExecuterError( "Could not resolve variable %s! Value is none!" % name, node) Console.debug("Resolving variable: %s at line %s with %s from %s", name, node.line, values[name].type, values[name].line) node.parent.replace(node, copy.deepcopy(values[name])) # Decide which sub tree of an if-condition is relevant based on current variable situation elif node.type == "if": Console.debug("Processing if-condition at %s", node.line) # Pre-process condition # We manually process each child in for if-types __recurser(node.condition, scope, values, profile) # Cast condition to Python boolean type resultValue = Operation.castToBool(node.condition) # Process relevant part of the sub tree if resultValue is True: # Fix missing processing of result node __recurser(node.thenPart, scope, values, profile) # Finally replace if-node with result node node.parent.replace(node, node.thenPart) elif resultValue is False and hasattr(node, "elsePart"): # Fix missing processing of result node __recurser(node.elsePart, scope, values, profile) # Finally replace if-node with result node node.parent.replace(node, node.elsePart) else: # Cleanup original if-node node.parent.remove(node) # Nothing to do here as content is already processed return # Update scope of new block starts if hasattr(node, "scope"): relation = getattr(node, "rel", None) # Conditional blocks are not exactly blocks in this variable resolution engine if not relation in ("thenPart", "elsePart"): scope = node.scope values = copy.copy(values) node.values = values # Reset all local variables to None # which enforces not to keep values from outer scope for name in scope.modified: values[name] = None # Process children / content for child in list(node): # Ignore non-children... through possible interactive structure changes if child and child.parent is node: __recurser(child, scope, values, profile) # Update values of variables # This happens after processing children to possibly reduce child structure to an easy to assign (aka preprocessed value) if (node.type == "declaration" and hasattr(node, "initializer")) or node.type == "assign": if node.type == "declaration": name = node.name init = node.initializer Console.debug("Found declaration of %s at line %s", name, node.line) else: name = node[0].name init = node[1] Console.debug("Found assignment of %s at line %s", name, node.line) # Modify value instead of replace when assign operator is set if hasattr(node, "assignOp") and node.assignOp is not None: if name not in values: raise ExecuterError( "Assign operator is not supported as left hand variable is missing: %s" % name, node) repl = Operation.compute(node, values[name], init, node.assignOp) if repl is not None: values[name] = repl else: # Update internal variable mapping # Console.debug("Update value of %s to %s" % (name, init)) values[name] = init # Remove declaration node from tree node.parent.remove(node) # Support for variables inside property names or selectors elif node.type in ("property", "selector") and getattr( node, "dynamic", False): def replacer(matchObj): name = matchObj.group(1) if name not in values: raise ExecuterError( "Could not resolve variable %s! Missing value!" % name, node) value = values[name] if value is None: raise ExecuterError( "Could not resolve variable %s! Value is none!" % name, node) if value.type == "identifier": return value.value elif value.type == "string": return value.value elif value.type == "number": return "%s%s" % (value.value, getattr(value, "unit", "")) else: raise ExecuterError( "Could not replace property inline variable with value of type: %s" % value.type, node) # Fix all selectors if node.type == "selector": selectors = node.name for pos, selector in enumerate(selectors): selectors[pos] = RE_INLINE_VARIABLE.sub(replacer, selector) else: node.name = RE_INLINE_VARIABLE.sub(replacer, node.name) # Execute system commands elif node.type == "command": repl = Util.executeCommand(node, profile) if not repl is node: node.parent.replace(node, repl) # Support typical operators elif node.type in Util.ALL_OPERATORS: repl = Operation.compute(node) if repl is not None: node.parent.replace(node, repl)
def addImageSpriteData(self, id, left, top): Console.debug("Registering sprite location for %s: %s@%sx%s", self.id, id, left, top) self.__imageSpriteData = [id, left, top]
def checkLinksInItem(item): # Process types if "type" in item: if item["type"] == "Function": # Check param types if "params" in item: for paramName in item["params"]: paramEntry = item["params"][paramName] if "type" in paramEntry: for paramTypeEntry in paramEntry["type"]: if not paramTypeEntry[ "name"] in knownClasses and not paramTypeEntry[ "name"] in additionalTypes and not ( "builtin" in paramTypeEntry or "pseudo" in paramTypeEntry): item["errornous"] = True Console.error( 'Invalid param type "%s" in %s' % (paramTypeEntry["name"], className)) if not "pseudo" in paramTypeEntry and paramTypeEntry[ "name"] in knownClasses: paramTypeEntry["linkable"] = True # Check return types if "returns" in item: for returnTypeEntry in item["returns"]: if not returnTypeEntry[ "name"] in knownClasses and not returnTypeEntry[ "name"] in additionalTypes and not ( "builtin" in returnTypeEntry or "pseudo" in returnTypeEntry): item["errornous"] = True Console.error( 'Invalid return type "%s" in %s' % (returnTypeEntry["name"], className)) if not "pseudo" in returnTypeEntry and returnTypeEntry[ "name"] in knownClasses: returnTypeEntry["linkable"] = True elif not item["type"] in knownClasses and not item[ "type"] in builtinTypes and not item[ "type"] in pseudoTypes and not item[ "type"] in additionalTypes: item["errornous"] = True Console.error('Invalid type "%s" in %s' % (item["type"], className)) # Process doc if "doc" in item: def processInternalLink(match): linkUrl = match.group(2) if linkUrl.startswith("#"): linkCheck = checkInternalLink(linkUrl[1:], className) if linkCheck is not True: item["errornous"] = True if sectionName: Console.error( "%s in %s:%s~%s" % (linkCheck, sectionName, className, name)) else: Console.error("%s in %s" % (linkCheck, className)) linkExtract.sub(processInternalLink, item["doc"])
def __recurser(node, table): counter = 0 # Process children for child in list(node): if child is not None: counter += __recurser(child, table) # Process all method calls if node.type == "call": funcName = None funcNameNode = None # Uses global translation method (not typical) if node[0].type == "identifier": funcNameNode = node[0] # Uses namespaced translation method. # Typically core.locale.Translation.tr() or Translation.tr() elif node[0].type == "dot" and node[0][1].type == "identifier": funcNameNode = node[0][1] # Gettext methods only at the moment funcName = funcNameNode and funcNameNode.value if funcName in translationFunctions: Console.debug("Found translation method %s in %s", funcName, node.line) Console.indent() params = node[1] # Remove marktr() calls if funcName == "marktr": node.parent.remove(node) # Verify param types elif params[0].type is not "string": # maybe something marktr() relevant being used, in this case we need to keep the call and inline the data pass # Error handling elif (funcName == "trn" or funcName == "trc") and params[1].type != "string": Console.warn( "Expecting translation string to be type string: %s at line %s" % (params[1].type, params[1].line)) # Signature tr(msg, arg1, ...) elif funcName == "tr": key = params[0].value if key in table: params[0].value = table[key] counter += 1 if len(params) == 1: node.parent.replace(node, params[0]) else: replacement = __splitTemplate(params[0].value, params[1:]) if replacement: node.parent.replace(node, replacement) # Signature trc(context, msg, arg1, ...) elif funcName == "trc": key = "%s[C:%s]" % (params[1].value, params[0].value) if key in table: params[1].value = table[key] counter += 1 if len(params) == 2: node.parent.replace(node, params[1]) else: replacement = __splitTemplate(params[1].value, params[2:]) if replacement: node.parent.replace(node, replacement) # Signature trn(msgSingular, msgPlural, int, arg1, ...) elif funcName == "trn": key = "%s[N:%s]" % (params[0].value, params[1].value) if not key in table: Console.outdent() return counter counter += 1 # Use optimized trnc() method instead of trn() funcNameNode.value = "trnc" # Remove first two string parameters params.remove(params[0]) params.remove(params[0]) # Inject new object into params container = Node.Node(None, "object_init") params.insert(0, container) # Create new construction with all properties generated from the translation table for plural in table[key]: pluralEntry = Node.Node(None, "property_init") pluralEntryIdentifier = Node.Node(None, "identifier") pluralEntryIdentifier.value = plural pluralEntryValue = Node.Node(None, "string") pluralEntryValue.value = table[key][plural] pluralEntry.append(pluralEntryIdentifier) pluralEntry.append(pluralEntryValue) container.append(pluralEntry) # Replace strings with plus operations to omit complex client side string operation if len(params) > 2: for pluralEntry in container: replacement = __splitTemplate(pluralEntry[1].value, params[2:]) if replacement: pluralEntry.replace(pluralEntry[1], replacement) # When all variables have been patched in all string with placeholder # we are able to remove the whole list of placeholder values afterwards while len(params) > 2: params.pop() Console.outdent() return counter
def skip(self): """Eats comments and whitespace.""" input = self.source startLine = self.line # Whether this is the first called as happen on start parsing a file (eat leading comments/white space) startOfFile = self.cursor is 0 indent = "" while True: if len(input) > self.cursor: ch = input[self.cursor] else: return self.cursor += 1 if len(input) > self.cursor: next = input[self.cursor] else: next = None if ch == "\n" and not self.scanNewlines: self.line += 1 indent = "" elif ch == "/" and next == "*": self.cursor += 1 text = "/*" inline = startLine == self.line and startLine > 1 commentStartLine = self.line if startLine == self.line and not startOfFile: mode = "inline" elif (self.line - 1) > startLine: # distance before this comment means it is a comment block for a whole section (multiple lines of code) mode = "section" else: # comment for maybe multiple following lines of code, but not that important (no visual white space divider) mode = "block" while True: try: ch = input[self.cursor] self.cursor += 1 except IndexError: raise ParseError("Unterminated comment", self.fileId, self.line) if ch == "*": next = input[self.cursor] if next == "/": text += "*/" self.cursor += 1 break elif ch == "\n": self.line += 1 text += ch # Filter escaping on slash-star combinations in comment text text = text.replace("*\/", "*/") try: self.comments.append( Comment.Comment(text, mode, commentStartLine, indent, self.fileId)) except Comment.CommentException as commentError: Console.error("Ignoring comment in %s: %s", self.fileId, commentError) elif ch == "/" and next == "/": self.cursor += 1 text = "//" if startLine == self.line and not startOfFile: mode = "inline" elif (self.line - 1) > startLine: # distance before this comment means it is a comment block for a whole section (multiple lines of code) mode = "section" else: # comment for maybe multiple following lines of code, but not that important (no visual white space divider) mode = "block" while True: try: ch = input[self.cursor] self.cursor += 1 except IndexError: # end of file etc. break if ch == "\n": self.line += 1 break text += ch try: self.comments.append( Comment.Comment(text, mode, self.line - 1, "", self.fileId)) except Comment.CommentException: Console.error("Ignoring comment in %s: %s", self.fileId, commentError) # check for whitespace, also for special cases like 0xA0 elif ch in "\xA0 \t": indent += ch else: self.cursor -= 1 return
def __outdent(self, text, indent, startLineNo): """ Outdent multi line comment text and filtering empty lines """ lines = [] # First, split up the comments lines and remove the leading indentation for lineNo, line in enumerate((indent + text).split("\n")): if line.startswith(indent): lines.append(line[len(indent):].rstrip()) elif line.strip() == "": lines.append("") else: # Only warn for doc comments, otherwise it might just be code commented out # which is sometimes formatted pretty crazy when commented out if self.variant == "doc": Console.warn( "Could not outdent doc comment at line %s in %s", startLineNo + lineNo, self.fileId) return text # Find first line with real content, then grab the one after it to get the # characters which need outdentString = "" for lineNo, line in enumerate(lines): if line != "" and line.strip() != "": matchedDocIndent = docIndentReg.match(line) if not matchedDocIndent: # As soon as we find a non doc indent like line we stop break elif matchedDocIndent.group(2) != "": # otherwise we look for content behind the indent to get the # correct real indent (with spaces) outdentString = matchedDocIndent.group(1) break lineNo += 1 # Process outdenting to all lines (remove the outdentString from the start of the lines) if outdentString != "": lineNo = 0 outdentStringLen = len(outdentString) for lineNo, line in enumerate(lines): if len(line) <= outdentStringLen: lines[lineNo] = "" else: if not line.startswith(outdentString): # Only warn for doc comments, otherwise it might just be code commented out # which is sometimes formatted pretty crazy when commented out if self.variant == "doc": Console.warn( "Invalid indentation in doc comment at line %s in %s", startLineNo + lineNo, self.fileId) else: lines[lineNo] = line[outdentStringLen:] # Merge final lines and remove leading and trailing new lines return "\n".join(lines).strip("\n")
def checkSingleInstallation(keys, versions, packageName, minVersion, installPath, updatePath): if packageName.lower() in keys: if StrictVersion(minVersion) > StrictVersion("0.0"): if StrictVersion(versions[ packageName.lower()]) < StrictVersion(minVersion): Console.info( Console.colorize( Console.colorize( 'Jasy requirement error: "%s"' % packageName, "red"), "bold")) Console.indent() Console.info( Console.colorize( Console.colorize( 'Version is NOT OK (needed: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "red"), "bold")) Console.info( 'Update to the newest version of %s using %s' % (packageName, updatePath)) Console.outdent() return False else: Console.info( Console.colorize( Console.colorize( 'Jasy requirement error: "%s"' % packageName, "red"), "bold")) Console.indent() Console.info( Console.colorize( Console.colorize('Did NOT find installation', "red"), "bold")) Console.info('Install the newest version of %s using %s' % (packageName, installPath)) Console.outdent() return False return True
# # Jasy - Web Tooling Framework # Copyright 2010-2012 Zynga Inc. # import jasy.core.Console as Console from distutils.version import StrictVersion try: import pip except ImportError: Console.error("pip is required to run JASY!") sys.exit(1) needs = [{ "packageName": "Pygments", "minVersion": "1.5", "installPath": "'pip install Pygments'", "updatePath": "'pip install --upgrade pygments'" }, { "packageName": "polib", "minVersion": "1.0.1", "installPath": "'pip install polib'", "updatePath": "'pip install --upgrade polib'" }, { "packageName": "requests", "minVersion": "0.14", "installPath": "'pip install requests'", "updatePath": "'pip install --upgrade requests'" }, { "packageName": "CherryPy",
def clean(self): """Clears the cache of the project.""" Console.info("Clearing cache of %s..." % self.__name) self.__cache.clear()
def scan(self): if self.scanned: return updatemsg = "[updated]" if self.__modified else "[cached]" if self.version: Console.info("Scanning %s @ %s %s...", Console.colorize(self.getName(), "bold"), Console.colorize(self.version, "magenta"), Console.colorize(updatemsg, "grey")) else: Console.info("Scanning %s %s...", Console.colorize(self.getName(), "bold"), Console.colorize(updatemsg, "grey")) Console.indent() # Support for pre-initialize projects... setup = self.__setup if setup and self.__modified: Console.info("Running setup...") Console.indent() for cmd in setup: Console.info("Executing %s...", cmd) result = None try: result = None result = Util.executeCommand( cmd, "Failed to execute setup command %s" % cmd, path=self.__path) except Exception as ex: if result: Console.error(result) raise UserError("Could not scan project %s: %s" % (self.__name, ex)) Console.outdent() # Processing custom content section. Only supports classes and assets. if self.__config.has("content"): self.kind = "manual" self.__addContent(self.__config.get("content")) else: # Read scan path from config if not self.__config.has("scan"): if self.__hasDir("source"): self.kind = "application" scan = self.__resolveScanConfig(structures[self.kind]) elif self.__hasDir("src"): self.kind = "resource" scan = self.__resolveScanConfig(structures[self.kind]) else: self.kind = "flat" scan = self.__resolveScanConfig(structures[self.kind]) else: scan = self.__resolveScanConfig(self.__config.get("scan")) for config in scan: if isinstance(config["paths"], str): self.__addDir(config["paths"], config["regex"], config["type"], config["package"]) else: for path in config["paths"]: self.__addDir(path, config["regex"], config["type"], config["package"]) # Generate summary summary = [] for section in self.items.keys(): content = self.items[section] name, constructor = self.__resolveConstructor(section) if content: summary.append( Console.colorize("%s %s" % (len(content), name), "magenta")) # Print out if summary: Console.info("Content: %s" % (", ".join(summary))) self.scanned = True Console.outdent()
def __resolve(project): name = project.getName() # List of required projects Console.info("Getting requirements of %s...", Console.colorize(name, "bold")) Console.indent() requires = project.getRequires(checkoutDirectory, updateRepositories) Console.outdent() if not requires: return Console.debug("Processing %s requirements...", len(requires)) Console.indent() # Adding all project in reverse order. # Adding all local ones first before going down to their requirements childProjects = [] for requiredProject in reversed(requires): requiredName = requiredProject.getName() if requiredName not in names: Console.debug("Adding: %s %s (via %s)", requiredName, requiredProject.version, project.getName()) names[requiredName] = True result.append(requiredProject) childProjects.append(requiredProject) elif not requiredProject in result: Console.debug("Blocking: %s %s (via %s)", requiredName, requiredProject.version, project.getName()) requiredProject.pause() # Process all requirements of added projects for requiredProject in reversed(childProjects): if requiredProject.hasRequires(): __resolve(requiredProject) Console.outdent()
def __cleanup(node): """Reprocesses JavaScript to remove dead paths.""" optimized = False # Process from inside to outside for child in reversed(node): # None children are allowed sometimes e.g. during array_init like [1,2,,,7,8] if child is not None: if __cleanup(child): optimized = True # Optimize if cases if node.type == "if": check = __checkCondition(node.condition) if check is not None: optimized = True Console.debug("Optimizing if/else at line %s", node.line) if check is True: node.parent.replace(node, node.thenPart) elif check is False: if hasattr(node, "elsePart"): node.parent.replace(node, node.elsePart) else: node.parent.remove(node) # Optimize hook statement if node.type == "hook": check = __checkCondition(node[0]) if check is not None: Console.debug("Optimizing hook at line %s", node.line) optimized = True if check is True: node.parent.replace(node, node[1]) elif check is False: node.parent.replace(node, node[2]) # Optimize switch statement if node.type == "switch" and node.discriminant.type in ("string", "number"): discriminant = node.discriminant.value fallback = None matcher = None allowed = ["default", "case"] for child in node: # Require that every case block ends with a break (no fall-throughs) if child.type == "case": block = child[len(child) - 1] if len(block) == 0 or block[len(block) - 1].type != "break": Console.warn( "Could not optimize switch statement (at line %s) because of fallthrough break statement.", node.line) return False if child.type == "default": fallback = child.statements elif child.type == "case" and child.label.value == discriminant: matcher = child.statements # Remove break statement matcher.pop() if matcher or fallback: if not matcher: matcher = fallback node.parent.replace(node, matcher) Console.debug("Optimizing switch at line %s", node.line) optimized = True return optimized
def connectInterface(className, interfaceName, classApi, interfaceApi): Console.debug("- Connecting %s with %s", className, interfaceName) # # Properties # interfaceProperties = getattr(interfaceApi, "properties", None) if interfaceProperties: classProperties = getattr(classApi, "properties", {}) for name in interfaceProperties: if name not in classProperties: Console.warn( "Class %s is missing implementation for property %s of interface %s!", className, name, interfaceName) else: # Add reference to interface if "interface" not in classProperties[name]: classProperties[name]["defined"] = [] classProperties[name]["defined"].append({ "name": interfaceName, "link": "property:%s~%s" % (interfaceName, name) }) # Copy over documentation if "doc" not in classProperties[ name] and "doc" in interfaceProperties[name]: classProperties[name]["doc"] = interfaceProperties[name][ "doc"] if "summary" not in classProperties[ name] and "summary" in interfaceProperties[name]: classProperties[name]["summary"] = interfaceProperties[ name]["summary"] if "errornous" in classProperties[ name] and "errornous" not in interfaceProperties[name]: del classProperties[name]["errornous"] # Update tags with data from interface if "tags" in interfaceProperties[name]: if "tags" not in classProperties[name]: classProperties[name]["tags"] = {} safeUpdate(classProperties[name]["tags"], interfaceProperties[name]["tags"]) # # Events # interfaceEvents = getattr(interfaceApi, "events", None) if interfaceEvents: classEvents = getattr(classApi, "events", {}) for name in interfaceEvents: if name not in classEvents: Console.warn( "Class %s is missing implementation for event %s of interface %s!", className, name, interfaceName) else: # Add reference to interface if "interface" not in classEvents[name]: classEvents[name]["defined"] = [] classEvents[name]["defined"].append({ "name": interfaceName, "link": "event:%s~%s" % (interfaceName, name) }) # Copy user event type and documentation from interface if "doc" not in classEvents[name] and "doc" in interfaceEvents[ name]: classEvents[name]["doc"] = interfaceEvents[name]["doc"] if "summary" not in classEvents[ name] and "summary" in interfaceEvents[name]: classEvents[name]["summary"] = interfaceEvents[name][ "summary"] if "type" not in classEvents[ name] and "type" in interfaceEvents[name]: classEvents[name]["type"] = interfaceEvents[name]["type"] if "errornous" in classEvents[ name] and not "errornous" in interfaceEvents[name]: del classEvents[name]["errornous"] # Update tags with data from interface if "tags" in interfaceEvents[name]: if not "tags" in classEntry: classEvents[name]["tags"] = {} safeUpdate(classEvents[name]["tags"], interfaceEvents[name]["tags"]) # # Members # interfaceMembers = getattr(interfaceApi, "members", None) if interfaceMembers: classMembers = getattr(classApi, "members", {}) for name in interfaceMembers: if name not in classMembers: Console.warn( "Class %s is missing implementation for member %s of interface %s!", className, name, interfaceName) else: interfaceEntry = interfaceMembers[name] classEntry = classMembers[name] # Add reference to interface if not "interface" in classEntry: classEntry["defined"] = [] classEntry["defined"].append({ "name": interfaceName, "link": "member:%s~%s" % (interfaceName, name) }) # Copy over doc from interface if not "doc" in classEntry and "doc" in interfaceEntry: classEntry["doc"] = interfaceEntry["doc"] if not "summary" in classEntry and "summary" in interfaceEntry: classEntry["summary"] = interfaceEntry["summary"] if "errornous" in classEntry and not "errornous" in interfaceEntry: del classEntry["errornous"] # Priorize return value from interface (it's part of the interface feature set to enforce this) if "returns" in interfaceEntry: classEntry["returns"] = interfaceEntry["returns"] # Update tags with data from interface if "tags" in interfaceEntry: if not "tags" in classEntry: classEntry["tags"] = {} safeUpdate(classEntry["tags"], interfaceEntry["tags"]) # Copy over params from interface if "params" in interfaceEntry: # Fix completely missing parameters if not "params" in classEntry: classEntry["params"] = {} for paramName in interfaceEntry["params"]: # Prefer data from interface if not paramName in classEntry["params"]: classEntry["params"][paramName] = {} classEntry["params"][paramName].update( interfaceEntry["params"][paramName]) # Clear errournous documentation flags if "errornous" in classEntry["params"][ paramName] and not "errornous" in interfaceEntry[ "params"][paramName]: del classEntry["params"][paramName]["errornous"]
def __recurser(node, unused): """ The cleanup part which always processes one scope and cleans up params and variable definitions which are unused """ retval = False # Process children if node.type != "function": for child in node: # None children are allowed sometimes e.g. during array_init like [1,2,,,7,8] if child != None: if __recurser(child, unused): retval = True if node.type == "script" and hasattr(node, "parent"): # Remove unused parameters params = getattr(node.parent, "params", None) if params: # Start from back, as we can only remove params as long # as there is not a required one after the current one for identifier in reversed(params): if identifier.value in unused: Console.debug("Removing unused parameter '%s' in line %s", identifier.value, identifier.line) params.remove(identifier) retval = True else: break # Remove function names which are unused if node.parent.functionForm == "expressed_form": funcName = getattr(node.parent, "name", None) if funcName != None and funcName in unused: Console.debug("Removing unused function name at line %s" % node.line) del node.parent.name retval = True elif node.type == "function": # Remove full unused functions (when not in top-level scope) if node.functionForm == "declared_form" and getattr( node, "parent", None) and node.parent.type != "call": funcName = getattr(node, "name", None) if funcName != None and funcName in unused: Console.debug( "Removing unused function declaration %s at line %s" % (funcName, node.line)) node.parent.remove(node) retval = True elif node.type == "var": for decl in reversed(node): if getattr(decl, "name", None) in unused: if hasattr(decl, "initializer"): init = decl.initializer if init.type in ("null", "this", "true", "false", "identifier", "number", "string", "regexp"): Console.debug( "Removing unused primitive variable %s at line %s" % (decl.name, decl.line)) node.remove(decl) retval = True elif init.type == "function" and ( not hasattr(init, "name") or init.name in unused): Console.debug( "Removing unused function variable %s at line %s" % (decl.name, decl.line)) node.remove(decl) retval = True # If we have only one child, we replace the whole var statement with just the init block elif len(node) == 1: semicolon = Node.Node(init.tokenizer, "semicolon") semicolon.append(init, "expression") # Protect non-expressions with parens if init.type in ("array_init", "object_init"): init.parenthesized = True elif init.type == "call" and init[0].type == "function": init[0].parenthesized = True node.parent.replace(node, semicolon) retval = True # If we are the last declaration, move it out of node and append after var block elif node[-1] == decl or node[0] == decl: isFirst = node[0] == decl node.remove(decl) nodePos = node.parent.index(node) semicolon = Node.Node(init.tokenizer, "semicolon") semicolon.append(init, "expression") # Protect non-expressions with parens if init.type in ("array_init", "object_init"): init.parenthesized = True elif init.type == "call" and init[0].type == "function": init[0].parenthesized = True if isFirst: node.parent.insert(nodePos, semicolon) else: node.parent.insert(nodePos + 1, semicolon) retval = True else: Console.debug( "Could not automatically remove unused variable %s at line %s without possible side-effects" % (decl.name, decl.line)) else: node.remove(decl) retval = True if len(node) == 0: Console.debug("Removing empty 'var' block at line %s" % node.line) node.parent.remove(node) return retval
def __getLoadDepsRecurser(self, classObj, stack): """ This is the main routine which tries to control over a system of unsorted classes. It directly tries to fullfil every dependency a class have, but has some kind of exception based loop protection to prevent circular dependencies from breaking the build. It respects break information given by file specific meta data, but also adds custom hints where it found recursions. This lead to a valid sort, but might lead to problems between exeactly the two affected classes. Without doing an exact execution it's not possible to whether found out which of two each-other referencing classes needs to be loaded first. This is basically only interesting in cases where one class needs another during the definition phase which is not the case that often. """ if classObj in stack: stack.append(classObj) msg = " >> ".join( [x.getId() for x in stack[stack.index(classObj):]]) raise CircularDependency("Circular Dependency: %s" % msg) stack.append(classObj) classDeps = classObj.getDependencies(self.__permutation, classes=self.__names, warnings=False) classMeta = classObj.getMetaData(self.__permutation) result = set() circular = set() # Respect manually defined breaks # Breaks are dependencies which are down-priorized to break # circular dependencies between classes. for breakName in classMeta.breaks: if breakName in self.__names: circular.add(self.__names[breakName]) # Now process the deps of the given class loadDeps = self.__loadDeps for depObj in classDeps: if depObj is classObj: continue depName = depObj.getId() if depName in classMeta.breaks: Console.debug("Manual Break: %s => %s" % (classObj, depObj)) pass elif depObj in loadDeps: result.update(loadDeps[depObj]) result.add(depObj) else: current = self.__getLoadDepsRecurser(depObj, stack[:]) result.update(current) result.add(depObj) # Sort dependencies by number of other dependencies # For performance reasions we access the __loadDeps # dict directly as this data is already stored result = sorted(result, key=lambda depObj: len(self.__loadDeps[depObj])) loadDeps[classObj] = result if circular: self.__circularDeps[classObj] = circular return result
def doCompleteDoctor(): """Checks for uninstalled or too old versions of requirements and gives a complete output""" Console.header("Doctor") dists = [dist for dist in pip.get_installed_distributions()] keys = [dist.key for dist in pip.get_installed_distributions()] versions = {} for dist in dists: versions[dist.key] = dist.version def checkSingleInstallation(keys, versions, packageName, minVersion, installPath, updatePath): Console.info('%s:' % packageName) Console.indent() if packageName.lower() in keys: Console.info(Console.colorize('Found installation', "green")) if StrictVersion(minVersion) > StrictVersion("0.0"): if StrictVersion(versions[ packageName.lower()]) >= StrictVersion(minVersion): Console.info( Console.colorize( 'Version is OK (needed: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "green")) else: Console.info( Console.colorize( Console.colorize( '- Version is NOT OK (needed: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "red"), "bold")) Console.info( 'Update to the newest version of %s using %s' % (packageName, updatePath)) else: Console.info( Console.colorize( Console.colorize('Did NOT find installation', "red"), "bold")) Console.info('Install the newest version of %s using %s' % (packageName, installPath)) Console.outdent() # Required packages Console.info(Console.colorize("Required Packages:", "bold")) Console.indent() for entry in needs: checkSingleInstallation(keys, versions, entry["packageName"], entry["minVersion"], entry["installPath"], entry["updatePath"]) Console.outdent() # Optional packages Console.info("") Console.info(Console.colorize("Optional Packages:", "bold")) Console.indent() for entry in optionals: checkSingleInstallation(keys, versions, entry["packageName"], entry["minVersion"], entry["installPath"], entry["updatePath"]) Console.outdent()
def processSprites(self): """Processes jasysprite files to merge sprite data into asset registry.""" assets = self.__assets configs = [ fileId for fileId in assets if assets[fileId].isImageSpriteConfig() ] if configs: Console.info( "Processing %s...", Console.colorize("%s sprites", "magenta") % len(configs)) sprites = [] Console.indent() for fileId in configs: Console.debug("Processing %s...", fileId) asset = assets[fileId] spriteBase = os.path.dirname(fileId) try: spriteConfig = asset.getParsedObject() except ValueError as err: raise UserError("Could not parse jasysprite at %s: %s" % (fileId, err)) Console.indent() for spriteImage in spriteConfig: spriteImageId = "%s/%s" % (spriteBase, spriteImage) singleRelPaths = spriteConfig[spriteImage] Console.debug("Image %s combines %s images", spriteImageId, len(singleRelPaths)) for singleRelPath in singleRelPaths: singleId = "%s/%s" % (spriteBase, singleRelPath) singleData = singleRelPaths[singleRelPath] singleItem = assets[singleId] # Verify that sprite sheet is up-to-date fileChecksum = singleItem.getChecksum() storedChecksum = singleData["checksum"] Console.debug("Checksum Compare: %s <=> %s", fileChecksum, storedChecksum) if storedChecksum != fileChecksum: raise UserError( "Sprite Sheet is not up-to-date. Checksum of %s differs." % singleId) if spriteImageId not in sprites: spriteImageIndex = len(sprites) sprites.append(spriteImageId) else: spriteImageIndex = sprites.index(spriteImageId) # Add relevant data to find image on sprite sheet singleItem.addImageSpriteData(spriteImageIndex, singleData["left"], singleData["top"]) Console.outdent() # The config file does not make any sense on the client side Console.debug("Deleting sprite config from assets: %s", fileId) del assets[fileId] Console.outdent() self.__sprites = sprites
def __init__(self, locale): Console.info("Parsing CLDR files for %s..." % locale) Console.indent() splits = locale.split("_") # Store for internal usage self.__locale = locale self.__language = splits[0] self.__territory = splits[1] if len(splits) > 1 else None # This will hold all data extracted data self.__data = {} # Add info section self.__data["info"] = { "LOCALE": self.__locale, "LANGUAGE": self.__language, "TERRITORY": self.__territory } # Add keys (fallback to C-default locale) path = "%s.xml" % os.path.join(CLDR_DIR, "keys", self.__language) try: Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) except IOError: path = "%s.xml" % os.path.join(CLDR_DIR, "keys", "C") Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) self.__data["key"] = { "Short": { key.get("type"): key.text for key in tree.findall("./keys/short/key") }, "Full": { key.get("type"): key.text for key in tree.findall("./keys/full/key") } } # Add main CLDR data: Fallback chain for locales main = os.path.join(CLDR_DIR, "main") files = [] while True: files.append("%s.xml" % os.path.join(main, locale)) if "_" in locale: locale = locale[:locale.rindex("_")] else: break # Extend data with root data files.append(os.path.join(main, "root.xml")) # Finally import all these files in order for path in reversed(files): Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) self.__addDisplayNames(tree) self.__addDelimiters(tree) self.__addCalendars(tree) self.__addNumbers(tree) # Add supplemental CLDR data self.__addSupplementals(self.__territory) Console.outdent()
def __process(self, apiData, classFilter=None, internals=False, privates=False, printErrors=True, highlightCode=True): knownClasses = set(list(apiData)) # # Attaching Links to Source Code (Lines) # Building Documentation Summaries # Console.info("Adding Source Links...") for className in apiData: classApi = apiData[className] constructData = getattr(classApi, "construct", None) if constructData is not None: if "line" in constructData: constructData["sourceLink"] = "source:%s~%s" % ( className, constructData["line"]) for section in ("properties", "events", "statics", "members"): sectionData = getattr(classApi, section, None) if sectionData is not None: for name in sectionData: if "line" in sectionData[name]: sectionData[name][ "sourceLink"] = "source:%s~%s" % ( className, sectionData[name]["line"]) # # Including Mixins / IncludedBy # Console.info("Resolving Mixins...") Console.indent() # Just used temporary to keep track of which classes are merged mergedClasses = set() def getApi(className): classApi = apiData[className] if className in mergedClasses: return classApi classIncludes = getattr(classApi, "includes", None) if classIncludes: for mixinName in classIncludes: if mixinName not in apiData: Console.error("Invalid mixin %s in class %s", className, mixinName) continue mixinApi = apiData[mixinName] if not hasattr(mixinApi, "includedBy"): mixinApi.includedBy = set() mixinApi.includedBy.add(className) mergeMixin(className, mixinName, classApi, getApi(mixinName)) mergedClasses.add(className) return classApi for className in apiData: apiData[className] = getApi(className) Console.outdent() # # Checking links # Console.info("Checking Links...") additionalTypes = ("Call", "Identifier", "Map", "Integer", "Node", "Element", "Class", "Exception", "Uri") def checkInternalLink(link, className): match = internalLinkParse.match(link) if not match: return 'Invalid link "#%s"' % link if match.group(3) is not None: className = match.group(3) if not className in knownClasses and not className in apiData: return 'Invalid class in link "#%s"' % link # Accept all section/item values for named classes, # as it might be pretty complicated to verify this here. if className not in apiData: return True classApi = apiData[className] sectionName = match.group(2) itemName = match.group(5) if itemName is None: return True if sectionName is not None: if sectionName not in linkMap: return 'Invalid section in link "#%s"' % link section = getattr(classApi, linkMap[sectionName], None) if section is None: return 'Invalid section in link "#%s"' % link else: if itemName in section: return True return 'Invalid item in link "#%s"' % link for sectionName in ("statics", "members", "properties", "events"): section = getattr(classApi, sectionName, None) if section and itemName in section: return True return 'Invalid item link "#%s"' % link def checkLinksInItem(item): # Process types if "type" in item: if item["type"] == "Function": # Check param types if "params" in item: for paramName in item["params"]: paramEntry = item["params"][paramName] if "type" in paramEntry: for paramTypeEntry in paramEntry["type"]: if not paramTypeEntry[ "name"] in knownClasses and not paramTypeEntry[ "name"] in additionalTypes and not ( "builtin" in paramTypeEntry or "pseudo" in paramTypeEntry): item["errornous"] = True Console.error( 'Invalid param type "%s" in %s' % (paramTypeEntry["name"], className)) if not "pseudo" in paramTypeEntry and paramTypeEntry[ "name"] in knownClasses: paramTypeEntry["linkable"] = True # Check return types if "returns" in item: for returnTypeEntry in item["returns"]: if not returnTypeEntry[ "name"] in knownClasses and not returnTypeEntry[ "name"] in additionalTypes and not ( "builtin" in returnTypeEntry or "pseudo" in returnTypeEntry): item["errornous"] = True Console.error( 'Invalid return type "%s" in %s' % (returnTypeEntry["name"], className)) if not "pseudo" in returnTypeEntry and returnTypeEntry[ "name"] in knownClasses: returnTypeEntry["linkable"] = True elif not item["type"] in knownClasses and not item[ "type"] in builtinTypes and not item[ "type"] in pseudoTypes and not item[ "type"] in additionalTypes: item["errornous"] = True Console.error('Invalid type "%s" in %s' % (item["type"], className)) # Process doc if "doc" in item: def processInternalLink(match): linkUrl = match.group(2) if linkUrl.startswith("#"): linkCheck = checkInternalLink(linkUrl[1:], className) if linkCheck is not True: item["errornous"] = True if sectionName: Console.error( "%s in %s:%s~%s" % (linkCheck, sectionName, className, name)) else: Console.error("%s in %s" % (linkCheck, className)) linkExtract.sub(processInternalLink, item["doc"]) Console.indent() # Process APIs for className in apiData: classApi = apiData[className] sectionName = None constructData = getattr(classApi, "construct", None) if constructData is not None: checkLinksInItem(constructData) for sectionName in ("properties", "events", "statics", "members"): section = getattr(classApi, sectionName, None) if section is not None: for name in section: checkLinksInItem(section[name]) Console.outdent() # # Filter Internals/Privates # Console.info("Filtering Items...") def isVisible(entry): if "visibility" in entry: visibility = entry["visibility"] if visibility == "private" and not privates: return False if visibility == "internal" and not internals: return False return True def filterInternalsPrivates(classApi, field): data = getattr(classApi, field, None) if data: for name in list(data): if not isVisible(data[name]): del data[name] for className in apiData: filterInternalsPrivates(apiData[className], "statics") filterInternalsPrivates(apiData[className], "members") # # Connection Interfaces / ImplementedBy # Console.info("Connecting Interfaces...") Console.indent() for className in apiData: classApi = getApi(className) if not hasattr(classApi, "main"): continue classType = classApi.main["type"] if classType == "core.Class": classImplements = getattr(classApi, "implements", None) if classImplements: for interfaceName in classImplements: interfaceApi = apiData[interfaceName] implementedBy = getattr(interfaceApi, "implementedBy", None) if not implementedBy: implementedBy = interfaceApi.implementedBy = [] implementedBy.append(className) connectInterface(className, interfaceName, classApi, interfaceApi) Console.outdent() # # Merging Named Classes # Console.info("Merging Named Classes...") Console.indent() for className in list(apiData): classApi = apiData[className] destName = classApi.main["name"] if destName is not None and destName != className: Console.debug("Extending class %s with %s", destName, className) if destName in apiData: destApi = apiData[destName] destApi.main["from"].append(className) else: destApi = apiData[destName] = Data.ApiData( destName, highlight=highlightCode) destApi.main = { "type": "Extend", "name": destName, "from": [className] } # If there is a "main" tag found in the class use its API description if "tags" in classApi.main and classApi.main[ "tags"] is not None and "main" in classApi.main["tags"]: if "doc" in classApi.main: destApi.main["doc"] = classApi.main["doc"] classApi.main["extension"] = True # Read existing data construct = getattr(classApi, "construct", None) statics = getattr(classApi, "statics", None) members = getattr(classApi, "members", None) if construct is not None: if hasattr(destApi, "construct"): Console.warn( "Overriding constructor in extension %s by %s", destName, className) destApi.construct = copy.copy(construct) if statics is not None: if not hasattr(destApi, "statics"): destApi.statics = {} for staticName in statics: destApi.statics[staticName] = copy.copy( statics[staticName]) destApi.statics[staticName]["from"] = className destApi.statics[staticName][ "fromLink"] = "static:%s~%s" % (className, staticName) if members is not None: if not hasattr(destApi, "members"): destApi.members = {} for memberName in members: destApi.members[memberName] = copy.copy( members[memberName]) destApi.members[memberName]["from"] = className destApi.members[memberName][ "fromLink"] = "member:%s~%s" % (className, memberName) Console.outdent() # # Connecting Uses / UsedBy # Console.info("Collecting Use Patterns...") # This matches all uses with the known classes and only keeps them if matched allClasses = set(list(apiData)) for className in apiData: uses = apiData[className].uses # Rebuild use list cleanUses = set() for use in uses: if use != className and use in allClasses: cleanUses.add(use) useEntry = apiData[use] if not hasattr(useEntry, "usedBy"): useEntry.usedBy = set() useEntry.usedBy.add(className) apiData[className].uses = cleanUses # # Building Search Index # Console.info("Building Search Index...") search = {} def addSearch(classApi, field): data = getattr(classApi, field, None) if data: for name in data: if name not in search: search[name] = set() search[name].add(className) for className in apiData: classApi = apiData[className] addSearch(classApi, "statics") addSearch(classApi, "members") addSearch(classApi, "properties") addSearch(classApi, "events") # # Post Process (dict to sorted list) # Console.info("Post Processing Data...") for className in sorted(apiData): classApi = apiData[className] convertTags(classApi.main) construct = getattr(classApi, "construct", None) if construct: convertFunction(construct) convertTags(construct) for section in ("statics", "members", "properties", "events"): items = getattr(classApi, section, None) if items: sortedList = [] for itemName in sorted(items): item = items[itemName] item["name"] = itemName if "type" in item and item["type"] == "Function": convertFunction(item) convertTags(item) sortedList.append(item) setattr(classApi, section, sortedList) # # Collecting Package Docs # Console.info("Collecting Package Docs...") Console.indent() # Inject existing package docs into api data for project in self.__session.getProjects(): docs = project.getDocs() for packageName in docs: if self.__isIncluded(packageName, classFilter): Console.debug("Creating package documentation %s", packageName) apiData[packageName] = docs[packageName].getApi() # Fill missing package docs for className in sorted(apiData): splits = className.split(".") packageName = splits[0] for split in splits[1:]: if packageName not in apiData: Console.warn("Missing package documentation %s", packageName) apiData[packageName] = Data.ApiData( packageName, highlight=highlightCode) apiData[packageName].main = { "type": "Package", "name": packageName } packageName = "%s.%s" % (packageName, split) # Now register all classes in their parent namespace/package for className in sorted(apiData): splits = className.split(".") packageName = ".".join(splits[:-1]) if packageName: package = apiData[packageName] # debug("- Registering class %s in parent %s", className, packageName) entry = { "name": splits[-1], "link": className, } classMain = apiData[className].main if "doc" in classMain and classMain["doc"]: summary = Text.extractSummary(classMain["doc"]) if summary: entry["summary"] = summary if "type" in classMain and classMain["type"]: entry["type"] = classMain["type"] if not hasattr(package, "content"): package.content = [entry] else: package.content.append(entry) Console.outdent() # # Writing API Index # Console.debug("Building Index...") index = {} for className in sorted(apiData): classApi = apiData[className] mainInfo = classApi.main # Create structure for className current = index for split in className.split("."): if split not in current: current[split] = {} current = current[split] # Store current type current["$type"] = mainInfo["type"] # Keep information if if hasattr(classApi, "content"): current["$content"] = True # # Return # return apiData, index, search
# # Jasy - Web Tooling Framework # Copyright 2010-2012 Zynga Inc. # Copyright 2013-2014 Sebastian Werner # from jasy import UserError import jasy.core.Console as Console # Make PIL (native module) optional try: from PIL import Image from PIL import ImageDraw Console.debug("Using Pillow (PIL)") except ImportError as err1: Image = None ImageDraw = None Console.debug("No support for Pillow (PIL)!") class SpriteSheet(): def __init__(self, packer, blocks): self.packer = packer self.width = packer.root.w self.height = packer.root.h self.blocks = blocks self.area = self.width * self.height
def __processSprites(self): """Processes jasysprite.json files to merge sprite data into asset registry""" assets = self.__assets configs = [ fileId for fileId in assets if assets[fileId].isImageSpriteConfig() ] if configs: Console.info("Processing %s image sprite configs...", len(configs)) sprites = [] Console.indent() for fileId in configs: Console.debug("Processing %s...", fileId) asset = assets[fileId] spriteBase = os.path.dirname(fileId) try: spriteConfig = asset.getParsedObject() except ValueError as err: raise UserError("Could not parse jasysprite.json at %s: %s" % (fileId, err)) Console.indent() for spriteImage in spriteConfig: spriteImageId = "%s/%s" % (spriteBase, spriteImage) singleRelPaths = spriteConfig[spriteImage] Console.debug("Image %s combines %s images", spriteImageId, len(singleRelPaths)) for singleRelPath in singleRelPaths: singleId = "%s/%s" % (spriteBase, singleRelPath) singleData = singleRelPaths[singleRelPath] if singleId in assets: singleAsset = assets[singleId] else: Console.info("Creating new asset: %s", singleId) singleAsset = jasy.item.Asset.AssetItem(None) assets[singleId] = singleAsset if not spriteImageId in sprites: spriteImageIndex = len(sprites) sprites.append(spriteImageId) else: spriteImageIndex = sprites.index(spriteImageId) singleAsset.addImageSpriteData(spriteImageIndex, singleData["left"], singleData["top"]) if "width" in singleData and "height" in singleData: singleAsset.addImageDimensionData( singleData["width"], singleData["height"]) # Verify that sprite sheet is up-to-date if "checksum" in singleData: fileChecksum = singleAsset.getChecksum() storedChecksum = singleData["checksum"] Console.debug("Checksum Compare: %s <=> %s", fileChecksum[0:6], storedChecksum[0:6]) if storedChecksum != fileChecksum: raise UserError( "Sprite Sheet is not up-to-date. Checksum of %s differs.", singleId) Console.outdent() Console.debug("Deleting sprite config from assets: %s", fileId) del assets[fileId] Console.outdent() self.__sprites = sprites
def write(self, distFolder, classFilter=None, callback="apiload", showInternals=False, showPrivates=False, printErrors=True, highlightCode=True): """ Writes API data generated from JavaScript into distFolder. :param distFolder: Where to store the API data :param classFilter: Tuple of classes or method to use for filtering :param callback: Name of callback to use for loading or None if pure JSON should be used :param showInternals: Include internal methods inside API data :param showPrivates: Include private methods inside API data :param printErrors: Whether errors should be printed to the console :param highlightCode: Whether to enable code highlighting using Pygments :type distFolder: str :type classFilter: tuple or function :type callback: function :type showInternals: bool :type showPrivates: bool :type printErrors: bool :type highlightCode: bool """ # # Collecting # Console.info("Collecting API Data...") Console.indent() apiData = {} highlightedCode = {} for project in self.__session.getProjects(): classes = project.getScripts() Console.info("Loading API of project %s: %s...", Console.colorize(project.getName(), "bold"), Console.colorize("%s classes" % len(classes), "cyan")) Console.indent() for className in classes: if self.__isIncluded(className, classFilter): data = classes[className].getApi(highlightCode) if not data.isEmpty: apiData[className] = data highlightedCode[className] = classes[ className].getHighlightedCode() else: Console.info("Skipping %s, class is empty." % className) Console.outdent() Console.outdent() # # Processing # Console.info("Processing API Data...") Console.indent() data, index, search = self.__process(apiData, classFilter=classFilter, internals=showInternals, privates=showPrivates, printErrors=printErrors, highlightCode=highlightCode) Console.outdent() # # Writing # Console.info("Storing API data...") Console.indent() writeCounter = 0 extension = "js" if callback else "json" compress = True class JsonEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, set): return list(obj) return json.JSONEncoder.default(self, obj) def encode(content, name): if compress: jsonContent = json.dumps(content, sort_keys=True, cls=JsonEncoder, separators=(',', ':')) else: jsonContent = json.dumps(content, sort_keys=True, cls=JsonEncoder, indent=2) if callback: return "%s(%s,'%s');" % (callback, jsonContent, name) else: return jsonContent Console.info("Saving class data (%s files)...", len(data)) Console.indent() for className in data: try: classData = data[className] if isinstance(classData, dict): classExport = classData else: classExport = classData.export() File.write( self.__profile.expandFileName( os.path.join(distFolder, "%s.%s" % (className, extension))), encode(classExport, className)) except TypeError as writeError: Console.error("Could not write API data of: %s: %s", className, writeError) continue Console.outdent() if highlightCode: Console.info("Saving highlighted code (%s files)...", len(highlightedCode)) Console.indent() for className in highlightedCode: try: File.write( self.__profile.expandFileName( os.path.join(distFolder, "%s.html" % className)), highlightedCode[className]) except TypeError as writeError: Console.error( "Could not write highlighted code of: %s: %s", className, writeError) continue Console.outdent() Console.info("Writing index...") Console.indent() File.write( self.__profile.expandFileName( os.path.join(distFolder, "meta-index.%s" % extension)), encode(index, "meta-index")) File.write( self.__profile.expandFileName( os.path.join(distFolder, "meta-search.%s" % extension)), encode(search, "meta-search")) Console.outdent() Console.outdent()
def __addSupplementals(self, territory): """ Converts data from supplemental folder """ supplemental = os.path.join(CLDR_DIR, "supplemental") # Plurals path = os.path.join(supplemental, "plurals.xml") Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) self.__data["Plural"] = {} for item in tree.findall("plurals/pluralRules"): attr = item.get("locales") if attr != None: if self.__language in attr.split(" "): for rule in item.findall("pluralRule"): jsPlural = pluralToJavaScript(rule.text) self.__data["Plural"][rule.get( "count").upper()] = jsPlural # Telephone Codes path = os.path.join(supplemental, "telephoneCodeData.xml") Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) for item in tree.findall("telephoneCodeData/codesByTerritory"): territoryId = item.get("territory") if territoryId == territory: for rule in item.findall("telephoneCountryCode"): self.__data["PhoneCode"] = {"CODE": int(rule.get("code"))} # Respect first only break # Postal Codes path = os.path.join(supplemental, "postalCodeData.xml") Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) for item in tree.findall("postalCodeData/postCodeRegex"): territoryId = item.get("territoryId") if territory == territoryId: self.__data["PostalCode"] = {"CODE": item.text} break # Supplemental Data path = os.path.join(supplemental, "supplementalData.xml") Console.info("Processing %s..." % os.path.relpath(path, CLDR_DIR)) tree = xml.etree.ElementTree.parse(path) # :: Calendar Preference ordering = None for item in tree.findall("calendarPreferenceData/calendarPreference"): if item.get("territories") == "001" and ordering == None: ordering = item.get("ordering") elif territory in item.get("territories").split(" "): ordering = item.get("ordering") break self.__data["CalendarPref"] = {"ORDERING": ordering.split(" ")} # :: Week Data self.__data["Week"] = {} weekData = tree.find("weekData") for key in ["firstDay", "weekendStart", "weekendEnd"]: day = None for item in weekData.findall(key): if item.get("territories") == "001" and day == None: day = item.get("day") elif territory in item.get("territories").split(" "): day = item.get("day") break self.__data["Week"][camelCaseToUpper(key)] = day # :: Measurement System self.__data["Measurement"] = {} measurementData = tree.find("measurementData") for key in ["measurementSystem", "paperSize"]: mtype = None for item in measurementData.findall(key): if item.get("territories") == "001" and mtype == None: mtype = item.get("type") elif territory in item.get("territories").split(" "): mtype = item.get("type") break self.__data["Measurement"][camelCaseToUpper(key)] = mtype
def __processAnimations(self): """Processes jasyanimation.json files to merge animation data into asset registry""" assets = self.__assets configs = [ fileId for fileId in assets if assets[fileId].isImageAnimationConfig() ] if configs: Console.info("Processing %s image animation configs...", len(configs)) Console.indent() for fileId in configs: Console.debug("Processing %s...", fileId) asset = assets[fileId] base = os.path.dirname(fileId) try: config = json.loads(asset.getText()) except ValueError as err: raise UserError( "Could not parse jasyanimation.json at %s: %s" % (fileId, err)) for relPath in config: imageId = "%s/%s" % (base, relPath) data = config[relPath] if not imageId in assets: raise UserError("Unknown asset %s in %s" % (imageId, fileId)) animationAsset = assets[imageId] if "rows" in data or "columns" in data: rows = Util.getKey(data, "rows", 1) columns = Util.getKey(data, "columns", 1) frames = Util.getKey(data, "frames") animationAsset.addImageAnimationData(columns, rows, frames) if frames is None: frames = rows * columns elif "layout" in data: layout = data["layout"] animationAsset.addImageAnimationData(None, None, layout=layout) frames = len(layout) else: raise UserError("Invalid image frame data for: %s" % imageId) Console.debug(" - Animation %s has %s frames", imageId, frames) Console.debug(" - Deleting animation config from assets: %s", fileId) del assets[fileId] Console.outdent()
def checkSingleInstallation(keys, versions, packageName, minVersion, installPath, updatePath): Console.info('%s:' % packageName) Console.indent() if packageName.lower() in keys: if LooseVersion(minVersion) > LooseVersion("0.0"): if LooseVersion(versions[packageName.lower()]) >= LooseVersion( minVersion): Console.info( Console.colorize( 'Version is OK (required: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "green")) else: Console.info( Console.colorize( Console.colorize( 'Version installed is too old (required: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "red"), "bold")) Console.info( 'Update to the newest version of %s using %s' % (packageName, updatePath)) else: Console.info(Console.colorize('Found installation', "green")) else: Console.info( Console.colorize( Console.colorize('Did NOT find installation', "red"), "bold")) Console.info('Install the newest version of %s using %s' % (packageName, installPath)) Console.outdent()
def addImageDimensionData(self, width, height): Console.debug("Adding dimension data for %s: %sx%s", self.id, width, height) self.__imageDimensionData = [width, height]
def optimize(node, contextId=""): Console.debug("Crypting private fields...") Console.indent() coll = __search(node) repl = {} for name in coll: repl[name] = "__%s" % __encode("%s.%s" % (contextId, name[2:])) Console.debug("Replacing private field %s with %s (context: %s)", name, repl[name], contextId) Console.debug("Found %s private fields" % len(repl)) modified, reduction = __replace(node, repl) Console.debug("Reduced size by %s bytes" % reduction) Console.outdent() return modified
def __init__(self, filename): try: self.fp = open(filename, "rb") except IOError as err: Console.error("Could not open file: %s" % filename) raise err
def requestUrl(url, content_type="text/plain", headers=None, method="GET", port=None, body="", user=None, password=None): """Generic HTTP request wrapper with support for basic authentification and automatic parsing of response content""" Console.info("Opening %s request to %s..." % (method, url)) parsed = urllib.parse.urlparse(url) if parsed.scheme == "http": request = http.client.HTTPConnection(parsed.netloc) elif parsed.scheme == "https": request = http.client.HTTPSConnection(parsed.netloc) else: raise Exception("Unsupported url: %s" % url) if parsed.query: request.putrequest(method, parsed.path + "?" + parsed.query) else: request.putrequest(method, parsed.path) request.putheader("Content-Type", content_type) request.putheader("Content-Length", str(len(body))) if user is not None and password is not None: auth = "Basic %s" % base64.b64encode( ("%s:%s" % (user, password)).encode("utf-8")).decode("utf-8") request.putheader("Authorization", auth) request.endheaders() if body: Console.info("Sending data (%s bytes)..." % len(body)) else: Console.info("Sending request...") Console.indent() request.send(body) response = request.getresponse() res_code = int(response.getcode()) res_headers = dict(response.getheaders()) res_content = response.read() res_success = False if res_code >= 200 and res_code <= 300: Console.debug("HTTP Success!") res_success = True else: Console.error("HTTP Failure Code: %s!", res_code) if "Content-Type" in res_headers: res_type = res_headers["Content-Type"] if ";" in res_type: res_type = res_type.split(";")[0] if res_type in ("application/json", "text/html", "text/plain"): res_content = res_content.decode("utf-8") if res_type == "application/json": res_content = json.loads(res_content) if "error" in res_content: Console.error("Error %s: %s", res_content["error"], res_content["reason"]) elif "reason" in res_content: Console.info("Success: %s" % res_content["reason"]) Console.outdent() return res_success, res_headers, res_content
def __extend(node): """ Finds extend requests for mixins aka. - mixins calls without params - simple variables in a block For all found extend requests it detects the flattened selector and appends the selector section of the extendable mixin accordingly. After that it removes the original mixin request. """ modified = 0 for child in reversed(list(node)): # Ignore all mixin declarations. Can't operate inside them. # For these things to work we have to wait for the include mechanics to resolve them first # (which actually just remove these mixin declarations though) if child is not None: modified += __extend(child) if isExtendCall(node): name = node.name Console.debug("Extend request to mixin %s at: %s", name, node.line) Console.indent() mixin = __findMixin(node.parent, name) if not mixin: raise Exception( "Could not find mixin %s as required by extend request at line %s" % (node.name, node.line)) Console.debug("Found matching mixin declaration at line: %s", mixin.line) selector, media, supports = Util.combineSelector(node.parent, stop=mixin.parent) # There is no possibility to handle this in a series of CSS selectors. This is why # we have to use an include like approach instead of extend to correctly deal # with the situation. This should work well, but is not as efficient regarding # output file size. if media or supports: Console.warn( "Extending inside a @media/@support structure behaves like including (larger result size): %s %s + %s", media, supports, ", ".join(selector)) replacements = __resolveMixin(mixin, None) Console.debug( "Replacing call %s at line %s with mixin from line %s" % (name, node.line, replacements.line)) # Reverse inject all children of that block # at the same position as the original call parent = node.parent pos = parent.index(node) parent.insertAll(pos, replacements) elif selector: Console.debug("Extending selector of mixin by: %s", ", ".join(selector)) if hasattr(mixin, "selector"): # We iterate from in inverse mode, so add new selectors to the front mixin.selector[0:0] = selector else: mixin.selector = selector virtualBlock = Node.Node(type="block") __extendContent(mixin.rules, node, virtualBlock, mixin) if len(virtualBlock) > 0: callSelector, callMedia, callSupports = Util.combineSelector( node) if callSelector: virtualSelector = Node.Node(type="selector") virtualSelector.name = callSelector if callMedia: virtualMedia = Node.Node(type="media") virtualMedia.name = callMedia if callSupports: virtualSupports = Node.Node(type="supports") virtualSupports.name = callSupports if callSelector: virtualSelector.append(virtualBlock, "rules") elif callMedia: virtualMedia.append(virtualBlock, "rules") elif callSupports: virtualSupports.append(virtualBlock, "rules") if callSupports: virtualTop = virtualSupports elif callMedia: virtualTop = virtualMedia elif callSelector: virtualTop = virtualSelector pos = mixin.parent.index(mixin) mixin.parent.insert(pos + 1, virtualTop) node.parent.remove(node) Console.outdent() modified += 1 return modified