def __load_level(self, lvl): ''' For the given level, create a help table. ''' if wcache.is_cached("lvl_help_tab_%s" % lvl): return wcache.retrieve("lvl_help_tab_%s" % lvl) if not self.key_pos and not self.load_index(): return None self.help_f = self.open_file(self.help_text_file, "r") if not self.help_f: return None lvl_s = "cmdhelp_%s" % lvl if not lvl_s in self.leveld: if not self.bad_index: common_warn("help table for level %s not found" % lvl) self.drop_index() return None common_debug("loading help table for level %s" % lvl) help_tab = odict() help_tab["."] = self.__load_help_one(lvl_s) try: for key in self.leveld[lvl_s]: cmd = key[len(lvl_s) + 1:] help_tab[cmd] = self.__load_help_one(key) except: pass self.help_f.close() help_tab["quit"] = ("exit the program", "") help_tab["help"] = ("show help", "") help_tab["end"] = ("go back one level", "") return help_tab
def mk_index(self): ''' Prepare an index file, sorted by topic, with seek positions Do we need a hash on content? ''' if self.no_help_file: return False crm_help_v = os.getenv("CRM_HELP_FILE") if crm_help_v: self.help_text_file = crm_help_v help_f = self.open_file(self.help_text_file, "r") if not help_f: return False idx_f = self.open_file(self.index_file, "w") if not idx_f: return False common_debug("building help index") key_pos = odict() while 1: pos = help_f.tell() s = help_f.readline() if not s: break if s.startswith("[["): r = re.search(r'..([^,]+),', s) if r: key_pos[r.group(1)] = pos help_f.close() for key in key_pos: print >> idx_f, '%s %d' % (key, key_pos[key]) idx_f.close() return True
def __load_level(self,lvl): ''' For the given level, create a help table. ''' if wcache.is_cached("lvl_help_tab_%s" % lvl): return wcache.retrieve("lvl_help_tab_%s" % lvl) if not self.key_pos and not self.load_index(): return None self.help_f = self.open_file(self.help_text_file,"r") if not self.help_f: return None lvl_s = "cmdhelp_%s" % lvl if not lvl_s in self.leveld: if not self.bad_index: common_warn("help table for level %s not found" % lvl) self.drop_index() return None common_debug("loading help table for level %s" % lvl) help_tab = odict() help_tab["."] = self.__load_help_one(lvl_s) try: for key in self.leveld[lvl_s]: cmd = key[len(lvl_s)+1:] help_tab[cmd] = self.__load_help_one(key) except: pass self.help_f.close() help_tab["quit"] = ("exit the program", "") help_tab["help"] = ("show help", "") help_tab["end"] = ("go back one level", "") return help_tab
def mk_index(self): ''' Prepare an index file, sorted by topic, with seek positions Do we need a hash on content? ''' if self.no_help_file: return False crm_help_v = os.getenv("CRM_HELP_FILE") if crm_help_v: self.help_text_file = crm_help_v help_f = self.open_file(self.help_text_file,"r") if not help_f: return False idx_f = self.open_file(self.index_file,"w") if not idx_f: return False common_debug("building help index") key_pos = odict() while 1: pos = help_f.tell() s = help_f.readline() if not s: break if s.startswith("[["): r = re.search(r'..([^,]+),', s) if r: key_pos[r.group(1)] = pos help_f.close() for key in key_pos: print >>idx_f, '%s %d' % (key,key_pos[key]) idx_f.close() return True
def attrs(self): if '(' == self.input[0]: index = self.indexOfDelimiters('(',')') string = self.input[1:index] tok = self.tok('attrs') l = len(string) colons = self.colons states = ['key'] class Namespace: key = u'' val = u'' quote = u'' literal = True def reset(self): self.key = self.val = self.quote = u'' self.literal = True def __str__(self): return dict(key=self.key,val=self.val,quote=self.quote,literal=self.literal).__str__() ns = Namespace() def state(): return states[-1] def interpolate(attr): attr, num = self.RE_ATTR_INTERPOLATE.subn(lambda matchobj:'%s+%s.__str__()+%s'%(ns.quote,matchobj.group(1),ns.quote),attr) return attr, (num>0) self.consume(index+1) from utils import odict tok.attrs = odict() tok.static_attrs = set() str_nums = map(str,range(10)) def parse(c): real = c if colons and ':'==c: c = '=' ns.literal = ns.literal and (state() not in ('object','array','expr')) if c in (',','\n'): s = state() if s in ('expr','array','string','object'): ns.val += c else: states.append('key') ns.val = ns.val.strip() ns.key = ns.key.strip() if not ns.key: return # ns.literal = ns.quote if not ns.literal: if '!'==ns.key[-1]: ns.literal = True ns.key = ns.key[:-1] ns.key = ns.key.strip("'\"") if not ns.val: tok.attrs[ns.key] = True else: tok.attrs[ns.key], is_interpolated = interpolate(ns.val) ns.literal = ns.literal and not is_interpolated if ns.literal: tok.static_attrs.add(ns.key) ns.reset() elif '=' == c: s = state() if s == 'key char': ns.key += real elif s in ('val','expr','array','string','object'): ns.val+= real else: states.append('val') elif '(' == c: if state() in ('val','expr'): states.append('expr') ns.val+=c elif ')' == c: if state() in ('val','expr'): states.pop() ns.val+=c elif '{' == c: if 'val'==state(): states.append('object') ns.val+=c elif '}' == c: if 'object'==state(): states.pop() ns.val+=c elif '[' == c: if 'val'==state(): states.append('array') ns.val+=c elif ']' == c: if 'array'==state(): states.pop() ns.val+=c elif c in ('"',"'"): s = state() if 'key'==s: states.append('key char') elif 'key char'==s: states.pop() elif 'string'==s: if c==ns.quote: states.pop() ns.val +=c else: states.append('string') ns.val +=c ns.quote = c elif ''== c: pass else: s = state() ns.literal = ns.literal and (s in ('key','string') or c in str_nums) # print c, s, ns.literal if s in ('key','key char'): ns.key += c else: ns.val += c for char in string: parse(char) parse(',') return tok
def StartElementHandler(self, tag, attributes): attrs = utils.odict() while attributes: key = attributes.pop(0) value = attributes.pop(0) attrs[key] = value # update prefix to namespace mapping if self.root is None: self.index = self.expat.CurrentByteIndex nsmap = utils.odict() else: nsmap = self.root.nsmap.copy() # process namespace declarations for key, value in attrs.items(): if key.startswith('xmlns:'): prefix, name = key.split(':') nsmap[name] = value del attrs[key] for key, value in attrs.items(): try: prefix, name = key.split(':') except (ValueError, TypeError): continue del attrs[key] namespace = nsmap.get(prefix) if namespace is None: if self.root is not None: element = self.element while element is not None: namespace = element.nsmap.get(prefix) if namespace is not None: nsmap[prefix] = namespace break element = element.getparent() if namespace is None: try: namespace = config.DEFAULT_NS_MAP[prefix] except KeyError: raise KeyError( "Attribute prefix unknown: '%s'." % prefix) attrs['{%s}%s' % (namespace, name)] = value # process tag try: prefix, name = tag.split(':') namespace = nsmap.get(prefix) or config.DEFAULT_NS_MAP[prefix] tag = '{%s}%s' % (namespace, name) except ValueError: pass # create element using parser element = self.parser.makeelement(tag, attrs, nsmap=nsmap) element.position = (self.expat.CurrentLineNumber, self.expat.CurrentColumnNumber) if self.root is None: # cook doctype doctype = [] if (self._doctype is not None and self._doctype["pubid"] is None and self._doctype["sysid"] is None and self._doctype["has_internal_subset"] and self._doctype["entities"]): doctype.append("<!DOCTYPE %(doctype_name)s [" % self._doctype) for entity, repl in self._doctype["entities"]: doctype.append('<!ENTITY %s "%s">' % ( entity, repl.encode("ascii", "xmlcharrefreplace"))) doctype.append("]>") elif self._doctype: d = self._doctype if d['pubid'] is None or d['sysid'] is None: doctype.append( '<!DOCTYPE %(doctype_name)s>' % d) else: doctype.append( '<!DOCTYPE %(doctype_name)s PUBLIC ' '"%(pubid)s" "%(sysid)s">' % d) self.doctype = "\n".join(doctype).encode("utf-8") ElementTree( self.parser, element, self.xml_version, self.encoding, self.standalone, self.doctype) # set this element as tree root self.root = element else: self.element.append(element) # validate element self._validate(element) # set as current element self.element = element
def serialize(self): """Serialize element into clause-statements.""" _ = [] # i18n domain if self.translation_domain is not None: _.append(clauses.Define( self.symbols.domain, types.value(repr(self.translation_domain)))) # variable definitions adding = set() if self.define is not None: # as an optimization, we only define the `default` # symbol, if it's present in the definition clause (as # string representation) if self.symbols.default in repr(self.define): default = types.value(self.symbols.default_marker_symbol) _.append(clauses.Assign(default, self.symbols.default)) for declaration, expression in self.define: if self.symbols.remote_scope in self.stream.scope[1]: define = clauses.Define( declaration, expression, self.symbols.remote_scope) else: define = clauses.Define(declaration, expression) for name in define.declaration: adding.add(name) _.append(define) # tag tail (deferred) tail = self.tail if self.fill_slot is None and self.translation_name is None: for part in reversed(tail): if isinstance(part, types.expression): _.append(clauses.Write(part, defer=True)) else: _.append(clauses.Out(part, defer=True)) # macro method macro = self.macro if macro is not None: if self.symbols.remote_scope in self.stream.scope[1]: dictionary = self.symbols.remote_scope else: dictionary = self.symbols.scope exclude = set(( self.symbols.scope, self.symbols.slots)) | \ self.stream.scope[0] | set(macro.args) scope = set(itertools.chain(*self.stream.scope[1:])) | set(( self.symbols.out, self.symbols.write)) args = tuple(macro.args) + tuple( "%s=%s" % (name, name) for name in scope if name not in exclude) _.append(clauses.Method( macro.name, args, decorators=macro.decorators, dictionary=dictionary)) # condition if self.condition is not None: _.append(clauses.Condition(self.condition)) # repeat if self.repeat is not None: variables, expression = self.repeat newline = True for element in self.element.walk(): node = element.node if node and node.omit is False: break else: newline = False if len(variables) > 1: repeat = clauses.Repeat( variables, expression, repeatdict=False, newline=newline) else: repeat = clauses.Repeat(variables, expression, newline=newline) _.append(repeat) # assign if self.assign is not None: for declaration, expression in self.assign: if len(declaration) != 1: raise ValueError("Can only assign single variable.") variable = declaration[0] _.append(clauses.Assign(expression, variable)) content = self.content omit = self.omit if self.define_slot: name = self.define_slot scope = set(itertools.chain(*self.stream.scope[1:])) # assemble arguments that we pass into the macro # fill-slot callback exclude = set((self.symbols.scope, self.symbols.slots, self.symbols.out, self.symbols.write)).union(self.stream.scope[0]) scope_args = tuple(variable for variable in scope if variable not in exclude) # look up fill-slot value _.append(clauses.Assign( types.template('%%(slots)s.get(%s)' % repr(name)), self.symbols.tmp)) # if slot has been filled, either use it as is (if # it's a string), or pass in required arguments (if # it's a callback function) _.append(clauses.Condition( types.template('%(tmp)s is not None'), (clauses.Condition( types.template('isinstance(%(tmp)s, basestring)'), (clauses.Slot( types.template("%(tmp)s(%(scope)s)"), scope_args),), finalize=True, invert=True), clauses.Else(( clauses.Write(types.template("%(tmp)s")),)) ))) _.append(clauses.Else()) # set dynamic content flag dynamic = content or self.translate is not None # if an attribute ordering is required, setting a default # trivial value for each attribute will ensure that the order # is preserved attributes = utils.odict() if self.attribute_ordering is not None: for name in self.attribute_ordering: attributes[name] = None # static attributes (including those with a namespace prefix) # are at the bottom of the food chain attributes.update(self.static_attributes) attributes.update(self.ns_attributes) # dynamic attributes dynamic_attrs = self.dynamic_attributes or () dynamic_attr_names = [] for variables, expression in dynamic_attrs: if len(variables) != 1: raise ValueError("Tuple definitions in assignment clause " "is not supported.") variable = variables[0] attributes[variable] = expression dynamic_attr_names.append(variable) # translated attributes translated_attributes = self.translated_attributes or () for variable, msgid in translated_attributes: if msgid: if variable in dynamic_attr_names: raise ValueError( "Message id not allowed in conjunction with " "a dynamic attribute.") value = types.value('"%s"' % msgid) if variable in attributes: default = repr(attributes[variable]) expression = clauses.translate_expression(value, default=default) else: expression = clauses.translate_expression(value) else: value = attributes.get(variable) if value is not None: if variable not in dynamic_attr_names: value = repr(value) expression = clauses.translate_expression(value) else: raise ValueError("Must be either static or dynamic " "attribute when no message id " "is supplied.") attributes[variable] = expression # tag text = self.text if omit is not True: _.append(clauses.Attrs(self.static_attributes, "_attrs_%d" % id(self.element))) selfclosing = not text and not dynamic and len(self.element) == 0 tag = clauses.Tag( self.tag, attributes, expression=self.dict_attributes, selfclosing=selfclosing, cdata=self.cdata is not None, defaults=self.static_attributes) if omit: _.append(clauses.Condition( omit, [tag], finalize=False, invert=True)) else: _.append(tag) # tag text (if we're not replacing tag body) if len(text) and not dynamic and not self.use_macro and not self.extend_macro: for part in text: if isinstance(part, types.expression): _.append(clauses.Write(part)) else: _.append(clauses.Out(part)) # dynamic content if content: msgid = self.translate if msgid is not None: if msgid: raise ValueError( "Can't use message id with dynamic content translation.") _.append(clauses.Assign(content, self.symbols.tmp)) content = clauses.translate_expression( types.value(self.symbols.tmp)) else: value = types.value(repr(utils.serialize(self.element, omit=True))) _.insert(0, clauses.Assign( value, "%s.value = %s" % ( self.symbols.default_marker_symbol, self.symbols.default))) _.append(clauses.Write(content)) # dynamic text elif self.translate is not None and \ True in map(lambda part: isinstance(part, types.expression), text): if len(self.element): raise ValueError( "Can't translate dynamic text block with elements in it.") init_stream = types.value('_init_stream()') init_stream.symbol_mapping['_init_stream'] = generation.initialize_stream subclauses = [] subclauses.append(clauses.Define( types.declaration((self.symbols.out, self.symbols.write)), init_stream)) for part in text: if isinstance(part, types.expression): subclauses.append(clauses.Write(part)) else: part = ' '.join(part.split()) if part != "": subclauses.append(clauses.Out(part)) # attempt translation subclauses.append(clauses.Assign( clauses.translate_expression( types.value('%(out)s.getvalue()'), default=None), self.symbols.tmp)) _.append(clauses.Group(subclauses)) _.append(clauses.Write(types.value(self.symbols.tmp))) # include elif self.include: # compute macro function arguments and create argument string arguments = [ "%s=%s" % (arg, arg) for arg in \ set(itertools.chain(*self.stream.scope[1:]))] # XInclude's are similar to METAL macros, except the macro # is always defined as the entire template. # first we compute the filename expression and write it to # an internal variable _.append(clauses.Assign(self.include, self.symbols.include)) # call template _.append(clauses.Write( types.template( "%%(xincludes)s.get(%%(include)s, %s).render_xinclude(%s)" % \ (repr(self.format), ", ".join(arguments))))) # use or extend macro elif self.use_macro or self.extend_macro: # assign macro value to variable macro = self.use_macro or self.extend_macro _.append(clauses.Assign(macro, self.symbols.metal)) # for each fill-slot element, create a new output stream # and save value in a temporary variable kwargs = [] callbacks = {} # determine variable scope scope = set(itertools.chain(adding, *self.stream.scope[1:])) - \ self.stream.scope[0] # we pass in all variables from the current scope (as # keyword arguments, to allow first use before potential # reassignment) callback_args = ", ".join( "%s=%s" % (arg, arg) for arg in scope if arg not in (self.symbols.slots, self.symbols.scope)) macro_args = ", ".join( "%s=%s" % (arg, arg) for arg in scope if arg not in (self.symbols.slots,)) # loop through macro fill slot elements and generate # callback methods; the reason why we use callbacks is # convenience: it's an easy fit with the compiler elements = [element for element in self.element.walk() if element.node and element.node.fill_slot] for element in elements: # make sure we're not in a nested macro block parent = element.getparent() while parent is not self.element: if parent.node.use_macro or parent.node.extend_macro: element = None break parent = parent.getparent() if element is None: continue # determine and register callback name name = element.node.fill_slot callbacks[name] = callback = "%s_%s" % ( self.symbols.callback, utils.normalize_slot_name(name)) # pass in remote scope to callback method; this is # done because macros may add global variables to the # scope, which should be made available to the calling # template visitor = clauses.Visit(element.node) tail = element.tail newline = tail and '\n' in tail _.append(clauses.Callback( callback, visitor, callback_args, newline)) # if we're extending the macro, the current slots will be # carried over to the macro extend = self.extend_macro is not None defines = set() if extend: for element in self.element.walk(): if element.node is not None: define_slot = element.node.define_slot if define_slot is not None: defines.add(define_slot) # format slot arguments slot_args = ", ".join("'%s': %s" % kwarg for kwarg in callbacks.items()) _.append(clauses.Macro( types.value("{%s}" % slot_args), macro_args, extend=extend, extend_except=defines, label=macro.label )) # translate body elif self.translate is not None: msgid = self.translate # subelements are either named or unnamed; if there are # unnamed elements, the message id must be dynamic named_elements = [e for e in self.element if e.node.translation_name] unnamed_elements = [e for e in self.element if not e.node.translation_name] if not msgid and named_elements and not unnamed_elements: msgid = self.create_msgid() elements = named_elements else: elements = self.element if msgid and not named_elements: elements = () if named_elements: mapping = "%s_%d" % (self.symbols.mapping, id(self.element)) _.append(clauses.Assign(types.value('{}'), mapping)) else: mapping = 'None' if unnamed_elements or not msgid: text = utils.htmlescape(self.element.text.replace('%', '%') or "") _.append(clauses.Assign(types.value(repr(text)), self.symbols.msgid)) # for each named block, create a new output stream # and use the value in the translation mapping dict for element in elements: init_stream = types.value('_init_stream()') init_stream.symbol_mapping[ '_init_stream'] = generation.initialize_stream subclauses = [] subclauses.append(clauses.Define( types.declaration((self.symbols.out, self.symbols.write)), init_stream)) subclauses.append(clauses.Visit(element.node)) # if the element is named, record it in the mapping if element in named_elements: name = element.node.translation_name subclauses.append(clauses.Assign( types.template('%(out)s.getvalue()'), "%s['%s']" % (mapping, name))) # when computing a dynamic message id, add a # reference to the named block if not msgid: if not unnamed_elements: subclauses.append(clauses.Assign( types.value(repr("${%s}" % name)), self.symbols.msgid)) else: subclauses.append(clauses.Assign( types.template('%(msgid)s + ' + repr("${%s}" % name) + ' + ' + repr(element.tail)), self.symbols.msgid)) # else add it to the dynamic message id else: subclauses.append(clauses.Assign( types.template('%(msgid)s + %(out)s.getvalue()'), self.symbols.msgid)) # XXX: note that this should read: # _.append(clauses.Group(subclauses)) # # but there's a problem with multiple temporary # variable assignments within the same block; this is # just an easy work-around _.append(clauses.Condition( types.value('True'), subclauses, finalize=True)) if msgid: value = types.value(repr(msgid)).replace('%', '%%') default = self.symbols.marker else: default = types.template('%(msgid)s') value = types.template("' '.join(%(msgid)s.split())") _.append(clauses.Assign( clauses.translate_expression( value, mapping=mapping, default=default), self.symbols.result)) # write translation to output if successful, otherwise # fallback to default rendition; result = types.value(self.symbols.result) result.symbol_mapping[self.symbols.marker] = i18n.marker if msgid: condition = types.template('%(result)s is not %(marker)s') _.append(clauses.Condition( condition, [clauses.UnicodeWrite(result)], finalize=True)) subclauses = [] if self.element.text: subclauses.append(clauses.Out( utils.htmlescape(self.element.text))) for element in self.element: name = element.node.translation_name if name: value = types.value("%s['%s']" % (mapping, name)) subclauses.append(clauses.UnicodeWrite(value)) for part in reversed(element.node.tail): if isinstance(part, types.expression): subclauses.append(clauses.Write(part)) else: subclauses.append(clauses.Out( utils.htmlescape(part))) else: subclauses.append(clauses.Visit(element.node)) if subclauses: _.append(clauses.Else(subclauses)) else: _.append(clauses.UnicodeWrite(result)) return _
def attrs(self): if "(" == self.input[0]: index = self.indexOfDelimiters("(", ")") str = self.input[1:index] tok = self.tok("attrs") l = len(str) colons = self.colons states = ["key"] class Namespace: key = "" val = "" quote = "" def reset(self): self.key = self.val = self.quote = "" ns = Namespace() def state(): return states[-1] def interpolate(attr): return self.RE_ATTR_INTERPOLATE.sub( lambda matchobj: "%s+%s.__str__()+%s" % (ns.quote, matchobj.group(1), ns.quote), attr ) self.consume(index + 1) tok.attrs = odict() tok.static_attrs = set() def parse(c): real = c if colons and ":" == c: c = "=" if c in (",", "\n"): s = state() if s in ("expr", "array", "string", "object"): ns.val += c else: states.append("key") ns.val = ns.val.strip() ns.key = ns.key.strip() if not ns.key: return ns.key = ns.key.strip("'\"") if ns.quote: tok.static_attrs.add(ns.key) elif ns.key in tok.static_attrs: tok.static_attrs.remove(ns.key) tok.attrs[ns.key] = True if not ns.val else interpolate(ns.val) ns.reset() elif "=" == c: s = state() if s == "key char": ns.key += real elif s in ("val", "expr", "array", "string", "object"): ns.val += real else: states.append("val") elif "(" == c: if state() in ("val", "expr"): states.append("expr") ns.val += c elif ")" == c: if state() in ("val", "expr"): states.pop() ns.val += c elif "{" == c: if "val" == state(): states.append("object") ns.val += c elif "}" == c: if "object" == state(): states.pop() ns.val += c elif "[" == c: if "val" == state(): states.append("array") ns.val += c elif "]" == c: if "array" == state(): states.pop() ns.val += c elif c in ('"', "'"): s = state() if "key" == s: states.push("key char") elif "key char" == s: states.pop() elif "string" == s: if c == ns.quote: states.pop() ns.val += c else: states.append("string") ns.val += c ns.quote = c elif "" == c: pass else: s = state() if s in ("key", "key char"): ns.key += c else: ns.val += c for char in str: parse(char) parse(",") return tok
def attrs(self): if '(' == self.input[0]: index = self.indexOfDelimiters('(',')') str = self.input[1:index] tok = self.tok('attrs') l = len(str) colons = self.colons states = ['key'] class Namespace: key = '' val = '' quote = '' def reset(self): self.key = self.val = self.quote = '' ns = Namespace() def state(): return states[-1] def interpolate(attr): return self.RE_ATTR_INTERPOLATE.sub(lambda matchobj:'%s+%s.__str__()+%s'%(ns.quote,matchobj.group(1),ns.quote),attr) self.consume(index+1) tok.attrs = odict() tok.static_attrs = set() def parse(c): real = c if colons and ':'==c: c = '=' if c in (',','\n'): s = state() if s in ('expr','array','string','object'): ns.val += c else: states.append('key') ns.val = ns.val.strip() ns.key = ns.key.strip() if not ns.key: return ns.key = ns.key.strip("'\"") if ns.quote: tok.static_attrs.add(ns.key) elif ns.key in tok.static_attrs: tok.static_attrs.remove(ns.key) tok.attrs[ns.key] = True if not ns.val else interpolate(ns.val) ns.reset() elif '=' == c: s = state() if s == 'key char': ns.key += real elif s in ('val','expr','array','string','object'): ns.val+= real else: states.append('val') elif '(' == c: if state() in ('val','expr'): states.append('expr') ns.val+=c elif ')' == c: if state() in ('val','expr'): states.pop() ns.val+=c elif '{' == c: if 'val'==state(): states.append('object') ns.val+=c elif '}' == c: if 'object'==state(): states.pop() ns.val+=c elif '[' == c: if 'val'==state(): states.append('array') ns.val+=c elif ']' == c: if 'array'==state(): states.pop() ns.val+=c elif c in ('"',"'"): s = state() if 'key'==s: states.append('key char') elif 'key char'==s: states.pop() elif 'string'==s: if c==ns.quote: states.pop() ns.val +=c else: states.append('string') ns.val +=c ns.quote = c elif ''== c: pass else: s = state() if s in ('key','key char'): ns.key += c else: ns.val += c for char in str: parse(char) parse(',') return tok
def attrs(self): if '(' == self.input[0]: index = self.indexOfDelimiters('(',')') str = self.input[1:index] tok = self.tok('attrs') l = len(str) colons = self.colons states = ['key'] class Namespace: key = '' val = '' quote = '' ns = Namespace() def state(): return states[-1] def interpolate(attr): return self.RE_ATTR_INTERPOLATE.sub(lambda matchobj:'%s+%s.__str__()+%s'%(ns.quote,matchobj.group(1),ns.quote),attr) self.consume(index+1) tok.attrs = odict() def parse(c): real = c if colons and ':'==c: c = '=' if c in (',','\n'): s = state() if s in ('expr','array','string','object'): ns.val += c else: states.append('key') ns.val = ns.val.strip() ns.key = ns.key.strip() if not ns.key: return ns.key = ns.key.strip("'\"") tok.attrs[ns.key] = True if not ns.val else interpolate(ns.val) ns.key = ns.val = ''; elif '=' == c: s = state() if s == 'key char': ns.key += real elif s in ('val','expr','array','string','object'): ns.val+= real else: states.append('val') elif '(' == c: if state() in ('val','expr'): states.append('expr') ns.val+=c elif ')' == c: if state() in ('val','expr'): states.pop() ns.val+=c elif '{' == c: if 'val'==state(): states.append('object') ns.val+=c elif '}' == c: if 'object'==state(): states.pop() ns.val+=c elif '[' == c: if 'val'==state(): states.append('array') ns.val+=c elif ']' == c: if 'array'==state(): states.pop() ns.val+=c elif c in ('"',"'"): s = state() if 'key'==s: states.push('key char') elif 'key char'==s: states.pop() elif 'string'==s: if c==ns.quote: states.pop() ns.val +=c else: states.append('string') ns.val +=c ns.quote = c elif ''== c: pass else: s = state() if s in ('key','key char'): ns.key += c else: ns.val += c for char in str: parse(char) parse(',') return tok
def attrs(self): if '(' == self.input[0]: index = self.indexOfDelimiters('(', ')') string = self.input[1:index] tok = self.tok('attrs') l = len(string) colons = self.colons states = ['key'] class Namespace: key = u'' val = u'' quote = u'' literal = True def reset(self): self.key = self.val = self.quote = u'' self.literal = True def __str__(self): return dict(key=self.key, val=self.val, quote=self.quote, literal=self.literal).__str__() ns = Namespace() def state(): return states[-1] def interpolate(attr): attr, num = self.RE_ATTR_INTERPOLATE.subn( lambda matchobj: '%s+%s+%s' % (ns.quote, matchobj.group(1), ns.quote), attr) return attr, (num > 0) self.consume(index + 1) from utils import odict tok.attrs = odict() tok.static_attrs = set() str_nums = map(str, range(10)) def parse(c): real = c if colons and ':' == c: c = '=' ns.literal = ns.literal and (state() not in ('object', 'array', 'expr')) if c in (',', '\n'): s = state() if s in ('expr', 'array', 'string', 'object'): ns.val += c else: states.append('key') ns.val = ns.val.strip() ns.key = ns.key.strip() if not ns.key: return # ns.literal = ns.quote if not ns.literal: if '!' == ns.key[-1]: ns.literal = True ns.key = ns.key[:-1] ns.key = ns.key.strip("'\"") if not ns.val: tok.attrs[ns.key] = True else: tok.attrs[ns.key], is_interpolated = interpolate( ns.val) ns.literal = ns.literal and not is_interpolated if ns.literal: tok.static_attrs.add(ns.key) ns.reset() elif '=' == c: s = state() if s == 'key char': ns.key += real elif s in ('val', 'expr', 'array', 'string', 'object'): ns.val += real else: states.append('val') elif '(' == c: if state() in ('val', 'expr'): states.append('expr') ns.val += c elif ')' == c: if state() in ('val', 'expr'): states.pop() ns.val += c elif '{' == c: if 'val' == state(): states.append('object') ns.val += c elif '}' == c: if 'object' == state(): states.pop() ns.val += c elif '[' == c: if 'val' == state(): states.append('array') ns.val += c elif ']' == c: if 'array' == state(): states.pop() ns.val += c elif c in ('"', "'"): s = state() if 'key' == s: states.append('key char') elif 'key char' == s: states.pop() elif 'string' == s: if c == ns.quote: states.pop() ns.val += c else: states.append('string') ns.val += c ns.quote = c elif '' == c: pass else: s = state() ns.literal = ns.literal and (s in ('key', 'string') or c in str_nums) # print c, s, ns.literal if s in ('key', 'key char'): ns.key += c else: ns.val += c for char in string: parse(char) parse(',') return tok
def StartElementHandler(self, tag, attributes): attrs = utils.odict() while attributes: key = attributes.pop(0) value = attributes.pop(0) attrs[key] = value # update prefix to namespace mapping if self.root is None: self.index = self.expat.CurrentByteIndex nsmap = utils.odict() else: nsmap = self.root.nsmap.copy() # process namespace declarations for key, value in attrs.items(): if key.startswith('xmlns:'): prefix, name = key.split(':') nsmap[name] = value del attrs[key] for key, value in attrs.items(): try: prefix, name = key.split(':') except (ValueError, TypeError): continue del attrs[key] namespace = nsmap.get(prefix) if namespace is None: if self.root is not None: element = self.element while element is not None: namespace = element.nsmap.get(prefix) if namespace is not None: nsmap[prefix] = namespace break element = element.getparent() if namespace is None: try: namespace = config.DEFAULT_NS_MAP[prefix] except KeyError: raise KeyError("Attribute prefix unknown: '%s'." % prefix) attrs['{%s}%s' % (namespace, name)] = value # process tag try: prefix, name = tag.split(':') namespace = nsmap.get(prefix) or config.DEFAULT_NS_MAP[prefix] tag = '{%s}%s' % (namespace, name) except ValueError: pass # create element using parser element = self.parser.makeelement(tag, attrs, nsmap=nsmap) element.position = (self.expat.CurrentLineNumber, self.expat.CurrentColumnNumber) if self.root is None: # cook doctype doctype = [] if (self._doctype is not None and self._doctype["pubid"] is None and self._doctype["sysid"] is None and self._doctype["has_internal_subset"] and self._doctype["entities"]): doctype.append("<!DOCTYPE %(doctype_name)s [" % self._doctype) for entity, repl in self._doctype["entities"]: doctype.append( '<!ENTITY %s "%s">' % (entity, repl.encode("ascii", "xmlcharrefreplace"))) doctype.append("]>") elif self._doctype: d = self._doctype if d['pubid'] is None or d['sysid'] is None: doctype.append('<!DOCTYPE %(doctype_name)s>' % d) else: doctype.append('<!DOCTYPE %(doctype_name)s PUBLIC ' '"%(pubid)s" "%(sysid)s">' % d) self.doctype = "\n".join(doctype).encode("utf-8") ElementTree(self.parser, element, self.xml_version, self.encoding, self.standalone, self.doctype) # set this element as tree root self.root = element else: self.element.append(element) # validate element self._validate(element) # set as current element self.element = element
def serialize(self): """Serialize element into clause-statements.""" _ = [] # i18n domain if self.translation_domain is not None: _.append( clauses.Define(self.symbols.domain, types.value(repr(self.translation_domain)))) # variable definitions adding = set() if self.define is not None: # as an optimization, we only define the `default` # symbol, if it's present in the definition clause (as # string representation) if self.symbols.default in repr(self.define): default = types.value(self.symbols.default_marker_symbol) _.append(clauses.Assign(default, self.symbols.default)) for declaration, expression in self.define: if len(expression) == 0: raise ValueError( "Must have one or more assignment values.") if self.symbols.remote_scope in self.stream.scope[1]: define = clauses.Define(declaration, expression, self.symbols.remote_scope) else: define = clauses.Define(declaration, expression) for name in define.declaration: adding.add(name) _.append(define) # tag tail (deferred) tail = self.tail if self.fill_slot is None and self.translation_name is None: for part in reversed(tail): if isinstance(part, types.expression): _.append(clauses.Write(part, defer=True)) else: _.append(clauses.Out(part, defer=True)) # macro method macro = self.macro if macro is not None: if self.symbols.remote_scope in self.stream.scope[1]: dictionary = self.symbols.remote_scope else: dictionary = self.symbols.scope exclude = set(( self.symbols.scope, self.symbols.slots)) | \ self.stream.scope[0] | set(macro.args) scope = set(itertools.chain(*self.stream.scope[1:])) | set( (self.symbols.out, self.symbols.write)) args = tuple(macro.args) + tuple("%s=%s" % (name, name) for name in scope if name not in exclude) _.append( clauses.Method(macro.name, args, decorators=macro.decorators, dictionary=dictionary)) # condition if self.condition is not None: _.append(clauses.Condition(self.condition)) content = self.content omit = self.omit if self.define_slot: name = self.define_slot scope = set(itertools.chain(*self.stream.scope[1:])) # assemble arguments that we pass into the macro # fill-slot callback exclude = set( (self.symbols.scope, self.symbols.slots, self.symbols.out, self.symbols.write)).union(self.stream.scope[0]) scope_args = tuple(variable for variable in scope if variable not in exclude) # look up fill-slot value _.append( clauses.Assign( types.template('%%(slots)s.get(%s)' % repr(name)), self.symbols.tmp)) # if slot has been filled, either use it as is (if # it's a string), or pass in required arguments (if # it's a callback function) _.append( clauses.Condition( types.template('%(tmp)s is not None'), (clauses.Condition( types.template('isinstance(%(tmp)s, basestring)'), (clauses.Slot( types.template("%(tmp)s(%(scope)s, %(repeat)s)"), scope_args), ), finalize=True, invert=True), clauses.Else( (clauses.Write(types.template("%(tmp)s")), ))))) _.append(clauses.Else()) # repeat if self.repeat is not None: variables, expression = self.repeat newline = True for element in self.element.walk(): node = element.node if node and node.omit is False: break else: newline = False if len(variables) > 1: repeat = clauses.Repeat(variables, expression, repeatdict=False, newline=newline) else: repeat = clauses.Repeat(variables, expression, newline=newline) _.append(repeat) # assign if self.assign is not None: for declaration, expression in self.assign: if len(declaration) != 1: raise ValueError("Can only assign single variable.") variable = declaration[0] _.append(clauses.Assign(expression, variable)) # set dynamic content flag dynamic = content or self.translate is not None # if an attribute ordering is required, setting a default # trivial value for each attribute will ensure that the order # is preserved attributes = utils.odict() if self.attribute_ordering is not None: for name in self.attribute_ordering: attributes[name] = None # static attributes (including those with a namespace prefix) # are at the bottom of the food chain attributes.update(self.static_attributes) attributes.update(self.ns_attributes) # dynamic attributes dynamic_attrs = self.dynamic_attributes or () dynamic_attr_names = [] for variables, expression in dynamic_attrs: if len(variables) != 1: raise ValueError("Tuple definitions in assignment clause " "is not supported.") if len(expression) == 0: raise ValueError("Must have one or more assignment values.") variable = variables[0] attributes[variable] = expression dynamic_attr_names.append(variable) # translated attributes translated_attributes = self.translated_attributes or () for variable, msgid in translated_attributes: if msgid: if variable in dynamic_attr_names: raise ValueError( "Message id not allowed in conjunction with " "a dynamic attribute.") value = types.value('"%s"' % msgid) if variable in attributes: default = repr(attributes[variable]) expression = clauses.translate_expression(value, default=default) else: expression = clauses.translate_expression(value) else: value = attributes.get(variable) if value is not None: if variable not in dynamic_attr_names: value = repr(value) expression = clauses.translate_expression(value) else: raise ValueError("Must be either static or dynamic " "attribute when no message id " "is supplied.") attributes[variable] = expression # tag text = self.text if omit is not True: _.append( clauses.Attrs(self.static_attributes, "_attrs_%d" % abs(id(self.element)))) selfclosing = not text and not dynamic and len(self.element) == 0 tag = clauses.Tag(self.tag, attributes, expression=self.dict_attributes, selfclosing=selfclosing, cdata=self.cdata is not None, defaults=self.static_attributes) if omit: _.append( clauses.Condition(omit, [tag], finalize=False, invert=True)) else: _.append(tag) # tag text (if we're not replacing tag body) if len( text ) and not dynamic and not self.use_macro and not self.extend_macro: for part in text: if isinstance(part, types.expression): _.append(clauses.Write(part)) else: _.append(clauses.Out(part)) # dynamic content if content: msgid = self.translate if msgid is not None: if msgid: raise ValueError( "Can't use message id with dynamic content translation." ) _.append(clauses.Assign(content, self.symbols.tmp)) content = clauses.translate_expression( types.value(self.symbols.tmp)) else: value = types.value( repr(utils.serialize(self.element, omit=True))) _.insert( 0, clauses.Assign( value, "%s.value = %s" % (self.symbols.default_marker_symbol, self.symbols.default))) _.append(clauses.Write(content)) # dynamic text elif self.translate is not None and \ True in map(lambda part: isinstance(part, types.expression), text): if len(self.element): raise ValueError( "Can't translate dynamic text block with elements in it.") init_stream = types.value('_init_stream()') init_stream.symbol_mapping[ '_init_stream'] = generation.initialize_stream subclauses = [] subclauses.append( clauses.Define( types.declaration((self.symbols.out, self.symbols.write)), init_stream)) for part in text: if isinstance(part, types.expression): subclauses.append(clauses.Write(part)) else: part = ' '.join(part.split()) if part != "": subclauses.append(clauses.Out(part)) # attempt translation subclauses.append( clauses.Assign( clauses.translate_expression( types.value('%(out)s.getvalue()'), default=None), self.symbols.tmp)) _.append(clauses.Group(subclauses)) _.append(clauses.Write(types.value(self.symbols.tmp))) # include elif self.include: # compute macro function arguments and create argument string arguments = [ "%s=%s" % (arg, arg) for arg in \ set(itertools.chain(*self.stream.scope[1:]))] # XInclude's are similar to METAL macros, except the macro # is always defined as the entire template. # first we compute the filename expression and write it to # an internal variable _.append(clauses.Assign(self.include, self.symbols.include)) # call template _.append(clauses.Write( types.template( "%%(xincludes)s.get(%%(include)s, %s).render_xinclude(%s)" % \ (repr(self.format), ", ".join(arguments))))) # use or extend macro elif self.use_macro or self.extend_macro: # assign macro value to variable macro = self.use_macro or self.extend_macro _.append(clauses.Assign(macro, self.symbols.metal)) # for each fill-slot element, create a new output stream # and save value in a temporary variable kwargs = [] callbacks = {} # determine variable scope scope = set(itertools.chain(adding, *self.stream.scope[1:])) - \ self.stream.scope[0] # we pass in all variables from the current scope (as # keyword arguments, to allow first use before potential # reassignment) callback_args = ", ".join("%s=%s" % (arg, arg) for arg in scope if arg not in (self.symbols.slots, self.symbols.scope)) macro_args = ", ".join("%s=%s" % (arg, arg) for arg in scope if arg not in (self.symbols.slots, )) # loop through macro fill slot elements and generate # callback methods; the reason why we use callbacks is # convenience: it's an easy fit with the compiler elements = [ element for element in self.element.walk() if element.node and element.node.fill_slot ] for element in elements: # make sure we're not in a nested macro block parent = element.getparent() while parent is not self.element: if parent.node.use_macro or parent.node.extend_macro: element = None break parent = parent.getparent() if element is None: continue # determine and register callback name name = element.node.fill_slot callbacks[name] = callback = "%s_%s" % ( self.symbols.callback, utils.normalize_slot_name(name)) # pass in remote scope to callback method; this is # done because macros may add global variables to the # scope, which should be made available to the calling # template visitor = clauses.Visit(element.node) tail = element.tail newline = tail and '\n' in tail _.append( clauses.Callback(callback, visitor, callback_args, newline)) # if we're extending the macro, the current slots will be # carried over to the macro extend = self.extend_macro is not None defines = set() if extend: for element in self.element.walk(): if element.node is not None: define_slot = element.node.define_slot if define_slot is not None: defines.add(define_slot) # format slot arguments slot_args = ", ".join("'%s': %s" % kwarg for kwarg in callbacks.items()) _.append( clauses.Macro(types.value("{%s}" % slot_args), macro_args, extend=extend, extend_except=defines, label=macro.label)) # translate body elif self.translate is not None: msgid = self.translate # subelements are either named or unnamed; if there are # unnamed elements, the message id must be dynamic named_elements = [ e for e in self.element if e.node.translation_name ] unnamed_elements = [ e for e in self.element if not e.node.translation_name ] if not msgid and named_elements and not unnamed_elements: msgid = self.create_msgid() elements = named_elements else: elements = self.element if msgid and not named_elements: elements = () if named_elements: mapping = "%s_%d" % (self.symbols.mapping, abs(id( self.element))) _.append(clauses.Assign(types.value('{}'), mapping)) else: mapping = 'None' if unnamed_elements or not msgid: text = utils.htmlescape( self.element.text.replace('%', '%') or "") _.append( clauses.Assign(types.value(repr(text)), self.symbols.msgid)) # for each named block, create a new output stream # and use the value in the translation mapping dict for element in elements: init_stream = types.value('_init_stream()') init_stream.symbol_mapping[ '_init_stream'] = generation.initialize_stream subclauses = [] subclauses.append( clauses.Define( types.declaration( (self.symbols.out, self.symbols.write)), init_stream)) subclauses.append(clauses.Visit(element.node)) # if the element is named, record it in the mapping if element in named_elements: name = element.node.translation_name subclauses.append( clauses.Assign(types.template('%(out)s.getvalue()'), "%s['%s']" % (mapping, name))) # when computing a dynamic message id, add a # reference to the named block if not msgid: if not unnamed_elements: subclauses.append( clauses.Assign( types.value(repr("${%s}" % name)), self.symbols.msgid)) else: subclauses.append( clauses.Assign( types.template('%(msgid)s + ' + repr("${%s}" % name) + ' + ' + repr(element.tail)), self.symbols.msgid)) # else add it to the dynamic message id else: subclauses.append( clauses.Assign( types.template('%(msgid)s + %(out)s.getvalue()'), self.symbols.msgid)) # XXX: note that this should read: # _.append(clauses.Group(subclauses)) # # but there's a problem with multiple temporary # variable assignments within the same block; this is # just an easy work-around _.append( clauses.Condition(types.value('True'), subclauses, finalize=True)) if msgid: value = types.value(repr(msgid)).replace('%', '%%') default = self.symbols.marker else: default = types.template('%(msgid)s') value = types.template("' '.join(%(msgid)s.split())") _.append( clauses.Assign( clauses.translate_expression(value, mapping=mapping, default=default), self.symbols.result)) # write translation to output if successful, otherwise # fallback to default rendition; result = types.value(self.symbols.result) result.symbol_mapping[self.symbols.marker] = i18n.marker if msgid: condition = types.template('%(result)s is not %(marker)s') _.append( clauses.Condition(condition, [clauses.UnicodeWrite(result)], finalize=True)) subclauses = [] if self.element.text: subclauses.append( clauses.Out(utils.htmlescape(self.element.text))) for element in self.element: name = element.node.translation_name if name: value = types.value("%s['%s']" % (mapping, name)) subclauses.append(clauses.UnicodeWrite(value)) for part in reversed(element.node.tail): if isinstance(part, types.expression): subclauses.append(clauses.Write(part)) else: subclauses.append( clauses.Out(utils.htmlescape(part))) else: subclauses.append(clauses.Visit(element.node)) if subclauses: _.append(clauses.Else(subclauses)) else: _.append(clauses.UnicodeWrite(result)) return _