def handle_starttag(self, tag, attrib): fixed_attrib = [(QName(name), name if value is None else value) for name, value in attrib] self._enqueue(START, (QName(tag), Attrs(fixed_attrib))) if tag in self._EMPTY_ELEMS: self._enqueue(END, QName(tag)) else: self._open_tags.append(tag)
def test_map_element(self): self.assertEqual( self._map('foo'), [(QName('foo'), Attrs([(QName('name'), 'foo'), (QName('size'), '100')])), 'FOO', QName('foo')] )
def _rewrite_stream(stream, directives, ctxt, vars, bind): stream = list(stream) mutable_attrs = {} for control_attribute in directives: control_attribute.inject(mutable_attrs, ctxt, vars) kind, (tagname, attrs), pos = stream[0] if len(stream) == 2: contents = None else: contents = _simplify_stream(stream[1:-1], ctxt, vars) existing_attributes = {} for qname, value in attrs: if qname.namespace is None: if not isinstance(value, unicode): value = _simplify_stream(value, ctxt, vars) attrs |= ((qname, value), ) existing_attributes[qname.localname] = qname mutable_attrs[qname.localname] = value try: render_context = ctxt['flatland_render_context'] except KeyError: ctxt['flatland_render_context'] = render_context = Context() new_contents = transform(tagname.localname, mutable_attrs, contents, render_context, bind) if new_contents is None: new_contents = () elif isinstance(new_contents, unicode): new_contents = [(TEXT, new_contents, (None, -1, -1))] pairs = sorted(mutable_attrs.iteritems(), key=_attribute_sort_key) for attribute_name, value in pairs: if attribute_name in existing_attributes: qname = existing_attributes.pop(attribute_name) else: qname = QName(attribute_name) attrs |= ((qname, value), ) for qname in existing_attributes.values(): attrs -= qname stream[0] = (kind, (tagname, attrs), pos) if new_contents and tagname.localname == u'select' and bind is not None: if tagname.namespace: sub_tag = Namespace(tagname.namespace).option else: # pragma: nocover sub_tag = QName('option') new_contents = _bind_unbound_tags(new_contents, sub_tag, bind) if new_contents: stream[1:-1] = new_contents return iter(stream)
def to_genshi(walker): """Convert a tree to a genshi tree :arg walker: the treewalker to use to walk the tree to convert it :returns: generator of genshi nodes """ text = [] for token in walker: type = token["type"] if type in ("Characters", "SpaceCharacters"): text.append(token["data"]) elif text: yield TEXT, "".join(text), (None, -1, -1) text = [] if type in ("StartTag", "EmptyTag"): if token["namespace"]: name = "{%s}%s" % (token["namespace"], token["name"]) else: name = token["name"] attrs = Attrs( [ (QName("{%s}%s" % attr if attr[0] is not None else attr[1]), value) for attr, value in token["data"].items() ] ) yield (START, (QName(name), attrs), (None, -1, -1)) if type == "EmptyTag": type = "EndTag" if type == "EndTag": if token["namespace"]: name = "{%s}%s" % (token["namespace"], token["name"]) else: name = token["name"] yield END, QName(name), (None, -1, -1) elif type == "Comment": yield COMMENT, token["data"], (None, -1, -1) elif type == "Doctype": yield DOCTYPE, (token["name"], token["publicId"], token["systemId"]), ( None, -1, -1, ) else: pass # FIXME: What to do? if text: yield TEXT, "".join(text), (None, -1, -1)
def test_cache_markup(self): loc = (None, -1, -1) stream = Stream([(Stream.START, (QName('foo'), Attrs()), loc), (Stream.TEXT, u'…', loc), (Stream.END, QName('foo'), loc), (Stream.START, (QName('bar'), Attrs()), loc), (Stream.TEXT, Markup('…'), loc), (Stream.END, QName('bar'), loc)]) output = stream.render(XMLSerializer, encoding=None, strip_whitespace=False) self.assertEqual('<foo>&hellip;</foo><bar>…</bar>', output)
def handle_starttag(self, tag, attrib): fixed_attrib = [] for name, value in attrib: # Fixup minimized attributes if value is None: value = name fixed_attrib.append((QName(name), stripentities(value))) self._enqueue(START, (QName(tag), Attrs(fixed_attrib))) if tag in self._EMPTY_ELEMS: self._enqueue(END, QName(tag)) else: self._open_tags.append(tag)
def handle_starttag(self, tag, attrib): fixed_attrib = [] for name, value in attrib: # Fixup minimized attributes if value is None: value = str(name) elif not isinstance(value, str): value = value.decode(self.encoding, 'replace') fixed_attrib.append((QName(name), stripentities(value))) self._enqueue(START, (QName(tag), Attrs(fixed_attrib))) if tag in self._EMPTY_ELEMS: self._enqueue(END, QName(tag)) else: self._open_tags.append(tag)
def test_convert_ElementTree_to_markup_stream(self): tree = ElementTree.fromstring( u'<div class="test_div">text<span>some more text</span></div>' ) events = list(ET(tree)) self.assertEqual(6, len(events)) self.assertEqual( (Stream.START, (QName("div"), Attrs([(QName("class"), "test_div")]))), events[0][:2], ) self.assertEqual((Stream.TEXT, "text"), events[1][:2]) self.assertEqual((Stream.START, (QName("span"), Attrs())), events[2][:2]) self.assertEqual((Stream.TEXT, "some more text"), events[3][:2]) self.assertEqual((Stream.END, QName("span")), events[4][:2]) self.assertEqual((Stream.END, QName("div")), events[5][:2])
def _generate(): try: bufsize = 4 * 1024 # 4K done = False while 1: while not done and len(self._queue) == 0: data = self.source.read(bufsize) if not data: # end of data self.close() done = True else: if not isinstance(data, str): # bytes if self.encoding: data = data.decode(self.encoding) else: raise UnicodeError("source returned bytes, but no encoding specified") self.feed(data) for kind, data, pos in self._queue: yield kind, data, pos self._queue = [] if done: open_tags = self._open_tags open_tags.reverse() for tag in open_tags: yield END, QName(tag), pos break except html.HTMLParseError as e: msg = '%s: line %d, column %d' % (e.msg, e.lineno, e.offset) raise ParseError(msg, self.filename, e.lineno, e.offset)
def post_process_request(self, req, template, data, content_type): """post process request filter""" # update the nav item links root = req.href() rootproducts = req.href.products() pid = req.args.get('productid') if pid: for navkey, section in req.chrome['nav'].iteritems(): for item in section: try: href = item['label'].attrib.get('href') except AttributeError: continue if href.startswith(rootproducts): continue if href.startswith(root): tail = href[len(root):] if tail not in self.NAVITEM_DO_NOT_TRANSFORM: attrs = [ attr for attr in item['label'].attrib if attr[0] != 'href' ] newhref = req.href.products(pid, tail) item['label'].attrib = Attrs([(QName('href'), newhref)] + attrs) return (template, data, content_type)
def handle_endtag(self, tag): if tag not in self._EMPTY_ELEMS: while self._open_tags: open_tag = self._open_tags.pop() self._enqueue(END, QName(open_tag)) if open_tag.lower() == tag.lower(): break
def sanitize_attrs(self, tag, attrs): """Remove potentially dangerous attributes and sanitize the style attribute . :param tag: the tag name of the element :type attrs: dict corresponding to tag attributes :return: a dict containing only safe or sanitized attributes :rtype: dict """ new_attrs = {} for attr, value in attrs.iteritems(): if value is None: value = attr if attr not in self.safe_attrs: continue elif attr in self.uri_attrs: # Don't allow URI schemes such as "javascript:" if not self.is_safe_uri(value): continue elif attr == 'style': # Remove dangerous CSS declarations from inline styles decls = self.sanitize_css(value) if not decls: continue value = '; '.join(decls) new_attrs[attr] = value if tag == 'img' and 'src' in new_attrs and \ not self._is_safe_origin(new_attrs['src']): attr = 'crossorigin' if QName and isinstance(tag, QName): attr = QName(attr) new_attrs[attr] = 'anonymous' return new_attrs
def GenshiAdapter(tree): text = None for token in treewalkers.getTreeWalker("dom")(tree): type = token["type"] if type in ("Characters", "SpaceCharacters"): if text is None: text = token["data"] else: text += token["data"] elif text is not None: yield TEXT, text, (None, -1, -1) text = None if type in ("StartTag", "EmptyTag"): if token["namespace"]: name = "{%s}%s" % (token["namespace"], token["name"]) else: name = token["name"] attrs = Attrs([ (QName("{%s}%s" % attr if attr[0] is not None else attr[1]), value) for attr, value in token["data"].items() ]) yield (START, (QName(name), attrs), (None, -1, -1)) if type == "EmptyTag": type = "EndTag" if type == "EndTag": if token["namespace"]: name = "{%s}%s" % (token["namespace"], token["name"]) else: name = token["name"] yield END, QName(name), (None, -1, -1) elif type == "Comment": yield COMMENT, token["data"], (None, -1, -1) elif type == "Doctype": yield DOCTYPE, (token["name"], token["publicId"], token["systemId"]), (None, -1, -1) else: pass # FIXME: What to do? if text is not None: yield TEXT, text, (None, -1, -1)
def GenshiAdapter(tree): text = None for token in treewalkers.getTreeWalker('dom')(tree): type = token['type'] if type in ('Characters', 'SpaceCharacters'): if text is None: text = token['data'] else: text += token['data'] elif text is not None: yield TEXT, text, (None, -1, -1) text = None if type in ('StartTag', 'EmptyTag'): if token['namespace']: name = '{%s}%s' % (token['namespace'], token['name']) else: name = token['name'] attrs = Attrs([ (QName('{%s}%s' % attr if attr[0] is not None else attr[1]), value) for attr, value in token['data'].items() ]) yield (START, (QName(name), attrs), (None, -1, -1)) if type == 'EmptyTag': type = 'EndTag' if type == 'EndTag': if token['namespace']: name = '{%s}%s' % (token['namespace'], token['name']) else: name = token['name'] yield END, QName(name), (None, -1, -1) elif type == 'Comment': yield COMMENT, token['data'], (None, -1, -1) elif type == 'Doctype': yield DOCTYPE, (token['name'], token['publicId'], token['systemId']), (None, -1, -1) else: pass # FIXME: What to do? if text is not None: yield TEXT, text, (None, -1, -1)
def mark_text(self, pos, text, tag): ws, text = self.cut_leading_space(text) tag = QName(tag) if ws: self.append(TEXT, ws, pos) self.append(START, (tag, Attrs()), pos) self.append(TEXT, text, pos) self.append(END, tag, pos)
def test_attr_selection_with_namespace(self): xml = XML('<root xmlns:ns1="http://example.com">' '<foo ns1:bar="abc"></foo>' '</root>') path = Path('foo/@ns1:bar') result = path.select(xml, namespaces={'ns1': 'http://example.com'}) self.assertEqual(list(result), [Attrs([(QName('http://example.com}bar'), 'abc')])])
def block_process(self, events): for event in events: type, data, pos = event if type == START: self.enter(pos, *data) elif type == END: self.leave(pos, data) elif type == TEXT: if self._context is not None and data.strip(): tag = QName(self._context) self.append(START, (QName(tag), Attrs()), pos) self.append(type, data, pos) self.append(END, tag, pos) else: self.append(type, data, pos) else: self.append(type, data, pos)
def __call__(self, stream): """Apply the transform filter to the marked stream. :param stream: The marked event stream to filter """ callable_value = hasattr(self.value, '__call__') for mark, (kind, data, pos) in stream: if mark is ENTER: if callable_value: value = self.value(self.name, (kind, data, pos)) else: value = self.value if value is None: attrs = data[1] - [QName(self.name)] else: attrs = data[1] | [(QName(self.name), value)] data = (data[0], attrs) yield mark, (kind, data, pos)
def _kwargs_to_attrs(kwargs): attrs = [] names = set() for name, value in kwargs.items(): name = name.rstrip('_').replace('_', '-') if value is not None and name not in names: attrs.append((QName(name), unicode(value))) names.add(name) return Attrs(attrs)
def test_pickle(self): qname = QName('http://www.example.org/namespace}elem') buf = BytesIO() pickle.dump(qname, buf, 2) buf.seek(0) unpickled = pickle.load(buf) self.assertEquals('{http://www.example.org/namespace}elem', unpickled) self.assertEquals('http://www.example.org/namespace', unpickled.namespace) self.assertEquals('elem', unpickled.localname)
def extract(fileobj, keywords, comment_tags, options): """Babel extraction method for Genshi templates. :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ template_class = options.get('template_class', MarkupTemplate) if isinstance(template_class, basestring): module, clsname = template_class.split(':', 1) template_class = getattr(__import__(module, {}, {}, [clsname]), clsname) encoding = options.get('encoding', None) extract_text = options.get('extract_text', True) if isinstance(extract_text, basestring): extract_text = extract_text.lower() in ('1', 'on', 'yes', 'true') ignore_tags = options.get('ignore_tags', Translator.IGNORE_TAGS) if isinstance(ignore_tags, basestring): ignore_tags = ignore_tags.split() ignore_tags = [QName(tag) for tag in ignore_tags] include_attrs = options.get('include_attrs', Translator.INCLUDE_ATTRS) if isinstance(include_attrs, basestring): include_attrs = include_attrs.split() include_attrs = [QName(attr) for attr in include_attrs] tmpl = template_class(fileobj, filename=getattr(fileobj, 'name', None), encoding=encoding) tmpl.loader = None translator = Translator(None, ignore_tags, include_attrs, extract_text) if hasattr(tmpl, 'add_directives'): tmpl.add_directives(Translator.NAMESPACE, translator) for message in translator.extract(tmpl.stream, gettext_functions=keywords): yield message
def ET(element): """Convert a given ElementTree element to a markup stream. :param element: an ElementTree element :return: a markup stream """ tag_name = QName(element.tag.lstrip('{')) attrs = Attrs([(QName(attr.lstrip('{')), value) for attr, value in element.items()]) yield START, (tag_name, attrs), (None, -1, -1) if element.text: yield TEXT, element.text, (None, -1, -1) for child in element.getchildren(): for item in ET(child): yield item yield END, tag_name, (None, -1, -1) if element.tail: yield TEXT, element.tail, (None, -1, -1)
def _add_title(self, data, tagname, attr_name, prefix, after_stream, depth): """ In case of data parameter as element has same tagname attribute for the parameter and attribute named the attr_name starts with the prefix, add description in title attribute and rel attribute to the element and store div element generated to after_stream[depth] for later use. (In Japanese/KANJI) data で与えられた要素が、引数で指定された tagname であり、attr_name 属性の値が prefix で始まる場合、 説明文をtitle属性およびrel属性としてその要素に設定します。 またそのとき、after_stream[depth] に DIV 要素を格納します。 """ element, attrs = data attr_value = attrs.get(attr_name) if element.localname == tagname and attr_value and attr_value.startswith( prefix): attr_value = attr_value[len(prefix):] attr_value_locale = "%s.%s" % (attr_value, self.locale) text = self.parent.pages.get( attr_value_locale, self.parent.pages.get( attr_value, FieldTooltip._default_pages.get( attr_value_locale, FieldTooltip._default_pages.get(attr_value)))) if text: attrs |= [(QName('title'), attr_value + ' | ' + text)] attrs |= [(QName('rel'), '#tooltip-' + attr_value)] img = tag.img(src='%s/chrome/common/wiki.png' \ % self.context.req.base_url, alt='?', align='right') a = tag.a(img, href='%s/wiki/%s%s' \ % (self.context.req.base_url, FieldTooltip._wiki_prefix, attr_value)) after_stream[str(depth)] = \ tag.div(a, '%s:\n' % attr_value, format_to_html(self.parent.env, self.context, text, False), id='tooltip-' + attr_value, class_='tooltip', style='display: none') data = element, attrs return data
def test_nested_default_namespaces(self): stream = Stream([ (Stream.START_NS, ('', 'http://example.org/'), (None, -1, -1)), (Stream.START, (QName('http://example.org/}div'), Attrs()), (None, -1, -1)), (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.START_NS, ('', 'http://example.org/'), (None, -1, -1)), (Stream.START, (QName('http://example.org/}p'), Attrs()), (None, -1, -1)), (Stream.END, QName('http://example.org/}p'), (None, -1, -1)), (Stream.END_NS, '', (None, -1, -1)), (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.START_NS, ('', 'http://example.org/'), (None, -1, -1)), (Stream.START, (QName('http://example.org/}p'), Attrs()), (None, -1, -1)), (Stream.END, QName('http://example.org/}p'), (None, -1, -1)), (Stream.END_NS, '', (None, -1, -1)), (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.END, QName('http://example.org/}div'), (None, -1, -1)), (Stream.END_NS, '', (None, -1, -1)) ]) output = stream.render(XMLSerializer, encoding=None) self.assertEqual( """<div xmlns="http://example.org/"> <p/> <p/> </div>""", output)
def test_nested_bound_namespaces(self): stream = Stream([ (Stream.START_NS, ('x', 'http://example.org/'), (None, -1, -1)), (Stream.START, (QName('http://example.org/}div'), Attrs()), (None, -1, -1)), (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.START_NS, ('x', 'http://example.org/'), (None, -1, -1)), (Stream.START, (QName('http://example.org/}p'), Attrs()), (None, -1, -1)), (Stream.END, QName('http://example.org/}p'), (None, -1, -1)), (Stream.END_NS, 'x', (None, -1, -1)), (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.START_NS, ('x', 'http://example.org/'), (None, -1, -1)), (Stream.START, (QName('http://example.org/}p'), Attrs()), (None, -1, -1)), (Stream.END, QName('http://example.org/}p'), (None, -1, -1)), (Stream.END_NS, 'x', (None, -1, -1)), (Stream.TEXT, '\n ', (None, -1, -1)), (Stream.END, QName('http://example.org/}div'), (None, -1, -1)), (Stream.END_NS, 'x', (None, -1, -1)) ]) output = stream.render(XMLSerializer) self.assertEqual( """<x:div xmlns:x="http://example.org/"> <x:p/> <x:p/> </x:div>""", output)
def GenshiAdapter(tree): text = None for token in treewalkers.getTreeWalker(u"simpletree")(tree): type = token[u"type"] if type in (u"Characters", u"SpaceCharacters"): if text is None: text = token[u"data"] else: text += token[u"data"] elif text is not None: yield TEXT, text, (None, -1, -1) text = None if type in (u"StartTag", u"EmptyTag"): if token[u"namespace"]: name = u"{%s}%s" % (token[u"namespace"], token[u"name"]) else: name = token[u"name"] yield (START, (QName(name), Attrs([(QName(attr),value) for attr,value in token[u"data"]])), (None, -1, -1)) if type == u"EmptyTag": type = u"EndTag" if type == u"EndTag": yield END, QName(token[u"name"]), (None, -1, -1) elif type == u"Comment": yield COMMENT, token[u"data"], (None, -1, -1) elif type == u"Doctype": yield DOCTYPE, (token[u"name"], token[u"publicId"], token[u"systemId"]), (None, -1, -1) else: pass # FIXME: What to do? if text is not None: yield TEXT, text, (None, -1, -1)
def GenshiAdapter(treewalker, tree): """Generator to convert html5lib treewalker tokens into Genshi stream tokens""" text = None for token in treewalker(tree): token_type = token["type"] if token_type in ("Characters", "SpaceCharacters"): if text is None: text = token["data"] else: text += token["data"] elif text is not None: assert type(text) in (unicode, None) yield TEXT, text, (None, -1, -1) text = None if token_type in ("StartTag", "EmptyTag"): yield (START, (QName(token["name"]), Attrs([(QName(attr), value) for attr, value in token["data"]])), (None, -1, -1)) if token_type == "EmptyTag": token_type = "EndTag" if token_type == "EndTag": yield END, QName(token["name"]), (None, -1, -1) elif token_type == "Comment": yield COMMENT, token["data"], (None, -1, -1) elif token_type == "Doctype": yield DOCTYPE, (token["name"], None, None), (None, -1, -1) else: pass # FIXME: What to do? if text is not None: yield TEXT, text, (None, -1, -1)
def close_diff(self): """ Close diff marked piece of HTML. Detect if all operation events was the same, if so it means that only one kind of operation happen, so regular diff tag should be rendered. If operations are not the same this means that inside diffed element there is also original content, so in fact this was a change in formatting. Diffed sections are rendered lazily. Here, opening diff tag is put into result stream, then collected buffer events and closing tag. """ # child is an element which should be wrapped by diff, this could be a text node # or tag child = self._buffer[0] if self._all_same: # all events was the same operation node = getattr(DefaultDiffProducer, 'render_%s' % self.operation)( self.get_current_element()) else: # here text node is not valid, diff in text is by word, its not possible that # text nodes are inserted and not all in this context was inserted (if so, diff iterator # should return equal action and break insert processor) assert child[0] == START formatting_node = DOMNode(name=child[1][0], attrs=child[1][1]) node = getattr(DefaultDiffProducer, 'render_formatting_%s' % self.operation)(self.get_current_element(), formatting_node) logger.debug("Lazy diff '%s' of %d nodes marked using %r" % (self.operation, len(self._buffer), node)) self._result.append( (START, (QName(node.name), Attrs(node.attrs)), None)) self._result.extend(self._buffer) self._result.append((END, QName(node.name), None)) self._rendered = False self._buffer = []
def _generate(): kind, (tag, attrib), pos = stream.next() attrs = _eval_expr(self.expr, ctxt, vars) if attrs: if isinstance(attrs, Stream): try: attrs = iter(attrs).next() except StopIteration: attrs = [] elif not isinstance(attrs, list): # assume it's a dict attrs = attrs.items() attrib |= [(QName(n), v is not None and unicode(v).strip() or None) for n, v in attrs] yield kind, (tag, attrib), pos for event in stream: yield event
def __call__(self, stream): """Apply the transform filter to the marked stream. :param stream: the marked event stream to filter """ namespaces = {} variables = {} test = self.path.test() stream = iter(stream) next = stream.next for mark, event in stream: if mark is None: yield mark, event continue result = test(event, namespaces, variables) # XXX This is effectively genshi.core._ensure() for transform # streams. if result is True: if event[0] is START: yield ENTER, event depth = 1 while depth > 0: mark, subevent = next() if subevent[0] is START: depth += 1 elif subevent[0] is END: depth -= 1 if depth == 0: yield EXIT, subevent else: yield INSIDE, subevent test(subevent, namespaces, variables, updateonly=True) else: yield OUTSIDE, event elif isinstance(result, Attrs): # XXX Selected *attributes* are given a "kind" of None to # indicate they are not really part of the stream. yield ATTR, (ATTR, (QName(event[1][0] + '@*'), result), event[2]) yield None, event elif isinstance(result, tuple): yield OUTSIDE, result elif result: # XXX Assume everything else is "text"? yield None, (TEXT, unicode(result), (None, -1, -1)) else: yield None, event