def test_escaping(self): # issue 590 raw = "&'some \"stuff\"', <what up?>" cooked = "&'some "stuff"', <what up?>" esc1 = microdom.escape(raw) self.assertEqual(esc1, cooked) self.assertEqual(microdom.unescape(esc1), raw)
def testEscaping(self): # issue 590 raw = "&'some \"stuff\"', <what up?>" cooked = "&'some "stuff"', <what up?>" esc1 = microdom.escape(raw) self.assertEquals(esc1, cooked) self.assertEquals(microdom.unescape(esc1), raw)
def fetch_feed(self, url): feed_type = self.feeds[url].type err = None try: data = yield self.client.request(url) except HTTPClientError as err: log.warn('Failed to fetch feed ({}): {}'.format(url, err)) data = None finally: self.schedule_fetch(url, fast=bool(err)) # do faster re-fetch on errors if data is None: defer.returnValue(None) # cache hit, not modified, error data, headers = data if feed_type == 'feed': import feedparser parser = feedparser.parse(data, response_headers=headers) feed, posts = parser.feed, parser.entries elif feed_type == 'reddit-json': from lya import AttrDict # mandatory dep anyway data = json.loads(data)['data'] posts = list(AttrDict(post['data']) for post in data.pop('children')) feed = AttrDict(data) else: raise ValueError('Unrecognized feed type: {!r}'.format(self.feeds[url].type)) count = 0 for post in reversed(posts): if feed_type == 'reddit-json': # Some reddit-api-specific encoding hacks try: title = unescape(post['title']) except KeyError: pass else: post.title = title post_obj = FeedEntryInfo(feed, post, self.conf) post_id = list( force_bytes(post_obj.get_by_path(attr)) for attr in self.feeds[url].deduplication ) if not self.filter_db.add(url, post_id): continue first_err = None for template in self.feeds[url].template: try: event = template.format(**post_obj._asdict()) except (KeyError, IndexError, AttributeError) as err: if not first_err: first_err = ValueError( 'Failed to format template {!r} (data: {}): {}'\ .format(template, post_obj, err) ) continue event = RelayedEvent(event) event.data = post_obj # for any further tricky filtering reactor.callLater(0, self.interface.dispatch, event, source=self) break else: raise first_err # all templates failed count += 1 if self.feeds[url].process_max and count >= self.feeds[url].process_max: break
def _xml_node_text(node): result = '' for child in node.childNodes: if isinstance(child, microdom.CharacterData): result += child.value elif isinstance(child, microdom.EntityReference): result += microdom.unescape( child.toxml(), chars=microdom.XML_ESCAPE_CHARS) return result.strip()
def parse_post(self, request): print "Client IP: {0}".format(request.getClientIP()) # print request.args.keys() input_data = cgi.escape(request.args["payload"][0]) data = json.loads(input_data) # Format issues as below: # Issue CLOSED - saltstack/salt: #1888 (Add daemonize_if to service.restart for the minion) <https://github.com/saltstack/salt/issues/1888> issue_str = unescape('Issue {0} - {1}: #{2} ({3}) <{4}>'.format( data['action'].upper(), data['repository']['full_name'], data['issue']['number'], data['issue']['title'], data['issue']['html_url'])) for channel in self.irc.state.channels: if self.channel == channel: self.irc.queueMsg(ircmsgs.privmsg(channel, issue_str))
def parse_post(self, request): print "Client IP: {0}".format(request.getClientIP()) # print request.args.keys() input_data = cgi.escape(request.args["payload"][0]) data = json.loads(input_data) # Format issues as below: # Issue CLOSED - saltstack/salt: #1888 (Add daemonize_if to service.restart for the minion) <https://github.com/saltstack/salt/issues/1888> issue_str = unescape('Issue {0} - {1}: #{2} ({3}) <{4}>'.format( data['action'].upper(), data['repository']['full_name'], data['issue']['number'], data['issue']['title'], data['issue']['html_url'] )) for channel in self.irc.state.channels: if self.channel == channel: self.irc.queueMsg(ircmsgs.privmsg(channel, issue_str))
def gatherTextNodes(iNode, dounescape=0, joinWith=""): """Visit each child node and collect its text data, if any, into a string. For example:: >>> doc=microdom.parseString('<a>1<b>2<c>3</c>4</b></a>') >>> gatherTextNodes(doc.documentElement) '1234' With dounescape=1, also convert entities back into normal characters. @return: the gathered nodes as a single string @rtype: str""" gathered = [] gathered_append = gathered.append slice = [iNode] while len(slice) > 0: c = slice.pop(0) if hasattr(c, "nodeValue") and c.nodeValue is not None: if dounescape: val = unescape(c.nodeValue) else: val = c.nodeValue gathered_append(val) slice[:0] = c.childNodes return joinWith.join(gathered)
def gatherTextNodes(iNode, dounescape=0, joinWith=""): """Visit each child node and collect its text data, if any, into a string. For example:: >>> doc=microdom.parseString('<a>1<b>2<c>3</c>4</b></a>') >>> gatherTextNodes(doc.documentElement) '1234' With dounescape=1, also convert entities back into normal characters. @return: the gathered nodes as a single string @rtype: str """ gathered=[] gathered_append=gathered.append slice=[iNode] while len(slice)>0: c=slice.pop(0) if hasattr(c, 'nodeValue') and c.nodeValue is not None: if dounescape: val=unescape(c.nodeValue) else: val=c.nodeValue gathered_append(val) slice[:0]=c.childNodes return joinWith.join(gathered)
def textEscaping(self): raw="&'some \"stuff\"', <what up?>" cooked="&'some "stuff"', <what up?>" esc1=microdom.escape(input) self.assertEquals(esc1, cooked) self.assertEquals(microdom.unescape(esc1), input)