def test_escaping(self):
# issue 590
raw = "&'some \"stuff\"', <what up?>"
cooked = "&'some "stuff"', <what up?>"
esc1 = microdom.escape(raw)
self.assertEqual(esc1, cooked)
self.assertEqual(microdom.unescape(esc1), raw)
def testEscaping(self):
# issue 590
raw = "&'some \"stuff\"', <what up?>"
cooked = "&'some "stuff"', <what up?>"
esc1 = microdom.escape(raw)
self.assertEquals(esc1, cooked)
self.assertEquals(microdom.unescape(esc1), raw)
def fetch_feed(self, url):
feed_type = self.feeds[url].type
err = None
try: data = yield self.client.request(url)
except HTTPClientError as err:
log.warn('Failed to fetch feed ({}): {}'.format(url, err))
data = None
finally: self.schedule_fetch(url, fast=bool(err)) # do faster re-fetch on errors
if data is None: defer.returnValue(None) # cache hit, not modified, error
data, headers = data
if feed_type == 'feed':
import feedparser
parser = feedparser.parse(data, response_headers=headers)
feed, posts = parser.feed, parser.entries
elif feed_type == 'reddit-json':
from lya import AttrDict # mandatory dep anyway
data = json.loads(data)['data']
posts = list(AttrDict(post['data']) for post in data.pop('children'))
feed = AttrDict(data)
else:
raise ValueError('Unrecognized feed type: {!r}'.format(self.feeds[url].type))
count = 0
for post in reversed(posts):
if feed_type == 'reddit-json':
# Some reddit-api-specific encoding hacks
try: title = unescape(post['title'])
except KeyError: pass
else: post.title = title
post_obj = FeedEntryInfo(feed, post, self.conf)
post_id = list(
force_bytes(post_obj.get_by_path(attr))
for attr in self.feeds[url].deduplication )
if not self.filter_db.add(url, post_id): continue
first_err = None
for template in self.feeds[url].template:
try: event = template.format(**post_obj._asdict())
except (KeyError, IndexError, AttributeError) as err:
if not first_err:
first_err = ValueError(
'Failed to format template {!r} (data: {}): {}'\
.format(template, post_obj, err) )
continue
event = RelayedEvent(event)
event.data = post_obj # for any further tricky filtering
reactor.callLater(0, self.interface.dispatch, event, source=self)
break
else: raise first_err # all templates failed
count += 1
if self.feeds[url].process_max and count >= self.feeds[url].process_max: break
def _xml_node_text(node):
result = ''
for child in node.childNodes:
if isinstance(child, microdom.CharacterData):
result += child.value
elif isinstance(child, microdom.EntityReference):
result += microdom.unescape(
child.toxml(), chars=microdom.XML_ESCAPE_CHARS)
return result.strip()
def parse_post(self, request):
print "Client IP: {0}".format(request.getClientIP())
# print request.args.keys()
input_data = cgi.escape(request.args["payload"][0])
data = json.loads(input_data)
# Format issues as below:
# Issue CLOSED - saltstack/salt: #1888 (Add daemonize_if to service.restart for the minion) <https://github.com/saltstack/salt/issues/1888>
issue_str = unescape('Issue {0} - {1}: #{2} ({3}) <{4}>'.format(
data['action'].upper(), data['repository']['full_name'],
data['issue']['number'], data['issue']['title'],
data['issue']['html_url']))
for channel in self.irc.state.channels:
if self.channel == channel:
self.irc.queueMsg(ircmsgs.privmsg(channel, issue_str))
def parse_post(self, request):
print "Client IP: {0}".format(request.getClientIP())
# print request.args.keys()
input_data = cgi.escape(request.args["payload"][0])
data = json.loads(input_data)
# Format issues as below:
# Issue CLOSED - saltstack/salt: #1888 (Add daemonize_if to service.restart for the minion) <https://github.com/saltstack/salt/issues/1888>
issue_str = unescape('Issue {0} - {1}: #{2} ({3}) <{4}>'.format(
data['action'].upper(),
data['repository']['full_name'],
data['issue']['number'],
data['issue']['title'],
data['issue']['html_url']
))
for channel in self.irc.state.channels:
if self.channel == channel:
self.irc.queueMsg(ircmsgs.privmsg(channel, issue_str))
def gatherTextNodes(iNode, dounescape=0, joinWith=""):
"""Visit each child node and collect its text data, if any, into a string.
For example::
>>> doc=microdom.parseString('<a>1<b>2<c>3</c>4</b></a>')
>>> gatherTextNodes(doc.documentElement)
'1234'
With dounescape=1, also convert entities back into normal characters.
@return: the gathered nodes as a single string
@rtype: str"""
gathered = []
gathered_append = gathered.append
slice = [iNode]
while len(slice) > 0:
c = slice.pop(0)
if hasattr(c, "nodeValue") and c.nodeValue is not None:
if dounescape:
val = unescape(c.nodeValue)
else:
val = c.nodeValue
gathered_append(val)
slice[:0] = c.childNodes
return joinWith.join(gathered)
def gatherTextNodes(iNode, dounescape=0, joinWith=""):
"""Visit each child node and collect its text data, if any, into a string.
For example::
>>> doc=microdom.parseString('<a>1<b>2<c>3</c>4</b></a>')
>>> gatherTextNodes(doc.documentElement)
'1234'
With dounescape=1, also convert entities back into normal characters.
@return: the gathered nodes as a single string
@rtype: str
"""
gathered=[]
gathered_append=gathered.append
slice=[iNode]
while len(slice)>0:
c=slice.pop(0)
if hasattr(c, 'nodeValue') and c.nodeValue is not None:
if dounescape:
val=unescape(c.nodeValue)
else:
val=c.nodeValue
gathered_append(val)
slice[:0]=c.childNodes
return joinWith.join(gathered)
def textEscaping(self):
raw="&'some \"stuff\"', <what up?>"
cooked="&'some "stuff"', <what up?>"
esc1=microdom.escape(input)
self.assertEquals(esc1, cooked)
self.assertEquals(microdom.unescape(esc1), input)
