def writexml(self, stream, *args, **kwargs):
if self.raw:
val = decode(self.nodeValue)
else:
v = decode(self.nodeValue)
v = ' '.join(v.split()) if kwargs.get('strip') else v
val = escape(v)
val = encode(val)
stream.write(val)
def writexml(self, stream, *args, **kwargs):
if self.raw:
val = decode(self.nodeValue)
else:
v = decode(self.nodeValue)
v = ' '.join(v.split()) if kwargs.get('strip') else v
val = escape(v)
val = encode(val)
stream.write(val)
def async_parser(base, objconf, skip=False, **kwargs):
""" Asynchronously parses the pipe content
Args:
base (str): The base currency (exchanging from)
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
assign (str): Attribute to assign parsed content (default: exchangerate)
stream (dict): The original item
Returns:
Deferred: twisted.internet.defer.Deferred item
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> def run(reactor):
... url = get_path('quote.json')
... conf = {
... 'url': url, 'currency': 'USD', 'sleep': 0, 'precision': 6}
... item = {'content': 'GBP'}
... objconf = Objectify(conf)
... kwargs = {'stream': item, 'assign': 'content'}
... d = async_parser(item['content'], objconf, **kwargs)
... return d.addCallbacks(print, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
1.545801
"""
same_currency = base == objconf.currency
if skip:
rate = kwargs['stream']
elif same_currency:
rate = Decimal(1)
elif objconf.url.startswith('http'):
r = yield treq.get(objconf.url, params=objconf.params)
json = yield treq.json(r)
else:
url = get_abspath(objconf.url)
content = yield io.async_url_read(url, delay=objconf.sleep)
json = loads(decode(content))
if not (skip or same_currency):
places = Decimal(10)**-objconf.precision
rates = parse_response(json)
rate = calc_rate(base, objconf.currency, rates, places=places)
return_value(rate)
def async_parser(base, objconf, skip, **kwargs):
""" Asynchronously parses the pipe content
Args:
base (str): The base currency (exchanging from)
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
assign (str): Attribute to assign parsed content (default: exchangerate)
stream (dict): The original item
Returns:
Deferred: twisted.internet.defer.Deferred Tuple of (item, skip)
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from riko.lib.utils import Objectify
>>>
>>> def run(reactor):
... callback = lambda x: print(x[0])
... url = get_path('quote.json')
... conf = {
... 'url': url, 'currency': 'USD', 'sleep': 0, 'precision': 6}
... item = {'content': 'GBP'}
... objconf = Objectify(conf)
... kwargs = {'stream': item, 'assign': 'content'}
... d = async_parser(item['content'], objconf, False, **kwargs)
... return d.addCallbacks(callback, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
1.545801
"""
if skip:
rate = kwargs['stream']
elif objconf.url.startswith('http'):
r = yield treq.get(objconf.url, params=objconf.params)
json = yield treq.json(r)
else:
url = utils.get_abspath(objconf.url)
content = yield io.async_url_read(url, delay=objconf.sleep)
json = loads(decode(content))
if not skip:
places = Decimal(10) ** -objconf.precision
rates = parse_response(json)
rate = calc_rate(base, objconf.currency, rates, places=places)
result = (rate, skip)
return_value(result)
def get_text(html, convert_charrefs=False):
try:
parser = LinkParser(convert_charrefs=convert_charrefs)
except TypeError:
parser = LinkParser()
try:
parser.feed(html)
except TypeError:
parser.feed(decode(html))
return parser.data.getvalue()
def get_text(html, convert_charrefs=False):
try:
parser = LinkParser(convert_charrefs=convert_charrefs)
except TypeError:
parser = LinkParser()
try:
parser.feed(html)
except TypeError:
parser.feed(decode(html))
return parser.data.getvalue()
def get_abspath(url):
url = 'http://%s' % url if url and '://' not in url else url
if url and url.startswith('file:///'):
# already have an abspath
pass
elif url and url.startswith('file://'):
parent = p.dirname(p.dirname(p.dirname(__file__)))
rel_path = url[7:]
abspath = p.abspath(p.join(parent, rel_path))
url = 'file://%s' % abspath
return decode(url)
def get_abspath(url):
url = 'http://%s' % url if url and '://' not in url else url
if url and url.startswith('file:///'):
# already have an abspath
pass
elif url and url.startswith('file://'):
parent = p.dirname(p.dirname(p.dirname(__file__)))
rel_path = url[7:]
abspath = p.abspath(p.join(parent, rel_path))
url = 'file://%s' % abspath
return decode(url)
def parse_rss(url, delay=0):
context = SleepyDict(delay=delay)
response = None
try:
response = urlopen(decode(url), context=context)
except TypeError:
try:
response = urlopen(decode(url))
except (ValueError, URLError):
parsed = rssparser.parse(url)
except (ValueError, URLError):
parsed = rssparser.parse(url)
if response:
content = response.read() if speedparser else response
try:
parsed = rssparser.parse(content)
finally:
response.close()
return parsed
def parse_rss(url, delay=0):
context = SleepyDict(delay=delay)
response = None
try:
response = urlopen(decode(url), context=context)
except TypeError:
try:
response = urlopen(decode(url))
except (ValueError, URLError):
parsed = rssparser.parse(url)
except (ValueError, URLError):
parsed = rssparser.parse(url)
if response:
content = response.read() if speedparser else response
try:
parsed = rssparser.parse(content)
finally:
response.close()
return parsed
def dataReceived(self, data):
stateTable = self._build_state_table()
self.encoding = self.encoding or detect(data)['encoding']
self.check_encoding(data)
self.state = self.state or 'begin'
content = decode(data, self.encoding)
# bring state, lineno, colno into local scope
lineno, colno = self.lineno, self.colno
curState = self.state
# replace saveMark with a nested scope function
saveMark = lambda: (lineno, colno)
self.saveMark, _saveMark = saveMark, self.saveMark
# fetch functions from the stateTable
beginFn, doFn, endFn = stateTable[curState]
try:
for char in content:
# do newline stuff
if char == '\n':
lineno += 1
colno = 0
else:
colno += 1
newState = doFn(char)
if newState and newState != curState:
# this is the endFn from the previous state
endFn()
curState = newState
beginFn, doFn, endFn = stateTable[curState]
beginFn(char)
finally:
self.saveMark = _saveMark
self.lineno, self.colno = lineno, colno
# state doesn't make sense if there's an exception..
self.state = curState
def dataReceived(self, data):
stateTable = self._build_state_table()
self.encoding = self.encoding or detect(data)['encoding']
self.check_encoding(data)
self.state = self.state or 'begin'
content = decode(data, self.encoding)
# bring state, lineno, colno into local scope
lineno, colno = self.lineno, self.colno
curState = self.state
# replace saveMark with a nested scope function
saveMark = lambda: (lineno, colno)
self.saveMark, _saveMark = saveMark, self.saveMark
# fetch functions from the stateTable
beginFn, doFn, endFn = stateTable[curState]
try:
for char in content:
# do newline stuff
if char == '\n':
lineno += 1
colno = 0
else:
colno += 1
newState = doFn(char)
if newState and newState != curState:
# this is the endFn from the previous state
endFn()
curState = newState
beginFn, doFn, endFn = stateTable[curState]
beginFn(char)
finally:
self.saveMark = _saveMark
self.lineno, self.colno = lineno, colno
# state doesn't make sense if there's an exception..
self.state = curState
def handle_data(self, data):
self.data.write('%s\n' % decode(data))
def file2entries(f, parser):
for line in f:
parser.feed(decode(line))
for entry in parser.entry:
yield entry
def gen_entries(f, parser):
for line in f:
parser.feed(decode(line))
for entry in parser.entry:
yield entry
def handle_data(self, data):
self.data.write('%s\n' % decode(data))