def writexml(self, stream, *args, **kwargs):
if self.raw:
val = decode(self.nodeValue)
else:
v = decode(self.nodeValue)
v = ' '.join(v.split()) if kwargs.get('strip') else v
val = escape(v)
val = encode(val)
stream.write(val)
def writexml(self, stream, *args, **kwargs):
if self.raw:
val = decode(self.nodeValue)
else:
v = decode(self.nodeValue)
v = ' '.join(v.split()) if kwargs.get('strip') else v
val = escape(v)
val = encode(val)
stream.write(val)
def open(self, url, **params):
if url.startswith('http') and params:
r = requests.get(url, params=params, stream=True)
r.raw.decode_content = self.decode
response = r.text if self.cache_type else r.raw
else:
try:
r = urlopen(url, context=self.context, timeout=self.timeout)
except TypeError:
r = urlopen(url, timeout=self.timeout)
text = r.read() if self.cache_type else None
if self.decode:
encoding = get_response_encoding(r, self.def_encoding)
if self.cache_type:
response = decode(text, encoding)
else:
response = reencode(r.fp, encoding, decode=True)
else:
response = text if self.cache_type else r
self.r = r
return response
def open(self, url, **params):
if url.startswith('http') and params:
r = requests.get(url, params=params, stream=True)
r.raw.decode_content = self.decode
response = r.text if self.cache_type else r.raw
else:
try:
r = urlopen(url, context=self.context, timeout=self.timeout)
except TypeError:
r = urlopen(url, timeout=self.timeout)
text = r.read() if self.cache_type else None
if self.decode:
encoding = get_response_encoding(r, self.def_encoding)
if text:
response = decode(text, encoding)
else:
response = reencode(r.fp, encoding, decode=True)
else:
response = text or r
content_type = get_response_content_type(r)
if 'xml' in content_type:
self.ext = 'xml'
elif 'json' in content_type:
self.ext = 'json'
else:
self.ext = content_type.split('/')[1].split(';')[0]
self.r = r
return response
def async_parser(base, objconf, skip=False, **kwargs):
""" Asynchronously parses the pipe content
Args:
base (str): The base currency (exchanging from)
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
assign (str): Attribute to assign parsed content (default: exchangerate)
stream (dict): The original item
Returns:
Deferred: twisted.internet.defer.Deferred item
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> def run(reactor):
... url = get_path('quote.json')
... conf = {
... 'url': url, 'currency': 'USD', 'delay': 0, 'precision': 6}
... item = {'content': 'GBP'}
... objconf = Objectify(conf)
... kwargs = {'stream': item, 'assign': 'content'}
... d = async_parser(item['content'], objconf, **kwargs)
... return d.addCallbacks(print, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
1.275201
"""
same_currency = base == objconf.currency
if skip:
rate = kwargs['stream']
elif same_currency:
rate = Decimal(1)
elif objconf.url.startswith('http'):
r = yield treq.get(objconf.url, params=objconf.params)
json = yield treq.json(r)
else:
url = get_abspath(objconf.url)
content = yield io.async_url_read(url, delay=objconf.delay)
json = loads(decode(content))
if not (skip or same_currency):
places = Decimal(10) ** -objconf.precision
rates = parse_response(json)
rate = calc_rate(base, objconf.currency, rates, places=places)
return_value(rate)
def async_parser(base, objconf, skip=False, **kwargs):
""" Asynchronously parses the pipe content
Args:
base (str): The base currency (exchanging from)
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
assign (str): Attribute to assign parsed content (default: exchangerate)
stream (dict): The original item
Returns:
Deferred: twisted.internet.defer.Deferred item
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> def run(reactor):
... url = get_path('quote.json')
... conf = {
... 'url': url, 'currency': 'USD', 'delay': 0, 'precision': 6}
... item = {'content': 'GBP'}
... objconf = Objectify(conf)
... kwargs = {'stream': item, 'assign': 'content'}
... d = async_parser(item['content'], objconf, **kwargs)
... return d.addCallbacks(print, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
1.545801
"""
same_currency = base == objconf.currency
if skip:
rate = kwargs['stream']
elif same_currency:
rate = Decimal(1)
elif objconf.url.startswith('http'):
r = yield treq.get(objconf.url, params=objconf.params)
json = yield treq.json(r)
else:
url = get_abspath(objconf.url)
content = yield io.async_url_read(url, delay=objconf.delay)
json = loads(decode(content))
if not (skip or same_currency):
places = Decimal(10) ** -objconf.precision
rates = parse_response(json)
rate = calc_rate(base, objconf.currency, rates, places=places)
return_value(rate)
def get_text(html, convert_charrefs=False):
try:
parser = LinkParser(convert_charrefs=convert_charrefs)
except TypeError:
parser = LinkParser()
try:
parser.feed(html)
except TypeError:
parser.feed(decode(html))
return parser.data.getvalue()
def get_text(html, convert_charrefs=False):
try:
parser = LinkParser(convert_charrefs=convert_charrefs)
except TypeError:
parser = LinkParser()
try:
parser.feed(html)
except TypeError:
parser.feed(decode(html))
return parser.data.getvalue()
def get_abspath(url):
url = 'http://%s' % url if url and '://' not in url else url
if url and url.startswith('file:///'):
# already have an abspath
pass
elif url and url.startswith('file://'):
parent = p.dirname(p.dirname(__file__))
rel_path = url[7:]
abspath = p.abspath(p.join(parent, rel_path))
url = 'file://%s' % abspath
return decode(url)
def get_abspath(url):
url = 'http://%s' % url if url and '://' not in url else url
if url and url.startswith('file:///'):
# already have an abspath
pass
elif url and url.startswith('file://'):
parent = p.dirname(p.dirname(__file__))
rel_path = url[7:]
abspath = p.abspath(p.join(parent, rel_path))
url = 'file://%s' % abspath
return decode(url)
def parse_rss(url=None, **kwargs):
try:
f = fetch(decode(url), **kwargs)
except (ValueError, URLError):
parsed = rssparser.parse(url)
else:
content = f.read() if speedparser else f
try:
parsed = rssparser.parse(content)
finally:
f.close()
return parsed
def parse_rss(url=None, **kwargs):
try:
f = fetch(decode(url), **kwargs)
except (ValueError, URLError):
parsed = rssparser.parse(url)
else:
content = f.read() if speedparser else f
try:
parsed = rssparser.parse(content)
finally:
f.close()
return parsed
def opener(url,
memoize=False,
delay=0,
encoding=ENCODING,
params=None,
**kwargs):
params = params or {}
timeout = kwargs.get('timeout')
decode = kwargs.get('decode')
if url.startswith('http') and params:
r = requests.get(url, params=params, stream=True)
r.raw.decode_content = decode
response = r.text if memoize else r.raw
else:
req = Request(url, headers={'User-Agent': default_user_agent()})
context = SleepyDict(delay=delay) if delay else None
try:
r = urlopen(req, context=context, timeout=timeout)
except TypeError:
r = urlopen(req, timeout=timeout)
except HTTPError as e:
raise URLError(f'{url} returned {e.code}: {e.reason}')
except URLError as e:
raise URLError(f'{url}: {e.reason}')
text = r.read() if memoize else None
if decode:
encoding = get_response_encoding(r, encoding)
if text:
response = compat.decode(text, encoding)
else:
response = reencode(r.fp, encoding, decode=True)
response.r = r
else:
response = text or r
content_type = get_response_content_type(r)
return (response, content_type)
def dataReceived(self, data):
stateTable = self._build_state_table()
self.encoding = self.encoding or detect(data)['encoding']
self.check_encoding(data)
self.state = self.state or 'begin'
content = decode(data, self.encoding)
# bring state, lineno, colno into local scope
lineno, colno = self.lineno, self.colno
curState = self.state
# replace saveMark with a nested scope function
saveMark = lambda: (lineno, colno)
self.saveMark, _saveMark = saveMark, self.saveMark
# fetch functions from the stateTable
beginFn, doFn, endFn = stateTable[curState]
try:
for char in content:
# do newline stuff
if char == '\n':
lineno += 1
colno = 0
else:
colno += 1
newState = doFn(char)
if newState and newState != curState:
# this is the endFn from the previous state
endFn()
curState = newState
beginFn, doFn, endFn = stateTable[curState]
beginFn(char)
finally:
self.saveMark = _saveMark
self.lineno, self.colno = lineno, colno
# state doesn't make sense if there's an exception..
self.state = curState
def dataReceived(self, data):
stateTable = self._build_state_table()
self.encoding = self.encoding or detect(data)['encoding']
self.check_encoding(data)
self.state = self.state or 'begin'
content = decode(data, self.encoding)
# bring state, lineno, colno into local scope
lineno, colno = self.lineno, self.colno
curState = self.state
# replace saveMark with a nested scope function
saveMark = lambda: (lineno, colno)
self.saveMark, _saveMark = saveMark, self.saveMark
# fetch functions from the stateTable
beginFn, doFn, endFn = stateTable[curState]
try:
for char in content:
# do newline stuff
if char == '\n':
lineno += 1
colno = 0
else:
colno += 1
newState = doFn(char)
if newState and newState != curState:
# this is the endFn from the previous state
endFn()
curState = newState
beginFn, doFn, endFn = stateTable[curState]
beginFn(char)
finally:
self.saveMark = _saveMark
self.lineno, self.colno = lineno, colno
# state doesn't make sense if there's an exception..
self.state = curState
def open(self, url, **params):
if url.startswith('http') and params:
r = requests.get(url, params=params, stream=True)
r.raw.decode_content = self.decode
response = r.text if self.cache_type else r.raw
else:
req = Request(url, headers={'User-Agent': default_user_agent()})
try:
r = urlopen(req, context=self.context, timeout=self.timeout)
except TypeError:
r = urlopen(req, timeout=self.timeout)
except HTTPError as e:
msg = '{} returned {}: {}'
raise URLError(msg.format(url, e.code, e.reason))
except URLError as e:
raise URLError('{}: {}'.format(url, e.reason))
text = r.read() if self.cache_type else None
if self.decode:
encoding = get_response_encoding(r, self.def_encoding)
if text:
response = decode(text, encoding)
else:
response = reencode(r.fp, encoding, decode=True)
else:
response = text or r
content_type = get_response_content_type(r)
if 'xml' in content_type:
self.ext = 'xml'
elif 'json' in content_type:
self.ext = 'json'
else:
self.ext = content_type.split('/')[1].split(';')[0]
self.r = r
return response
def open(self, url, **params):
if url.startswith('http') and params:
r = requests.get(url, params=params, stream=True)
r.raw.decode_content = self.decode
response = r.text if self.cache_type else r.raw
else:
try:
r = urlopen(url, context=self.context, timeout=self.timeout)
except TypeError:
r = urlopen(url, timeout=self.timeout)
except HTTPError as e:
msg = '{} returned {}: {}'
raise URLError(msg.format(url, e.code, e.reason))
except URLError as e:
raise URLError('{}: {}'.format(url, e.reason))
text = r.read() if self.cache_type else None
if self.decode:
encoding = get_response_encoding(r, self.def_encoding)
if text:
response = decode(text, encoding)
else:
response = reencode(r.fp, encoding, decode=True)
else:
response = text or r
content_type = get_response_content_type(r)
if 'xml' in content_type:
self.ext = 'xml'
elif 'json' in content_type:
self.ext = 'json'
else:
self.ext = content_type.split('/')[1].split(';')[0]
self.r = r
return response
def file2entries(f, parser):
for line in f:
parser.feed(decode(line))
for entry in parser.entry:
yield entry
def handle_data(self, data):
self.data.write('%s\n' % decode(data))
def file2entries(f, parser):
for line in f:
parser.feed(decode(line))
for entry in parser.entry:
yield entry
def handle_data(self, data):
self.data.write('%s\n' % decode(data))
