def get_string(content, start, end):
# TODO: convert relative links to absolute
# TODO: remove the closing tag if using an HTML tag stripped of HTML tags
# TODO: clean html with Tidy
content = encode(content)
start_pos = content.find(encode(start)) if start else 0
right = content[start_pos + (len(start) if start else 0):]
end_pos = right[1:].find(encode(end)) + 1 if end else len(right)
return right[:end_pos] if end_pos > 0 else right
def get_string(content, start, end):
# TODO: convert relative links to absolute
# TODO: remove the closing tag if using an HTML tag stripped of HTML tags
# TODO: clean html with Tidy
content = encode(content)
start_pos = content.find(encode(start)) if start else 0
right = content[start_pos + (len(start) if start else 0):]
end_pos = right[1:].find(encode(end)) + 1 if end else len(right)
return right[:end_pos] if end_pos > 0 else right
def async_url_read(url, timeout=0, **kwargs):
if url.startswith('http'):
content = getPage(encode(url), timeout=timeout)
else:
content = async_read_file(url, StringTransport(), **kwargs)
return content
def async_url_read(url, timeout=0, **kwargs):
if url.startswith('http'):
content = getPage(encode(url), timeout=timeout)
else:
content = async_read_file(url, StringTransport(), **kwargs)
return content
def writexml(self, stream, *args, **kwargs):
if self.raw:
val = decode(self.nodeValue)
else:
v = decode(self.nodeValue)
v = ' '.join(v.split()) if kwargs.get('strip') else v
val = escape(v)
val = encode(val)
stream.write(val)
def writexml(self, stream, *args, **kwargs):
if self.raw:
val = decode(self.nodeValue)
else:
v = decode(self.nodeValue)
v = ' '.join(v.split()) if kwargs.get('strip') else v
val = escape(v)
val = encode(val)
stream.write(val)
def async_url_open(url, timeout=0, **kwargs):
if url.startswith('http'):
page = NamedTemporaryFile(delete=False)
new_url = page.name
yield downloadPage(encode(url), page, timeout=timeout)
else:
page, new_url = None, url
f = yield async_get_file(new_url, StringTransport(), **kwargs)
if page:
page.close()
remove(page.name)
return_value(f)
def async_url_open(url, timeout=0, **kwargs):
if url.startswith('http'):
page = NamedTemporaryFile(delete=False)
new_url = page.name
yield downloadPage(encode(url), page, timeout=timeout)
else:
page, new_url = None, url
f = yield async_get_file(new_url, StringTransport(), **kwargs)
if page:
page.close()
remove(page.name)
return_value(f)
def async_parser(_, objconf, skip=False, **kwargs):
""" Asynchronously parses the pipe content
Args:
_ (None): Ignored
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
assign (str): Attribute to assign parsed content (default: content)
stream (dict): The original item
Returns:
Iter[dict]: The stream of items
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> @coroutine
... def run(reactor):
... xml_url = get_path('ouseful.xml')
... xml_conf = {'url': xml_url, 'xpath': '/rss/channel/item'}
... xml_objconf = Objectify(xml_conf)
... xml_args = (None, xml_objconf)
... html_url = get_path('sciencedaily.html')
... html_conf = {'url': html_url, 'xpath': '/html/head/title'}
... html_objconf = Objectify(html_conf)
... html_args = (None, html_objconf)
... kwargs = {'stream': {}}
...
... try:
... xml_stream = yield async_parser(*xml_args, **kwargs)
... html_stream = yield async_parser(*html_args, **kwargs)
... print(next(xml_stream)['title'][:44])
... print(next(html_stream))
... except Exception as e:
... logger.error(e)
... logger.error(traceback.format_exc())
...
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
Running “Native” Data Wrangling Applications
Help Page -- ScienceDaily
"""
if skip:
stream = kwargs['stream']
else:
url = get_abspath(objconf.url)
ext = splitext(url)[1].lstrip('.')
xml = (ext == 'xml') or objconf.strict
try:
f = yield io.async_url_open(url)
tree = yield util.xml2etree(f, xml=xml)
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
elements = xpath(tree, objconf.xpath)
f.close()
items = map(util.etree2dict, elements)
stringified = ({kwargs['assign']: encode(i)} for i in items)
stream = stringified if objconf.stringify else items
return_value(stream)
def writexml(self, stream, *args, **kwargs):
val = encode(self.data)
stream.write("<!--%s-->" % val)
def parseString(content, *args, **kwargs):
f = BytesIO(encode(content))
return parse(f, *args, **kwargs)
def writexml(self, stream, *args, **kwargs):
val = encode(self.data)
stream.write("<!--%s-->" % val)
def parseString(content, *args, **kwargs):
f = BytesIO(encode(content))
return parse(f, *args, **kwargs)
def async_parser(_, objconf, skip, **kwargs):
""" Asynchronously parses the pipe content
Args:
_ (None): Ignored
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
assign (str): Attribute to assign parsed content (default: content)
stream (dict): The original item
Returns:
Tuple(Iter[dict], bool): Tuple of (stream, skip)
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from riko.lib.utils import Objectify
>>>
>>> @coroutine
... def run(reactor):
... xml_url = get_path('ouseful.xml')
... xml_conf = {'url': xml_url, 'xpath': '/rss/channel/item'}
... xml_objconf = Objectify(xml_conf)
... xml_args = (None, xml_objconf, False)
... html_url = get_path('sciencedaily.html')
... html_conf = {'url': html_url, 'xpath': '/html/head/title'}
... html_objconf = Objectify(html_conf)
... html_args = (None, html_objconf, False)
... kwargs = {'stream': {}}
...
... try:
... xml_stream, _ = yield async_parser(*xml_args, **kwargs)
... html_stream, _ = yield async_parser(*html_args, **kwargs)
... print(next(xml_stream)['title'][:44])
... print(next(html_stream))
... except Exception as e:
... logger.error(e)
... logger.error(traceback.format_exc())
...
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
Running “Native” Data Wrangling Applications
Help Page -- ScienceDaily
"""
if skip:
stream = kwargs['stream']
else:
url = utils.get_abspath(objconf.url)
ext = splitext(url)[1].lstrip('.')
xml = (ext == 'xml') or objconf.strict
try:
f = yield io.async_url_open(url)
tree = yield util.xml2etree(f, xml=xml)
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
elements = utils.xpath(tree, objconf.xpath)
f.close()
items = map(util.etree2dict, elements)
stringified = ({kwargs['assign']: encode(i)} for i in items)
stream = stringified if objconf.stringify else items
result = (stream, skip)
return_value(result)