def parser(_, objconf, skip=False, **kwargs):
""" Parses the pipe content
Args:
_ (None): Ignored
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
Iter[dict]: The stream of items
Examples:
>>> from riko import get_path
>>> from meza.fntools import Objectify
>>>
>>> url = get_path('gigs.json')
>>> objconf = Objectify({'url': url, 'path': 'value.items'})
>>> result = parser(None, objconf, stream={})
>>> result[0]['title'] == 'Business System Analyst'
True
"""
if skip:
stream = kwargs['stream']
else:
url = get_abspath(objconf.url)
ext = splitext(url)[1].lstrip('.')
with closing(urlopen(url)) as f:
stream = any2dict(f, ext, objconf.html5, path=objconf.path)
return stream
def async_parser(_, objconf, skip=False, **kwargs):
""" Asynchronously parses the pipe content
Args:
_ (None): Ignored
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
Iter[dict]: The stream of items
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> def run(reactor):
... callback = lambda x: print(x[0]['title'])
... url = get_path('gigs.json')
... objconf = Objectify({'url': url, 'path': 'value.items'})
... d = async_parser(None, objconf, stream={})
... return d.addCallbacks(callback, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
Business System Analyst
"""
if skip:
stream = kwargs['stream']
else:
url = get_abspath(objconf.url)
ext = p.splitext(url)[1].lstrip('.')
f = yield io.async_url_open(url)
stream = any2dict(f, ext, objconf.html5, path=objconf.path)
f.close()
return_value(stream)
def async_parser(_, objconf, skip=False, **kwargs):
""" Asynchronously parses the pipe content
Args:
_ (None): Ignored
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
Iter[dict]: The stream of items
Examples:
>>> from riko import get_path
>>> from riko.bado import react
>>> from riko.bado.mock import FakeReactor
>>> from meza.fntools import Objectify
>>>
>>> def run(reactor):
... callback = lambda x: print(x[0]['title'])
... url = get_path('gigs.json')
... objconf = Objectify({'url': url, 'path': 'value.items'})
... d = async_parser(None, objconf, stream={})
... return d.addCallbacks(callback, logger.error)
>>>
>>> try:
... react(run, _reactor=FakeReactor())
... except SystemExit:
... pass
...
Business System Analyst
"""
if skip:
stream = kwargs['stream']
else:
url = get_abspath(objconf.url)
ext = p.splitext(url)[1].lstrip('.')
f = yield io.async_url_open(url)
stream = any2dict(f, ext, objconf.html5, path=objconf.path)
f.close()
return_value(stream)
def parser(_, objconf, skip=False, **kwargs):
""" Parses the pipe content
Args:
_ (None): Ignored
objconf (obj): The pipe configuration (an Objectify instance)
skip (bool): Don't parse the content
kwargs (dict): Keyword arguments
Kwargs:
stream (dict): The original item
Returns:
Iter[dict]: The stream of items
Examples:
>>> from riko import get_path
>>> from meza.fntools import Objectify
>>>
>>> url = get_path('gigs.json')
>>> objconf = Objectify({'url': url, 'path': 'value.items'})
>>> result = parser(None, objconf, stream={})
>>> result[0]['title'] == 'Business System Analyst'
True
"""
if skip:
stream = kwargs['stream']
else:
url = get_abspath(objconf.url)
ext = p.splitext(url)[1].lstrip('.')
with fetch(**objconf) as f:
ext = ext or f.ext
stream = any2dict(f, ext, objconf.html5, path=objconf.path)
return stream