def parser(_, objconf, skip=False, **kwargs): """ Parses the pipe content Args: _ (None): Ignored objconf (obj): The pipe configuration (an Objectify instance) skip (bool): Don't parse the content kwargs (dict): Keyword arguments Kwargs: assign (str): Attribute to assign parsed content (default: content) stream (dict): The original item Returns: Iter[dict]: The stream of items Examples: >>> from riko import get_path >>> from riko.utils import get_abspath >>> from meza.fntools import Objectify >>> >>> feed = 'http://feeds.feedburner.com/TechCrunch/' >>> url = 'http://query.yahooapis.com/v1/public/yql' >>> query = "select * from feed where url='%s'" % feed >>> conf = {'query': query, 'url': url, 'debug': False} >>> objconf = Objectify(conf) >>> url = get_abspath(get_path('yql.xml')) >>> >>> with fetch(url) as f: ... kwargs = {'stream': {}, 'response': f} ... result = parser(None, objconf, **kwargs) >>> >>> next(result)['title'] 'Bring pizza home' """ if skip: stream = kwargs['stream'] else: f = kwargs.get('response') if not f: params = {'q': objconf.query, 'diagnostics': objconf.debug} if objconf.memoize and not objconf.cache_type: objconf.cache_type = 'auto' f = fetch(params=params, **objconf) # TODO: consider paging for large result sets root = xml2etree(f).getroot() results = root.find('results') stream = map(etree2dict, results) return stream
def parser(_, objconf, skip=False, **kwargs): """ Parses the pipe content Args: _ (None): Ignored objconf (obj): The pipe configuration (an Objectify instance) skip (bool): Don't parse the content kwargs (dict): Keyword arguments Kwargs: assign (str): Attribute to assign parsed content (default: content) stream (dict): The original item Returns: Iter[dict]: The stream of items Examples: >>> from riko import get_path >>> from riko.utils import get_abspath >>> from meza.fntools import Objectify >>> >>> feed = 'http://feeds.feedburner.com/TechCrunch/' >>> url = 'http://query.yahooapis.com/v1/public/yql' >>> query = "select * from feed where url='%s'" % feed >>> conf = {'query': query, 'url': url, 'debug': False} >>> objconf = Objectify(conf) >>> url = get_abspath(get_path('yql.xml')) >>> >>> with fetch(url) as f: ... kwargs = {'stream': {}, 'response': f} ... result = parser(None, objconf, **kwargs) >>> >>> next(result)['title'] 'Bring pizza home' """ if skip: stream = kwargs['stream'] else: f = kwargs.get('response') if not f: params = {'q': objconf.query, 'diagnostics': objconf.debug} if objconf.memoize and not objconf.cache_type: objconf.cache_type = 'auto' f = fetch(params=params, **objconf) # TODO: consider paging for large result sets root = xml2etree(f).getroot() results = root.find('results') stream = map(etree2dict, results) return stream
def parser(_, objconf, skip=False, **kwargs): """ Parses the pipe content Args: _ (None): Ignored objconf (obj): The pipe configuration (an Objectify instance) skip (bool): Don't parse the content Returns: Iter[dict]: The stream of items Examples: >>> from meza.fntools import Objectify >>> from riko import get_path >>> >>> url = get_path('ouseful.xml') >>> objconf = Objectify({'url': url, 'xpath': '/rss/channel/item'}) >>> result = parser(None, objconf, stream={}) >>> title = 'Running “Native” Data Wrangling Applications' >>> next(result)['title'][:44] == title True """ if skip: stream = kwargs['stream'] else: url = get_abspath(objconf.url) ext = splitext(url)[1].lstrip('.') xml = (ext == 'xml') or objconf.strict if objconf.memoize and not objconf.cache_type: objconf.cache_type = 'auto' with fetch(**objconf) as f: root = xml2etree(f, xml=xml, html5=objconf.html5).getroot() elements = xpath(root, objconf.xpath) items = map(etree2dict, elements) stringified = ({kwargs['assign']: str(i)} for i in items) stream = stringified if objconf.stringify else items return stream
def parser(_, objconf, skip=False, **kwargs): """ Parses the pipe content Args: _ (None): Ignored objconf (obj): The pipe configuration (an Objectify instance) skip (bool): Don't parse the content Returns: Iter[dict]: The stream of items Examples: >>> from meza.fntools import Objectify >>> from riko import get_path >>> >>> url = get_path('ouseful.xml') >>> objconf = Objectify({'url': url, 'xpath': '/rss/channel/item'}) >>> result = parser(None, objconf, stream={}) >>> title = 'Running “Native” Data Wrangling Applications' >>> next(result)['title'][:44] == title True """ if skip: stream = kwargs['stream'] else: url = get_abspath(objconf.url) ext = splitext(url)[1].lstrip('.') xml = (ext == 'xml') or objconf.strict with fetch(**objconf) as f: root = xml2etree(f, xml=xml, html5=objconf.html5).getroot() elements = xpath(root, objconf.xpath) items = map(etree2dict, elements) stringified = ({kwargs['assign']: str(i)} for i in items) stream = stringified if objconf.stringify else items return stream