def writexml(self, stream, *args, **kwargs): if self.raw: val = decode(self.nodeValue) else: v = decode(self.nodeValue) v = ' '.join(v.split()) if kwargs.get('strip') else v val = escape(v) val = encode(val) stream.write(val)
def writexml(self, stream, *args, **kwargs): if self.raw: val = decode(self.nodeValue) else: v = decode(self.nodeValue) v = ' '.join(v.split()) if kwargs.get('strip') else v val = escape(v) val = encode(val) stream.write(val)
def async_parser(base, objconf, skip=False, **kwargs): """ Asynchronously parses the pipe content Args: base (str): The base currency (exchanging from) objconf (obj): The pipe configuration (an Objectify instance) skip (bool): Don't parse the content kwargs (dict): Keyword arguments Kwargs: assign (str): Attribute to assign parsed content (default: exchangerate) stream (dict): The original item Returns: Deferred: twisted.internet.defer.Deferred item Examples: >>> from riko import get_path >>> from riko.bado import react >>> from riko.bado.mock import FakeReactor >>> from meza.fntools import Objectify >>> >>> def run(reactor): ... url = get_path('quote.json') ... conf = { ... 'url': url, 'currency': 'USD', 'sleep': 0, 'precision': 6} ... item = {'content': 'GBP'} ... objconf = Objectify(conf) ... kwargs = {'stream': item, 'assign': 'content'} ... d = async_parser(item['content'], objconf, **kwargs) ... return d.addCallbacks(print, logger.error) >>> >>> try: ... react(run, _reactor=FakeReactor()) ... except SystemExit: ... pass ... 1.545801 """ same_currency = base == objconf.currency if skip: rate = kwargs['stream'] elif same_currency: rate = Decimal(1) elif objconf.url.startswith('http'): r = yield treq.get(objconf.url, params=objconf.params) json = yield treq.json(r) else: url = get_abspath(objconf.url) content = yield io.async_url_read(url, delay=objconf.sleep) json = loads(decode(content)) if not (skip or same_currency): places = Decimal(10)**-objconf.precision rates = parse_response(json) rate = calc_rate(base, objconf.currency, rates, places=places) return_value(rate)
def async_parser(base, objconf, skip, **kwargs): """ Asynchronously parses the pipe content Args: base (str): The base currency (exchanging from) objconf (obj): The pipe configuration (an Objectify instance) skip (bool): Don't parse the content kwargs (dict): Keyword arguments Kwargs: assign (str): Attribute to assign parsed content (default: exchangerate) stream (dict): The original item Returns: Deferred: twisted.internet.defer.Deferred Tuple of (item, skip) Examples: >>> from riko import get_path >>> from riko.bado import react >>> from riko.bado.mock import FakeReactor >>> from riko.lib.utils import Objectify >>> >>> def run(reactor): ... callback = lambda x: print(x[0]) ... url = get_path('quote.json') ... conf = { ... 'url': url, 'currency': 'USD', 'sleep': 0, 'precision': 6} ... item = {'content': 'GBP'} ... objconf = Objectify(conf) ... kwargs = {'stream': item, 'assign': 'content'} ... d = async_parser(item['content'], objconf, False, **kwargs) ... return d.addCallbacks(callback, logger.error) >>> >>> try: ... react(run, _reactor=FakeReactor()) ... except SystemExit: ... pass ... 1.545801 """ if skip: rate = kwargs['stream'] elif objconf.url.startswith('http'): r = yield treq.get(objconf.url, params=objconf.params) json = yield treq.json(r) else: url = utils.get_abspath(objconf.url) content = yield io.async_url_read(url, delay=objconf.sleep) json = loads(decode(content)) if not skip: places = Decimal(10) ** -objconf.precision rates = parse_response(json) rate = calc_rate(base, objconf.currency, rates, places=places) result = (rate, skip) return_value(result)
def get_text(html, convert_charrefs=False): try: parser = LinkParser(convert_charrefs=convert_charrefs) except TypeError: parser = LinkParser() try: parser.feed(html) except TypeError: parser.feed(decode(html)) return parser.data.getvalue()
def get_text(html, convert_charrefs=False): try: parser = LinkParser(convert_charrefs=convert_charrefs) except TypeError: parser = LinkParser() try: parser.feed(html) except TypeError: parser.feed(decode(html)) return parser.data.getvalue()
def get_abspath(url): url = 'http://%s' % url if url and '://' not in url else url if url and url.startswith('file:///'): # already have an abspath pass elif url and url.startswith('file://'): parent = p.dirname(p.dirname(p.dirname(__file__))) rel_path = url[7:] abspath = p.abspath(p.join(parent, rel_path)) url = 'file://%s' % abspath return decode(url)
def get_abspath(url): url = 'http://%s' % url if url and '://' not in url else url if url and url.startswith('file:///'): # already have an abspath pass elif url and url.startswith('file://'): parent = p.dirname(p.dirname(p.dirname(__file__))) rel_path = url[7:] abspath = p.abspath(p.join(parent, rel_path)) url = 'file://%s' % abspath return decode(url)
def parse_rss(url, delay=0): context = SleepyDict(delay=delay) response = None try: response = urlopen(decode(url), context=context) except TypeError: try: response = urlopen(decode(url)) except (ValueError, URLError): parsed = rssparser.parse(url) except (ValueError, URLError): parsed = rssparser.parse(url) if response: content = response.read() if speedparser else response try: parsed = rssparser.parse(content) finally: response.close() return parsed
def parse_rss(url, delay=0): context = SleepyDict(delay=delay) response = None try: response = urlopen(decode(url), context=context) except TypeError: try: response = urlopen(decode(url)) except (ValueError, URLError): parsed = rssparser.parse(url) except (ValueError, URLError): parsed = rssparser.parse(url) if response: content = response.read() if speedparser else response try: parsed = rssparser.parse(content) finally: response.close() return parsed
def dataReceived(self, data): stateTable = self._build_state_table() self.encoding = self.encoding or detect(data)['encoding'] self.check_encoding(data) self.state = self.state or 'begin' content = decode(data, self.encoding) # bring state, lineno, colno into local scope lineno, colno = self.lineno, self.colno curState = self.state # replace saveMark with a nested scope function saveMark = lambda: (lineno, colno) self.saveMark, _saveMark = saveMark, self.saveMark # fetch functions from the stateTable beginFn, doFn, endFn = stateTable[curState] try: for char in content: # do newline stuff if char == '\n': lineno += 1 colno = 0 else: colno += 1 newState = doFn(char) if newState and newState != curState: # this is the endFn from the previous state endFn() curState = newState beginFn, doFn, endFn = stateTable[curState] beginFn(char) finally: self.saveMark = _saveMark self.lineno, self.colno = lineno, colno # state doesn't make sense if there's an exception.. self.state = curState
def dataReceived(self, data): stateTable = self._build_state_table() self.encoding = self.encoding or detect(data)['encoding'] self.check_encoding(data) self.state = self.state or 'begin' content = decode(data, self.encoding) # bring state, lineno, colno into local scope lineno, colno = self.lineno, self.colno curState = self.state # replace saveMark with a nested scope function saveMark = lambda: (lineno, colno) self.saveMark, _saveMark = saveMark, self.saveMark # fetch functions from the stateTable beginFn, doFn, endFn = stateTable[curState] try: for char in content: # do newline stuff if char == '\n': lineno += 1 colno = 0 else: colno += 1 newState = doFn(char) if newState and newState != curState: # this is the endFn from the previous state endFn() curState = newState beginFn, doFn, endFn = stateTable[curState] beginFn(char) finally: self.saveMark = _saveMark self.lineno, self.colno = lineno, colno # state doesn't make sense if there's an exception.. self.state = curState
def handle_data(self, data): self.data.write('%s\n' % decode(data))
def file2entries(f, parser): for line in f: parser.feed(decode(line)) for entry in parser.entry: yield entry
def gen_entries(f, parser): for line in f: parser.feed(decode(line)) for entry in parser.entry: yield entry
def handle_data(self, data): self.data.write('%s\n' % decode(data))