def __init__(self, session, **fields):
self.session = session
self.next = fields.pop('_next', None)
self.li = fields.pop('_li', None)
self.__url = None
#self.fields = fields # xpaths of the fields applied to li nodes
self.fields = dict( (k,maybe(f)) if callable(f) else f for k,f in fields.items() )
def _values(node, selector):
if callable(selector):
res = (selector(node))
return res if isiterable(res) else [res]
else:
xpath_or_css = maybe(node.xpath, node.css)
return xpath_or_css(selector)
def _resolve(self, domain):
if not hasattr(self, 'query'):
from dns.resolver import query
self._query = ensure_iterable(maybe(query))
return imap(
str,
ifilter(
None,
chain(self._query(domain, 'A'), self._query(domain, 'AAAA'))))
def _value(node, selector):
if callable(selector):
return selector(node)
elif isinstance(selector,(list,tuple)):
for sel in selector:
result = PageIterator._value(node, sel)
if result: return result
return None
else:
xpath_or_css = maybe(node.at_xpath, node.at_css)
return xpath_or_css(selector)
def _values(node, selector):
if callable(selector):
result = (selector(node))
return result if isiterable(result) else [result]
elif isinstance(selector,(list,tuple)):
for sel in selector:
result = PageIterator._values(node, sel)
if result: return result
return []
else:
xpath_or_css = maybe(node.xpath, node.css)
return xpath_or_css(selector)
def _get_geo_code(self, ip_address):
if not hasattr(self, '_country_code_by_addr'):
self._country_code_by_addr = maybe(self.GEOIP.country_code_by_addr)
return self._country_code_by_addr(ip_address)
def _value(node, selector):
if callable(selector):
return selector(node)
else:
xpath_or_css = maybe(node.at_xpath, node.at_css)
return xpath_or_css(selector)
def _resolve(self, domain):
if not hasattr(self, 'query'):
from dns.resolver import query
self._query = ensure_iterable(maybe(query))
return imap(str, ifilter(None, chain(self._query(domain, 'A'),self._query(domain, 'AAAA'))))
def _get_geo_code(self, ip_address):
if not hasattr(self, '_country_code_by_addr'):
self._country_code_by_addr = maybe(self.GEOIP.country_code_by_addr)
return self._country_code_by_addr(ip_address)