def __iter__(self):
from quirks.iterable import count
from urlparse import urljoin
seq = count()
while True:
current_url = self.session.url()
absolute_url = lambda url: urljoin(current_url, url)
process_values = combinator(
lambda (key, value): (key, absolute_url(value) if key == 'url' else value),
)
results = self._values(self.session,self.li)
if not results:
results = []
for result in results :
yield dict( chain(
imap(process_values, ( (key,self._value(result, selector)) for key,selector in self.fields.iteritems() )),
( ('_seq', seq.next()), ('_url', self._url(self.session)) )
) )
if not self.next:
return
next = self._value(self.session,self.next)
if not next:
return
self.__url = None
next.click()
time.sleep(0.5)
self.session.wait()
logger.debug('visited "%s"'%self.session.url() )
def _resolve(self, domain):
if not hasattr(self, 'query'):
from dns.resolver import query
self._query = ensure_iterable(maybe(query))
return imap(
str,
ifilter(
None,
chain(self._query(domain, 'A'), self._query(domain, 'AAAA'))))
def __call__(self, url):
domain = self._get_domain(url)
return dict(
chain(
self._get_ip_addresses(domain).iteritems(),
self._get_whois(domain).iteritems()))
def __call__(self, url):
domain = self._get_domain(url)
return dict(chain(self._get_ip_addresses(domain).iteritems(), self._get_whois(domain).iteritems()))
def _resolve(self, domain):
if not hasattr(self, 'query'):
from dns.resolver import query
self._query = ensure_iterable(maybe(query))
return imap(str, ifilter(None, chain(self._query(domain, 'A'),self._query(domain, 'AAAA'))))
