示例#1
0
文件: scraping.py 项目: I-TREND/SASF
 def __init__(self,  session, **fields):
     self.session = session
     self.next = fields.pop('_next', None)
     self.li = fields.pop('_li', None)
     self.__url = None
     #self.fields = fields # xpaths of the fields applied to li nodes
     self.fields = dict( (k,maybe(f)) if callable(f) else f for k,f in  fields.items() )
示例#2
0
文件: scraping.py 项目: pborky/scrap
 def _values(node, selector):
     if callable(selector):
         res = (selector(node))
         return res if isiterable(res) else [res]
     else:
         xpath_or_css = maybe(node.xpath, node.css)
         return xpath_or_css(selector)
示例#3
0
 def _resolve(self, domain):
     if not hasattr(self, 'query'):
         from dns.resolver import query
         self._query = ensure_iterable(maybe(query))
     return imap(
         str,
         ifilter(
             None,
             chain(self._query(domain, 'A'), self._query(domain, 'AAAA'))))
示例#4
0
文件: scraping.py 项目: I-TREND/SASF
 def _value(node, selector):
     if callable(selector):
         return selector(node)
     elif isinstance(selector,(list,tuple)):
         for sel in selector:
             result = PageIterator._value(node, sel)
             if result: return result
         return None
     else:
         xpath_or_css = maybe(node.at_xpath, node.at_css)
         return xpath_or_css(selector)
示例#5
0
文件: scraping.py 项目: I-TREND/SASF
 def _values(node, selector):
     if callable(selector):
         result = (selector(node))
         return result if isiterable(result) else [result]
     elif isinstance(selector,(list,tuple)):
         for sel in selector:
             result = PageIterator._values(node, sel)
             if result: return result
         return []
     else:
         xpath_or_css = maybe(node.xpath, node.css)
         return xpath_or_css(selector)
示例#6
0
 def _get_geo_code(self, ip_address):
     if not hasattr(self, '_country_code_by_addr'):
         self._country_code_by_addr = maybe(self.GEOIP.country_code_by_addr)
     return self._country_code_by_addr(ip_address)
示例#7
0
文件: scraping.py 项目: pborky/scrap
 def _value(node, selector):
     if callable(selector):
         return selector(node)
     else:
         xpath_or_css = maybe(node.at_xpath, node.at_css)
         return xpath_or_css(selector)
示例#8
0
文件: addrutil.py 项目: pborky/scrap
 def _resolve(self, domain):
     if not hasattr(self, 'query'):
         from dns.resolver import query
         self._query = ensure_iterable(maybe(query))
     return imap(str, ifilter(None, chain(self._query(domain, 'A'),self._query(domain, 'AAAA'))))
示例#9
0
文件: addrutil.py 项目: pborky/scrap
 def _get_geo_code(self, ip_address):
     if not hasattr(self, '_country_code_by_addr'):
         self._country_code_by_addr = maybe(self.GEOIP.country_code_by_addr)
     return self._country_code_by_addr(ip_address)