Example #1
0
 def _get_metadata_url(self, url, item_id):
     return [
         {
             'url': x,
             'referer': url,
             'path': os.path.join(CONFIG.save_path, item_id)
         } for x in SpiderUtil.get_tags(url, CONFIG.xpath2, CONFIG.attribs2)
     ]
Example #2
0
 def _get_index_page(self, id=None, last=None):
     if id:
         url = CONFIG.format_url % (id, last)
     else:
         url = CONFIG.host
         id = 0
     return [
         (x[x.rfind('/')+1:], x)
         for x in SpiderUtil.get_tags(url, CONFIG.xpath1, CONFIG.attribs1)
     ]