def _get_metadata_url(self, url, item_id):
return [
{
'url': x,
'referer': url,
'path': os.path.join(CONFIG.save_path, item_id)
} for x in SpiderUtil.get_tags(url, CONFIG.xpath2, CONFIG.attribs2)
]
def _get_index_page(self, id=None, last=None):
if id:
url = CONFIG.format_url % (id, last)
else:
url = CONFIG.host
id = 0
return [
(x[x.rfind('/')+1:], x)
for x in SpiderUtil.get_tags(url, CONFIG.xpath1, CONFIG.attribs1)
]