def parse_mul(args):
global retry_num
obj, url = args
req = Req()
while True:
try:
response = req.get(url)
except (requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError):
# print('\n\033[0;37;41m远程主机强迫关闭了一个现有的连接。\033[0m')
if (retry and (retry_num < retry_max)):
retry_num += 1
# print('第%d次重试' % retry_num)
else:
print('\n爬取失败')
else:
if response.status_code != 200:
parse_mul(url)
return
soup = BeautifulSoup(response.text, features='lxml')
for i in soup.find_all('figure'):
img_id = i['data-wallpaper-id']
urls = 'https://w.wallhaven.cc/full/' + img_id[:
2] + '/wallhaven-' + img_id + '.jpg'
if urls not in obj.url_list:
obj.url_list.append(urls)
progress_bar(obj, '爬取进度')
break
