def job(url, proxy):
res = safe_http(url,
proxies={
'https': 'https://{}'.format(proxy),
'http': 'http://{}'.format(proxy)
}, want_obj=True, timeout=15)
if res is not None:
check_redirect(res)
store_movie(url, res.text, res.status_code)
else:
bad_proxies.add(proxy)
def get_latest():
"""
分页获取最新的作品链接
:param page:
:return:
"""
ptn_movie_href = '<a class="movie-box.*?" href=".*?/cn/movie/(.*?)">'
source = safe_http(home_url)
m = re.search(ptn_movie_href, source)
return m.group(1) if m is not None else None
def get_latest():
"""
分页获取最新的作品链接
:param page:
:return:
"""
ptn_movie_href = '<a class="movie-box.*?" href=".*?/cn/movie/(.*?)">'
source = safe_http(home_url)
m = re.search(ptn_movie_href, source)
return m.group(1) if m is not None else None
def job(url, proxy):
res = safe_http(url,
proxies={
'https': 'https://{}'.format(proxy),
'http': 'http://{}'.format(proxy)
},
want_obj=True,
timeout=15)
if res is not None:
check_redirect(res)
store_movie(url, res.text, res.status_code)
else:
bad_proxies.add(proxy)
