def update_cookies(site, cookies, cover=False):
if cover:
SessionMgr.set_proxy.clear_cookies(site=site)
SessionMgr.update_cookies(site=site, cookies=cookies)
cookies_path = get_cookies_path(site=site)
SessionMgr.export_cookies(site=site, path=cookies_path)
return SessionMgr.get_cookies(site=site)
def get_comicbook_from_cache(site, comicid=None):
comicbook = ComicBook(site=site, comicid=comicid)
proxy_config = current_app.config.get(ConfigKey.CRAWLER_PROXY, {})
proxy = proxy_config.get(site)
if proxy:
SessionMgr.set_proxy(site=site, proxy=proxy)
cookies_path = get_cookies_path(site=site)
if os.path.exists(cookies_path):
SessionMgr.load_cookies(site=site, path=cookies_path)
return comicbook
def init_session(app):
with app.app_context():
proxy_config = app.config.get(ConfigKey.CRAWLER_PROXY, {})
for site in ComicBook.CRAWLER_CLS_MAP:
proxy = proxy_config.get(site)
if proxy:
SessionMgr.set_proxy(site=site, proxy=proxy)
cookies_path = get_cookies_path(site=site)
if os.path.exists(cookies_path):
SessionMgr.load_cookies(site=site, path=cookies_path)
def init_session(app):
CrawlerBase.DRIVER_PATH = app.config.get('DRIVER_PATH', '')
CrawlerBase.DRIVER_TYPE = app.config.get('DRIVER_TYPE', '')
CrawlerBase.HEADLESS = True
with app.app_context():
proxy_config = app.config.get(ConfigKey.CRAWLER_PROXY, {})
for site in ComicBook.CRAWLER_CLS_MAP:
proxy = proxy_config.get(site)
if proxy:
SessionMgr.set_proxy(site=site, proxy=proxy)
cookies_path = get_cookies_path(site=site)
if os.path.exists(cookies_path):
SessionMgr.load_cookies(site=site, path=cookies_path)
def _test_crawl_comicbook(site, comicid=None,
chapter_number=1, proxy=None, test_search=True):
comicbook = ComicBook(site=site, comicid=comicid)
if proxy:
SessionMgr.set_proxy(site=site, proxy=proxy)
comicbook.start_crawler()
chapter = comicbook.Chapter(chapter_number=chapter_number)
assert len(chapter.image_urls) > 0
logger.info(chapter.to_dict())
logger.info(comicbook.to_dict())
if test_search:
result = comicbook.search(name=comicbook.crawler.DEFAULT_SEARCH_NAME)
assert len(result.to_dict()) > 0
return comicbook, chapter
def get_cookies(site):
return SessionMgr.get_cookies(site=site)
def get_proxy(site):
return SessionMgr.get_proxy(site=site)
def set_proxy(site, proxy):
SessionMgr.set_proxy(site=site, proxy=proxy)
return get_proxy(site)