def fetch_votes(cls, fetch_existing=False): scraper = Scrape(cls._get_title_urls(fetch_existing), cls._fetch_vote_response, max_clients=50) #for hkey, hvalue in cls._get_cookie_headers(): scraper.add_header('Cookie', '; '.join(cls._get_cookie_headers())) scraper.run() cls._scrape_response()
def fetch_ids(cls, title_types, type='title'): cls._type = type if type == 'title': url_source = cls._get_title_urls else: url_source = cls._get_person_urls #url_source = lambda t: iter([('Prowse, David', Produce._person_href(None, ident='Prowse, David'))]) scraper = Scrape(url_source(title_types), cls._fetch_id_response, follow_redirects=False, max_clients=8, delay=1, anon=True) scraper.run()