def test_batch_lookup(self): fetcher = Fetcher() lookup_urls = [lookup_url, lookup_url2] cast_list = fetcher.batch_lookup(lookup_urls) self.assertEqual(79687345, cast_list[0].itunes_id) self.assertEqual(256945396, cast_list[1].itunes_id)
def test_fetch(self): fetcher = Fetcher() scraper = fetcher.fetch(test_url) genre = scraper.get_currently_selected_genre() subgenre = scraper.get_currently_selected_subgenre() letter = scraper.get_currently_selected_letter() page = scraper.get_currently_selected_page() self.assertEqual(genre.string, "Arts") self.assertEqual(subgenre, None) self.assertEqual(letter, None) self.assertEqual(page, None)
def test_lookup(self): fetcher = Fetcher() cast = fetcher.lookup(lookup_url) self.assertEqual(79687345, cast.itunes_id)
def test_404_fetch(self): fetcher = Fetcher() self.assertRaises(HTTPError, fetcher.fetch, url_404)
#!/usr/bin/python """ Example script for how to run the crawler """ from podscrape.driver import Driver from podscrape.fetcher import Fetcher from podscrape.output import FileOutput # The url to start scraping from. This is one near the end # according to Podscrape's strategy. # the first url would be: "https://itunes.apple.com/us/genre/podcasts-arts/id1301?mt=2" starting_url = "https://itunes.apple.com/us/genre/podcasts-technology-tech-news/id1448?mt=2&letter=T" # Instantiate a Fetcher so we can make http requests fetcher = Fetcher() # Prepare File Output scraped_info_file = "./scraped_info.csv" lookup_info_file = "./lookup_info.csv" output = FileOutput(scraped_info_file, lookup_info_file) # Initialize the Driver object driver = Driver(starting_url, fetcher, output) # Start crawling! driver.crawl()