def test_batch_lookup(self):
fetcher = Fetcher()
lookup_urls = [lookup_url, lookup_url2]
cast_list = fetcher.batch_lookup(lookup_urls)
self.assertEqual(79687345, cast_list[0].itunes_id)
self.assertEqual(256945396, cast_list[1].itunes_id)
def test_fetch(self):
fetcher = Fetcher()
scraper = fetcher.fetch(test_url)
genre = scraper.get_currently_selected_genre()
subgenre = scraper.get_currently_selected_subgenre()
letter = scraper.get_currently_selected_letter()
page = scraper.get_currently_selected_page()
self.assertEqual(genre.string, "Arts")
self.assertEqual(subgenre, None)
self.assertEqual(letter, None)
self.assertEqual(page, None)
def test_lookup(self):
fetcher = Fetcher()
cast = fetcher.lookup(lookup_url)
self.assertEqual(79687345, cast.itunes_id)
def test_404_fetch(self):
fetcher = Fetcher()
self.assertRaises(HTTPError, fetcher.fetch, url_404)
#!/usr/bin/python """ Example script for how to run the crawler """ from podscrape.driver import Driver from podscrape.fetcher import Fetcher from podscrape.output import FileOutput # The url to start scraping from. This is one near the end # according to Podscrape's strategy. # the first url would be: "https://itunes.apple.com/us/genre/podcasts-arts/id1301?mt=2" starting_url = "https://itunes.apple.com/us/genre/podcasts-technology-tech-news/id1448?mt=2&letter=T" # Instantiate a Fetcher so we can make http requests fetcher = Fetcher() # Prepare File Output scraped_info_file = "./scraped_info.csv" lookup_info_file = "./lookup_info.csv" output = FileOutput(scraped_info_file, lookup_info_file) # Initialize the Driver object driver = Driver(starting_url, fetcher, output) # Start crawling! driver.crawl()
