def test_page_files(self): crawler = Crawler(URL,domain) target_links = map(crawler.get_Filename, crawler.get_Torrents_List()) dloaded_links = crawler.download_Page_Files() self.assertEqual(sorted(target_links), sorted(dloaded_links))
n_pages = 2 #Set how many pages should be searched. Zero defaults to all. target_path = "/media/141C-7592/Torrent Crawler/torrents" #Set where to save the torrent files. Leave blank for local directory. files = [] if n_pages!=0: count = 1 while count < abs(n_pages)+1: crawler = Crawler(first_page[:-1]+str(count),domain) if not(crawler.valid): break print "Downloading page %d..." % (count) files+=crawler.download_Page_Files() count+=1 else: count = 1 while True: crawler = Crawler(first_page[:-1]+str(count),domain) if not(crawler.valid): break if crawler.get_Torrents_List==[]: break files+=crawler.download_Page_Files() count+=1 if target_path != "": print "Moving files..." for torrent in files: