def test_save_and_load(self): aggr = SongAggregateScraper() aggr.scrape_from_echonest(BASEDIR, MIN_NUM_COMMENTS, MAX_NUM_COMMENTS, FILTER_LIST, limit=LIMIT) good_songs = aggr.get_passing_songs() aggr.save_to_file('test_file') # self.assertTrue(aggr.max_num_comments == MAX_NUM_COMMENTS, "aggregator metadata") # self.assertTrue(aggr.aggregate[3] == 2, "aggregate statistics") # self.assertTrue(len(good_songs) == 1, "scraper passing songs") # self.assertEqual(good_songs[0].comments[0], "Who still listening in 2018?", "passing songs comments") aggr2 = SongAggregateScraper() self.assertTrue(aggr2.max_num_comments is None, "aggr2 is empty") aggr2.load_from_file('test_file') recover_good_songs = aggr2.get_passing_songs() self.assertEqual(aggr.max_num_comments, aggr2.max_num_comments, "metadata preserved") self.assertEqual(aggr.aggregate[3], aggr2.aggregate[3], "aggregate statistics preserved") self.assertEqual(len(good_songs[0].comments), len(recover_good_songs[0].comments), "passing songs preserved") self.assertEqual(good_songs[0].comments[0], recover_good_songs[0].comments[0], "passing songs comments preserved")
def test_run_filters_simple(self): """ testing run_filters method with english filter alone """ aggr = SongAggregateScraper() aggr.load_from_file(SAVEFILE) passing_songs = aggr.get_passing_songs() song = passing_songs[2] filtered_comments = fltr.run_filters([english_filter], song.comments) self.english_filter_3rd_song(filtered_comments)
def test_run_filters_medium(self): """ testing english filter with the length filter """ aggr = SongAggregateScraper() aggr.load_from_file(SAVEFILE) passing_songs = aggr.get_passing_songs() song = passing_songs[8] filtered_comments = fltr.run_filters([english_filter, length_filter], song.comments) # for idx, comment in enumerate(filtered_comments): # print(idx, comment) self.assertEqual(20, len(filtered_comments), "run_filters 2 filters")
def load_comments(): aggr = SongAggregateScraper() aggr.load_from_file(SAVEFILE) passing_songs = aggr.get_passing_songs() #filter_list = [fltr.english_filter, fltr.length_filter, fltr.youtube_topics_filter] filter_list = [ fltr.english_filter, fltr.length_filter, fltr.youtube_topics_filter, fltr.brutish_music_filter ] print(len(passing_songs)) for song in passing_songs: filtered_comments = fltr.run_filters(filter_list, song.comments) for idx, comment in enumerate(filtered_comments): print(idx, comment)
def test_are_key_words_in_comment(self): """ Running english filter on a song with Brazilian comments """ aggr = SongAggregateScraper() aggr.load_from_file(SAVEFILE) passing_songs = aggr.get_passing_songs() song = passing_songs[2] filtered_comments = [] for idx, comment in enumerate(song.comments): #print(idx, comment) if english_filter.are_key_words_in_comment(comment): filtered_comments.append(comment) # for idx, comment in enumerate(filtered_comments): # print(idx, comment) self.assertEqual(len(song.comments), 40, "unfiltered length") self.english_filter_3rd_song(filtered_comments)
def test_length_filter(self): """ Running a length filter on a song with lyrics as part of comments""" aggr = SongAggregateScraper() aggr.load_from_file(SAVEFILE) passing_songs = aggr.get_passing_songs() song = passing_songs[8] filtered_comments = fltr.run_filters([english_filter], song.comments) # for idx, comment in enumerate(filtered_comments): # print(idx, comment) length_filtered_comments = fltr.run_filters([length_filter], filtered_comments) # for idx, comment in enumerate(length_filtered_comments): # print(idx, comment) self.assertEqual(26, len(filtered_comments), "length unfiltered quantity") self.assertEqual(20, len(length_filtered_comments), "length filtered quantity")