def test_save_and_load(self):
aggr = SongAggregateScraper()
aggr.scrape_from_echonest(BASEDIR,
MIN_NUM_COMMENTS,
MAX_NUM_COMMENTS,
FILTER_LIST,
limit=LIMIT)
good_songs = aggr.get_passing_songs()
aggr.save_to_file('test_file')
# self.assertTrue(aggr.max_num_comments == MAX_NUM_COMMENTS, "aggregator metadata")
# self.assertTrue(aggr.aggregate[3] == 2, "aggregate statistics")
# self.assertTrue(len(good_songs) == 1, "scraper passing songs")
# self.assertEqual(good_songs[0].comments[0], "Who still listening in 2018?", "passing songs comments")
aggr2 = SongAggregateScraper()
self.assertTrue(aggr2.max_num_comments is None, "aggr2 is empty")
aggr2.load_from_file('test_file')
recover_good_songs = aggr2.get_passing_songs()
self.assertEqual(aggr.max_num_comments, aggr2.max_num_comments,
"metadata preserved")
self.assertEqual(aggr.aggregate[3], aggr2.aggregate[3],
"aggregate statistics preserved")
self.assertEqual(len(good_songs[0].comments),
len(recover_good_songs[0].comments),
"passing songs preserved")
self.assertEqual(good_songs[0].comments[0],
recover_good_songs[0].comments[0],
"passing songs comments preserved")
def test_run_filters_simple(self):
""" testing run_filters method with english filter alone """
aggr = SongAggregateScraper()
aggr.load_from_file(SAVEFILE)
passing_songs = aggr.get_passing_songs()
song = passing_songs[2]
filtered_comments = fltr.run_filters([english_filter], song.comments)
self.english_filter_3rd_song(filtered_comments)
def test_run_filters_medium(self):
""" testing english filter with the length filter """
aggr = SongAggregateScraper()
aggr.load_from_file(SAVEFILE)
passing_songs = aggr.get_passing_songs()
song = passing_songs[8]
filtered_comments = fltr.run_filters([english_filter, length_filter],
song.comments)
# for idx, comment in enumerate(filtered_comments):
# print(idx, comment)
self.assertEqual(20, len(filtered_comments), "run_filters 2 filters")
def load_comments():
aggr = SongAggregateScraper()
aggr.load_from_file(SAVEFILE)
passing_songs = aggr.get_passing_songs()
#filter_list = [fltr.english_filter, fltr.length_filter, fltr.youtube_topics_filter]
filter_list = [
fltr.english_filter, fltr.length_filter, fltr.youtube_topics_filter,
fltr.brutish_music_filter
]
print(len(passing_songs))
for song in passing_songs:
filtered_comments = fltr.run_filters(filter_list, song.comments)
for idx, comment in enumerate(filtered_comments):
print(idx, comment)
def test_are_key_words_in_comment(self):
""" Running english filter on a song with Brazilian comments """
aggr = SongAggregateScraper()
aggr.load_from_file(SAVEFILE)
passing_songs = aggr.get_passing_songs()
song = passing_songs[2]
filtered_comments = []
for idx, comment in enumerate(song.comments):
#print(idx, comment)
if english_filter.are_key_words_in_comment(comment):
filtered_comments.append(comment)
# for idx, comment in enumerate(filtered_comments):
# print(idx, comment)
self.assertEqual(len(song.comments), 40, "unfiltered length")
self.english_filter_3rd_song(filtered_comments)
def test_length_filter(self):
""" Running a length filter on a song with lyrics as part of comments"""
aggr = SongAggregateScraper()
aggr.load_from_file(SAVEFILE)
passing_songs = aggr.get_passing_songs()
song = passing_songs[8]
filtered_comments = fltr.run_filters([english_filter], song.comments)
# for idx, comment in enumerate(filtered_comments):
# print(idx, comment)
length_filtered_comments = fltr.run_filters([length_filter],
filtered_comments)
# for idx, comment in enumerate(length_filtered_comments):
# print(idx, comment)
self.assertEqual(26, len(filtered_comments),
"length unfiltered quantity")
self.assertEqual(20, len(length_filtered_comments),
"length filtered quantity")