def test_strip_tweets_strips_many_tweets_totally(self): collectionone = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid']) iterator = collectionone.strip_tweets([]).get_iterator() first_tweet = next(iterator) second_tweet = next(iterator) #exhaust the iterator len(list(iterator)) self.assertTrue(first_tweet == {} and second_tweet == {})
def test_strip_tweets_strips_many_tweets_totally(self): collectionone = BsonCollection( os.path.dirname(os.path.realpath(__file__)) + '/' + config['bson']['valid']) iterator = collectionone.strip_tweets([]).get_iterator() first_tweet = next(iterator) second_tweet = next(iterator) #exhaust the iterator len(list(iterator)) self.assertTrue(first_tweet == {} and second_tweet == {})
def test_strip_tweets_keeps_fields(self): tweet_parser = TweetParser() collection = BsonCollection(os.path.dirname(os.path.realpath(__file__)) +'/'+ config['bson']['valid']) self.maxDiff = None it = collection.strip_tweets(['id', 'entities.user_mentions', 'user.profile_image_url_https']).get_iterator() def tweets_have_right_keys(iterator, fields): for tweet in iterator: keys = [key for key,value in tweet_parser.flatten_dict(tweet)] for elem in fields: if elem not in keys: return False return True self.assertTrue(tweets_have_right_keys(it, [['id'], ['entities', 'user_mentions'], ['user', 'profile_image_url_https']]))