def test_mongo_collection_custom_filter_filters(self): collectionone = MongoCollection( config['mongo']['host'], config['mongo']['port'], config['mongo']['user'], config['mongo']['password'], config['mongo']['database'], config['mongo']['collection'] ) full_collection_len = len(list(collectionone.set_limit(10).get_iterator())) def is_tweet_a_retweet(tweet): if 'retweeted' in tweet and tweet['retweeted']: return True else: return False num_retweets = len(list(collectionone.set_limit(10).set_custom_filter(is_tweet_a_retweet).get_iterator())) collectiontwo = MongoCollection( config['mongo']['host'], config['mongo']['port'], config['mongo']['user'], config['mongo']['password'], config['mongo']['database'], config['mongo']['collection'] ) def is_not_a_retweet(tweet): if 'retweeted' in tweet and tweet['retweeted']: return False else: return True num_non_retweets = len(list(collectiontwo.set_limit(10).set_custom_filter(is_not_a_retweet).get_iterator())) #the number of retweets and non retweets should add up to the whole collection self.assertEqual(num_retweets + num_non_retweets, full_collection_len)
def test_pass_in_mongo(self):
mongo_to_pass = pymongo.MongoClient(config['mongo']['host'],
int(config['mongo']['port']))
collection = MongoCollection(config['mongo']['user'],
config['mongo']['password'],
config['mongo']['database'],
config['mongo']['collection'],
passed_mongo=mongo_to_pass)
self.assertTrue(len(list(collection.set_limit(10).get_iterator())) > 0)
def test_iterator_returns_tweets(self):
collection = MongoCollection( \
config['mongo']['host'], \
config['mongo']['port'], \
config['mongo']['user'], \
config['mongo']['password'], \
config['mongo']['database'], \
config['mongo']['collection'] \
)
self.assertTrue(len(list(collection.set_limit(10).get_iterator())) > 0)
def test_iterator_returns_tweets(self): collection = MongoCollection( \ config['mongo']['host'], \ config['mongo']['port'], \ config['mongo']['user'], \ config['mongo']['password'], \ config['mongo']['database'], \ config['mongo']['collection'] \ ) self.assertTrue(len(list(collection.set_limit(10).get_iterator())) > 0)
def test_pass_in_mongo(self): mongo_to_pass = pymongo.MongoClient(config['mongo']['host'], int(config['mongo']['port'])) collection = MongoCollection( config['mongo']['user'], config['mongo']['password'], config['mongo']['database'], config['mongo']['collection'], passed_mongo=mongo_to_pass ) self.assertTrue(len(list(collection.set_limit(10).get_iterator())) > 0)
def test_mongo_collection_custom_filter_filters(self):
collectionone = MongoCollection(config['mongo']['host'],
config['mongo']['port'],
config['mongo']['user'],
config['mongo']['password'],
config['mongo']['database'],
config['mongo']['collection'])
full_collection_len = len(
list(collectionone.set_limit(10).get_iterator()))
def is_tweet_a_retweet(tweet):
if 'retweeted' in tweet and tweet['retweeted']:
return True
else:
return False
num_retweets = len(
list(
collectionone.set_limit(10).set_custom_filter(
is_tweet_a_retweet).get_iterator()))
collectiontwo = MongoCollection(config['mongo']['host'],
config['mongo']['port'],
config['mongo']['user'],
config['mongo']['password'],
config['mongo']['database'],
config['mongo']['collection'])
def is_not_a_retweet(tweet):
if 'retweeted' in tweet and tweet['retweeted']:
return False
else:
return True
num_non_retweets = len(
list(
collectiontwo.set_limit(10).set_custom_filter(
is_not_a_retweet).get_iterator()))
#the number of retweets and non retweets should add up to the whole collection
self.assertEqual(num_retweets + num_non_retweets, full_collection_len)
def test_strip_tweets_keeps_fields(self): tweet_parser = TweetParser() collection = MongoCollection( config['mongo']['host'], config['mongo']['port'], config['mongo']['user'], config['mongo']['password'], config['mongo']['database'], config['mongo']['collection'] ) self.maxDiff = None it = collection.set_limit(10).strip_tweets(['id', 'entities.user_mentions', 'user.profile_image_url_https']).get_iterator() def tweets_have_right_keys(iterator, fields): for tweet in iterator: keys = [key for key,value in tweet_parser.flatten_dict(tweet)] for elem in fields: if elem not in keys: return False return True self.assertTrue(tweets_have_right_keys(it, [['id'], ['entities', 'user_mentions'], ['user', 'profile_image_url_https']]))
def test_strip_tweets_keeps_fields(self):
tweet_parser = TweetParser()
collection = MongoCollection(config['mongo']['host'],
config['mongo']['port'],
config['mongo']['user'],
config['mongo']['password'],
config['mongo']['database'],
config['mongo']['collection'])
self.maxDiff = None
it = collection.set_limit(10).strip_tweets(
['id', 'entities.user_mentions',
'user.profile_image_url_https']).get_iterator()
def tweets_have_right_keys(iterator, fields):
for tweet in iterator:
keys = [key for key, value in tweet_parser.flatten_dict(tweet)]
for elem in fields:
if elem not in keys:
return False
return True
self.assertTrue(
tweets_have_right_keys(it, [['id'], ['entities', 'user_mentions'],
['user', 'profile_image_url_https']]))
