def test_put_public_on_private_user_stays_private(self): private_follower = RawFollower(**{'id': 'test', 'is_private': True}) self.target.put(private_follower) public_follower = RawFollower(**{'id': 'test'}) self.target.put(public_follower) stored = self.target.get('test') assert stored is not None assert stored.is_private
def test_get_public_users(self): private_follower = RawFollower(**{'id': 'test_1', 'is_private': True}) self.target.put(private_follower) public_follower = RawFollower(**{'id': 'test_2'}) self.target.put(public_follower) stored = self.target.get_public_users() assert stored is not None assert stored == {'test_2'}
def update_complete_follower(cls, follower, tweet, last_tweet_date): """ Update follower's last download date. """ try: today = datetime.datetime.today() updated_raw_follower = RawFollower( **{ 'id': follower, 'downloaded_on': today, 'last_tweet_date': last_tweet_date, 'is_private': False, 'has_tweets': True }) if 'user' in tweet: user_information = tweet['user'] updated_raw_follower.location = user_information['location'] updated_raw_follower.followers_count = user_information[ 'followers_count'] updated_raw_follower.friends_count = user_information[ 'friends_count'] updated_raw_follower.listed_count = user_information[ 'listed_count'] updated_raw_follower.favourites_count = user_information[ 'favourites_count'] updated_raw_follower.statuses_count = user_information[ 'statuses_count'] RawFollowerDAO().update_follower_data_with_has_tweets( updated_raw_follower) except NonExistentRawFollowerError: cls.get_logger().error(f'Follower {follower} does not exists')
def read_followers_for_candidate(cls, candidate): """ Read .csv file and load followers into database for specific candidate""" if RawFollowerDAO().candidate_was_loaded(candidate.screen_name): cls.get_logger().info( f'Candidate {candidate.screen_name} followers .csv file has already been loaded.' ) return cls.get_logger().info( f'Loading .csv file for {candidate.screen_name}.') # Generate file path and open file path = cls.FOLLOWERS_PATH_FORMAT % candidate.nickname with open(path, 'r') as fd: reader = csv.reader(fd, delimiter=',') # Skip title title = next(reader) # Load followers for row in reader: # There are some cases were we have a second row with a title, so we'll skip it if row == title: continue follower = RawFollower( **{ 'id': row[0], 'downloaded_on': datetime.strptime(row[1], CSVUtils.DATE_FORMAT), 'follows': candidate.screen_name }) RawFollowerDAO().put(follower) # Mark this candidate as already loaded. RawFollowerDAO().finish_candidate(candidate.screen_name) cls.get_logger().info( f'Finished loading {candidate.screen_name} raw followers from .csv file.' )
def load_followers(cls): with open('PUT PATH HERE', 'r') as fd: reader = csv.reader(fd, delimiter=',') for row in reader: follower = RawFollower(**{'_id': row[0], 'downloaded_on': datetime.datetime.strptime(row[1], PreProcessingTweetsUtil.DATE_FORMAT), 'follows': 'prueba'}) RawFollowerDAO().put(follower)
def test_update_raw_follower(self): date = datetime.strptime('1996-03-15', CSVUtils.DATE_FORMAT) raw_follower = RawFollower(**{ 'id': 'test', 'downloaded_on': date, 'follows': 'bodart' }) self.target.put(raw_follower) raw_follower = RawFollower(**{ 'id': 'test', 'downloaded_on': date, 'follows': 'the_commander' }) self.target.put(raw_follower) stored = self.target.get('test') assert stored is not None assert 'bodart' in stored.follows assert 'the_commander' in stored.follows assert stored.downloaded_on == date
def test_get_following_with_cursor(self): # Add many followers for i in range(0, 20): if i % 2 == 0: follower = RawFollower(**{'id': i, 'follows': 'bodart'}) else: follower = RawFollower(**{'id': i, 'follows': 'the_commander'}) self.target.put(follower) # Get first 10 first_10 = self.target.get_following_with_cursor('bodart', 0, 100) assert len(first_10) == 10 assert {follower['id'] for follower in first_10 } == {i for i in range(0, 20) if i % 2 == 0} # Check there are only 10 next_followers = self.target.get_following_with_cursor( 'bodart', 10, 10) assert len(next_followers) == 0
def test_put_new_raw_follower(self): date = datetime.strptime('1996-03-15', CSVUtils.DATE_FORMAT) raw_follower = RawFollower(**{ 'id': 'test', 'downloaded_on': date, 'follows': 'bodart' }) self.target.put(raw_follower) stored = self.target.get('test') assert stored is not None assert stored.follows == ['bodart'] assert stored.downloaded_on == date assert not stored.is_private
def store_new_followers(cls, ids, candidate_name): """ Create RawFollower instances for the received data and store them in the database. Also, we will store the number of new followers downloaded each day. """ today = datetime.today() # Create and store raw followers for follower_id in ids: raw_follower = RawFollower( **{ 'id': follower_id, 'follows': candidate_name, 'downloaded_on': today }) RawFollowerDAO().put(raw_follower) # Store the number of retrieved followers in the current day count = len(ids) CandidatesFollowersDAO().put_increase_for_candidate( candidate_name, count, today)
def update_follower_with_first_tweet(cls, follower, tweet): try: follower_result = RawFollowerDAO().get(follower) today = datetime.datetime.today() user_information = tweet['user'] updated_raw_follower = RawFollower(**{'id': follower, 'follows': follower_result.follows, 'downloaded_on': today, 'location': user_information['location'], 'followers_count': user_information['followers_count'], 'friends_count': user_information['friends_count'], 'listed_count': user_information['listed_count'], 'favourites_count': user_information['favourites_count'], 'statuses_count': user_information['statuses_count'] }) RawFollowerDAO().put(updated_raw_follower) except NonExistentRawFollowerError: cls.get_logger().error(f'Follower {follower} does not exists')
def update_follower_as_private(cls, follower): """ When an error occurs, follower is tagged as private. """ try: # Retrieve the follower from DB today = datetime.datetime.today() updated_raw_follower = RawFollower(**{ 'id': follower, 'downloaded_on': today, 'is_private': True }) RawFollowerDAO().update_follower_data_without_has_tweets( updated_raw_follower) # cls.get_logger().info(f'{follower} is tagged as private.') except NonExistentRawFollowerError as error: cls.get_logger().error( f'{follower} can not be tagged as private because does not exists.' ) cls.get_logger().error(error)
def test_get_all_with_cursor(self): # Add many followers for i in range(0, 20): self.target.put(RawFollower(**{'id': i})) # Get first 10 first_10 = self.target.get_all_with_cursor(0, 10) assert len(first_10) == 10 for follower in first_10: assert follower['id'] < 10 # Get last 10 last_10 = self.target.get_all_with_cursor(10, 10) assert len(last_10) == 10 for follower in last_10: assert 10 <= follower['id'] < 20 # Check there are no overlaps assert {follower['id'] for follower in last_10 }.intersection({follower['id'] for follower in first_10}) == set()
def test_get_public_users_empty(self): # This should never happen anyway private_follower = RawFollower(**{'id': 'test_1', 'is_private': True}) self.target.put(private_follower) stored = self.target.get_public_users() assert not stored
def test_get_candidates_followers_ids(self): for i in range(20): self.target.put(RawFollower(**{'id': i, 'follows': 'bodart'})) result = self.target.get_candidate_followers_ids('bodart') assert len(result) == 20 assert {i for i in range(20)} == result
def test_tag_as_private_ok(self): public_follower = RawFollower(**{'id': 'test'}) self.target.put(public_follower) self.target.tag_as_private(public_follower) stored = self.target.get('test') assert stored.is_private
def get_twitter_mock(): return RawFollower(**{'id': 'dummy', 'is_private': False})
def get_mock_follower_not_private(): return RawFollower(**{'id': 'dummy', 'is_private': False})
def get_mock_follower_private(): return RawFollower(**{'id': 'dummy', 'is_private': True})