def test_put_public_on_private_user_stays_private(self):
private_follower = RawFollower(**{'id': 'test', 'is_private': True})
self.target.put(private_follower)
public_follower = RawFollower(**{'id': 'test'})
self.target.put(public_follower)
stored = self.target.get('test')
assert stored is not None
assert stored.is_private
def test_get_public_users(self):
private_follower = RawFollower(**{'id': 'test_1', 'is_private': True})
self.target.put(private_follower)
public_follower = RawFollower(**{'id': 'test_2'})
self.target.put(public_follower)
stored = self.target.get_public_users()
assert stored is not None
assert stored == {'test_2'}
def update_complete_follower(cls, follower, tweet, last_tweet_date):
""" Update follower's last download date. """
try:
today = datetime.datetime.today()
updated_raw_follower = RawFollower(
**{
'id': follower,
'downloaded_on': today,
'last_tweet_date': last_tweet_date,
'is_private': False,
'has_tweets': True
})
if 'user' in tweet:
user_information = tweet['user']
updated_raw_follower.location = user_information['location']
updated_raw_follower.followers_count = user_information[
'followers_count']
updated_raw_follower.friends_count = user_information[
'friends_count']
updated_raw_follower.listed_count = user_information[
'listed_count']
updated_raw_follower.favourites_count = user_information[
'favourites_count']
updated_raw_follower.statuses_count = user_information[
'statuses_count']
RawFollowerDAO().update_follower_data_with_has_tweets(
updated_raw_follower)
except NonExistentRawFollowerError:
cls.get_logger().error(f'Follower {follower} does not exists')
def read_followers_for_candidate(cls, candidate):
""" Read .csv file and load followers into database for specific candidate"""
if RawFollowerDAO().candidate_was_loaded(candidate.screen_name):
cls.get_logger().info(
f'Candidate {candidate.screen_name} followers .csv file has already been loaded.'
)
return
cls.get_logger().info(
f'Loading .csv file for {candidate.screen_name}.')
# Generate file path and open file
path = cls.FOLLOWERS_PATH_FORMAT % candidate.nickname
with open(path, 'r') as fd:
reader = csv.reader(fd, delimiter=',')
# Skip title
title = next(reader)
# Load followers
for row in reader:
# There are some cases were we have a second row with a title, so we'll skip it
if row == title: continue
follower = RawFollower(
**{
'id':
row[0],
'downloaded_on':
datetime.strptime(row[1], CSVUtils.DATE_FORMAT),
'follows':
candidate.screen_name
})
RawFollowerDAO().put(follower)
# Mark this candidate as already loaded.
RawFollowerDAO().finish_candidate(candidate.screen_name)
cls.get_logger().info(
f'Finished loading {candidate.screen_name} raw followers from .csv file.'
)
def load_followers(cls):
with open('PUT PATH HERE', 'r') as fd:
reader = csv.reader(fd, delimiter=',')
for row in reader:
follower = RawFollower(**{'_id': row[0],
'downloaded_on': datetime.datetime.strptime(row[1],
PreProcessingTweetsUtil.DATE_FORMAT),
'follows': 'prueba'})
RawFollowerDAO().put(follower)
def test_update_raw_follower(self):
date = datetime.strptime('1996-03-15', CSVUtils.DATE_FORMAT)
raw_follower = RawFollower(**{
'id': 'test',
'downloaded_on': date,
'follows': 'bodart'
})
self.target.put(raw_follower)
raw_follower = RawFollower(**{
'id': 'test',
'downloaded_on': date,
'follows': 'the_commander'
})
self.target.put(raw_follower)
stored = self.target.get('test')
assert stored is not None
assert 'bodart' in stored.follows
assert 'the_commander' in stored.follows
assert stored.downloaded_on == date
def test_get_following_with_cursor(self):
# Add many followers
for i in range(0, 20):
if i % 2 == 0:
follower = RawFollower(**{'id': i, 'follows': 'bodart'})
else:
follower = RawFollower(**{'id': i, 'follows': 'the_commander'})
self.target.put(follower)
# Get first 10
first_10 = self.target.get_following_with_cursor('bodart', 0, 100)
assert len(first_10) == 10
assert {follower['id']
for follower in first_10
} == {i
for i in range(0, 20) if i % 2 == 0}
# Check there are only 10
next_followers = self.target.get_following_with_cursor(
'bodart', 10, 10)
assert len(next_followers) == 0
def test_put_new_raw_follower(self):
date = datetime.strptime('1996-03-15', CSVUtils.DATE_FORMAT)
raw_follower = RawFollower(**{
'id': 'test',
'downloaded_on': date,
'follows': 'bodart'
})
self.target.put(raw_follower)
stored = self.target.get('test')
assert stored is not None
assert stored.follows == ['bodart']
assert stored.downloaded_on == date
assert not stored.is_private
def store_new_followers(cls, ids, candidate_name):
""" Create RawFollower instances for the received data and store them in the database. Also, we will store
the number of new followers downloaded each day. """
today = datetime.today()
# Create and store raw followers
for follower_id in ids:
raw_follower = RawFollower(
**{
'id': follower_id,
'follows': candidate_name,
'downloaded_on': today
})
RawFollowerDAO().put(raw_follower)
# Store the number of retrieved followers in the current day
count = len(ids)
CandidatesFollowersDAO().put_increase_for_candidate(
candidate_name, count, today)
def update_follower_with_first_tweet(cls, follower, tweet):
try:
follower_result = RawFollowerDAO().get(follower)
today = datetime.datetime.today()
user_information = tweet['user']
updated_raw_follower = RawFollower(**{'id': follower,
'follows': follower_result.follows,
'downloaded_on': today,
'location': user_information['location'],
'followers_count': user_information['followers_count'],
'friends_count': user_information['friends_count'],
'listed_count': user_information['listed_count'],
'favourites_count': user_information['favourites_count'],
'statuses_count': user_information['statuses_count']
})
RawFollowerDAO().put(updated_raw_follower)
except NonExistentRawFollowerError:
cls.get_logger().error(f'Follower {follower} does not exists')
def update_follower_as_private(cls, follower):
""" When an error occurs, follower is tagged as private. """
try:
# Retrieve the follower from DB
today = datetime.datetime.today()
updated_raw_follower = RawFollower(**{
'id': follower,
'downloaded_on': today,
'is_private': True
})
RawFollowerDAO().update_follower_data_without_has_tweets(
updated_raw_follower)
# cls.get_logger().info(f'{follower} is tagged as private.')
except NonExistentRawFollowerError as error:
cls.get_logger().error(
f'{follower} can not be tagged as private because does not exists.'
)
cls.get_logger().error(error)
def test_get_all_with_cursor(self):
# Add many followers
for i in range(0, 20):
self.target.put(RawFollower(**{'id': i}))
# Get first 10
first_10 = self.target.get_all_with_cursor(0, 10)
assert len(first_10) == 10
for follower in first_10:
assert follower['id'] < 10
# Get last 10
last_10 = self.target.get_all_with_cursor(10, 10)
assert len(last_10) == 10
for follower in last_10:
assert 10 <= follower['id'] < 20
# Check there are no overlaps
assert {follower['id']
for follower in last_10
}.intersection({follower['id']
for follower in first_10}) == set()
def test_get_public_users_empty(self):
# This should never happen anyway
private_follower = RawFollower(**{'id': 'test_1', 'is_private': True})
self.target.put(private_follower)
stored = self.target.get_public_users()
assert not stored
def test_get_candidates_followers_ids(self):
for i in range(20):
self.target.put(RawFollower(**{'id': i, 'follows': 'bodart'}))
result = self.target.get_candidate_followers_ids('bodart')
assert len(result) == 20
assert {i for i in range(20)} == result
def test_tag_as_private_ok(self):
public_follower = RawFollower(**{'id': 'test'})
self.target.put(public_follower)
self.target.tag_as_private(public_follower)
stored = self.target.get('test')
assert stored.is_private
def get_twitter_mock():
return RawFollower(**{'id': 'dummy', 'is_private': False})
def get_mock_follower_not_private():
return RawFollower(**{'id': 'dummy', 'is_private': False})
def get_mock_follower_private():
return RawFollower(**{'id': 'dummy', 'is_private': True})