class CredentialService(metaclass=Singleton):
CREDENTIALS_PATH = f"{abspath(join(dirname(__file__), '../../..'))}/twitter_credentials.json"
def __init__(self):
self.logger = Logger(self.__class__.__name__)
self.in_use = set()
self.credentials = []
# Load credentials file and create objects to access their elements
try:
with open(CredentialService.CREDENTIALS_PATH, 'r') as file:
loaded = json.load(file)
for value in loaded:
self.credentials.append(Credential(**value))
except IOError:
self.logger.error('Credentials file do not found')
def get_all_credentials_for_service(self, service_id):
""" Return all credentials for a given service. """
self.logger.info(
f'Returning all credentials for service {service_id}.')
# Check if some credential has already been assigned
for credential in self.credentials:
if f"{credential.id}-{service_id}" in self.in_use:
raise CredentialsAlreadyInUseError(service_id)
self.logger.info('Checked credentials')
# Store in the in use set. We iterate twice because the number of credentials is small and it is easier than
# doing rollbacks with the already stored credentials if we need to raise an exception
for credential in self.credentials:
self.in_use.add(f"{credential.id}-{service_id}")
return self.credentials
def get_credential_for_service(self, service_id):
""" Get credential if current service is not using all of the available credentials. """
for credential in self.credentials:
if f"{credential.id}-{service_id}" not in self.in_use:
self.logger.info(
f'Returning credential {credential.id} for service {service_id}.'
)
self.in_use.add(f"{credential.id}-{service_id}")
return credential
raise NoAvailableCredentialsError(service_id)
def get_credential_with_id_for_service(self, credential_id, service_id):
""" Get credential if current service is not using all of the available credentials. """
for credential in self.credentials:
if credential_id == credential.id and f"{credential.id}-{service_id}" not in self.in_use:
self.logger.info(
f'Returning credential {credential.id} for service {service_id}.'
)
self.in_use.add(f"{credential.id}-{service_id}")
return credential
raise NoAvailableCredentialsError(service_id)
def unlock_credential(self, credential_id, service_id):
""" Unlock credential for a given service. """
key = f'{credential_id}-{service_id}'
if key not in self.in_use:
raise CredentialCurrentlyAvailableError(key)
self.logger.info(
f'Unlocking credential {credential_id} for service {service_id}.')
self.in_use.remove(key)
class FollowersQueueService(metaclass=Singleton):
def __init__(self):
self.logger = Logger(self.__class__.__name__)
self.updating_followers = {}
self.priority_updating_followers = {}
self.processing_followers = set()
ConcurrencyUtils().create_lock('followers_for_update_tweets')
def get_followers_to_update(self, followers_to_delete):
# Acquire lock for get the followers
ConcurrencyUtils().acquire_lock('followers_for_update_tweets')
self.logger.info(f'Getting followers to update their tweets. Queue\'s size: {len(self.updating_followers)} ')
followers_to_update = self.try_to_get_priority_followers()
if len(followers_to_update) == 0:
followers_to_update = self.get_followers_with_tweets_to_update()
self.processing_followers.update(set(followers_to_update.keys()))
self.processing_followers = self.processing_followers.difference(followers_to_delete)
ConcurrencyUtils().release_lock('followers_for_update_tweets')
return followers_to_update
def try_to_get_priority_followers(self):
# If we have recent downloaded followers
users_to_update = {}
if len(self.priority_updating_followers) != 0:
self.logger.warning(f'Getting {len(self.priority_updating_followers)} recent downloaded followers.')
users_to_update = self.priority_updating_followers.copy()
self.priority_updating_followers = {}
return users_to_update
def get_followers_with_tweets_to_update(self):
""" Get followers with tweets to update. """
max_users_per_window = ConfigurationManager().get_int('max_users_per_window')
self.check_if_have_followers(max_users_per_window)
# Get the min follower's quantity between length and max_users
min_length = min(max_users_per_window, len(self.updating_followers.keys()))
random_followers_keys = random.sample(self.updating_followers.keys(), min_length)
# Remove selected followers
followers_to_update = {}
for follower in random_followers_keys:
followers_to_update[follower] = self.updating_followers.pop(follower)
return followers_to_update
def check_if_have_followers(self, max_users_per_window):
if len(self.updating_followers) <= 2 * max_users_per_window:
# Retrieve more candidates from db
self.add_followers_to_be_updated()
if len(self.updating_followers) == 0:
SlackHelper().post_message_to_channel(
"No se obtuvieron seguidores de la base de datos.")
self.logger.error('There are not followers to update their tweets.')
raise NoMoreFollowersToUpdateTweetsError()
def add_followers_to_be_updated(self, timedelta=180):
self.logger.info(
f'Adding new followers to update their tweets. Actual size: {str(len(self.updating_followers))}')
followers = RawFollowerDAO().get_random_followers_sample(list(self.processing_followers), timedelta)
new_followers = self.add_followers(followers)
if len(new_followers) == 0:
# If there are no new results
self.logger.error('Can\'t retrieve followers to update their tweets. ')
raise NoMoreFollowersToUpdateTweetsError()
self.updating_followers.update(new_followers)
def add_not_updated_followers_2(self):
self.logger.info(
f'Adding not updated followers.')
self.add_followers_to_be_updated(85)
def add_not_updated_followers_1(self):
self.logger.info(
f'Adding not updated followers.')
self.add_followers_to_be_updated(95)
def add_last_downloaded_followers(self):
self.logger.info('Adding last downloaded followers')
users_to_be_updated = RawFollowerDAO().get_all({
'$and': [
{'has_tweets': {'$exists': False}},
{'is_private': {'$ne': True}}
]})
followers = self.add_followers(users_to_be_updated)
self.priority_updating_followers.update(followers)
self.logger.info('Finishing insertion of last downloaded followers')
def add_followers(self, downloaded):
followers = {}
for follower in downloaded:
date = datetime(2019, 1, 1)
if 'last_tweet_date' in follower and follower['last_tweet_date'] is not None:
date = follower['last_tweet_date']
if date is None:
self.logger.warning(f"None type for: {follower['_id']}")
followers[follower['_id']] = date
self.logger.info(f"Added {len(followers)} to queue.")
return followers
