def main(): parser = argparse.ArgumentParser(description='Yatcobot: a bot for entering twitter contests') parser.add_argument('--config', '-c', dest='config', default='config.yaml', help='Path of the config file') parser.add_argument('--ignore_list', '-i', dest='ignore_list', default='ignorelist', help='Path of the ignore file') parser.add_argument('--log', dest='logfile', default=None, help='Path of log file') parser.add_argument('--test-mail', action='store_true', dest='test_mail', default=False, help='Test mail settings') parser.add_argument('--debug', dest='debug', action='store_true', help='Enable debug') args = parser.parse_args() # Create logger if args.debug: create_logger(logging.DEBUG, args.logfile) else: create_logger(logging.INFO, args.logfile) # Check for old config if args.config.endswith('.json') or (os.path.isfile('config.json') and not os.path.isfile('config.yaml')): logger.error("Config file format changed, please update your config to the new yaml format!") logger.error("Visit documentation for more info: https://yatcobot.readthedocs.io/en/master/config.html") exit(1) logger.info("Loading configuration") TwitterConfig.load(args.config) logger.info("Configuration loaded") # Test mail settings and exit if args.test_mail: MailNotifier.from_config().test() exit(1) logger.info("Starting") print_logo() bot = Yatcobot(args.ignore_list) bot.run()
def test_enabled(self): TwitterConfig.get( )['search']['filter']['min_retweets']['enabled'] = True self.assertTrue(self.method.is_enabled()) TwitterConfig.get( )['search']['filter']['min_retweets']['enabled'] = False self.assertFalse(self.method.is_enabled())
def test_enabled(self): TwitterConfig.get( )['search']['sort']['by_retweets_count']['enabled'] = True self.assertTrue(self.method.is_enabled()) TwitterConfig.get( )['search']['sort']['by_retweets_count']['enabled'] = False self.assertFalse(self.method.is_enabled())
def test_get_enabled(self): for action in TwitterConfig.get().actions.values(): action['enabled'] = True self.assertEqual(len(ActionABC.get_enabled(self.client)), len(TwitterConfig.get().actions)) for action in TwitterConfig.get().actions.values(): action['enabled'] = False self.assertEqual(len(ActionABC.get_enabled(self.client)), 0)
def test_get_enabled(self): for method in TwitterConfig.get().search.sort.values(): method['enabled'] = True self.assertEqual(len(RatingABC.get_enabled()), len(TwitterConfig.get().search.sort)) for method in TwitterConfig.get().search.sort.values(): method['enabled'] = False self.assertEqual(len(RatingABC.get_enabled()), 0)
def test_remove_oldest_follow_full(self): follows = [ x for x in range( TwitterConfig.get()['actions']['follow']['max_following'] + 1) ] self.client.get_friends_ids.return_value = follows self.action.remove_oldest_follow() self.client.unfollow.assert_called_with( TwitterConfig.get()['actions']['follow']['max_following'])
def get_friends_required(self, post): text = preprocess_text(post['full_text']) # Create keyword mutations tag_keywords = create_keyword_mutations( *TwitterConfig.get().actions.tag_friend.tag_keywords) friend_keywords = create_keyword_mutations( *TwitterConfig.get().actions.tag_friend.friend_keywords) # Find all occurrences of the keywords tag_keywords_found = sorted( {i for x in tag_keywords for i in self.find_all(x, text)}) friend_keywords_found = sorted( {i for x in friend_keywords for i in self.find_all(x, text)}) # Remove indexes of friend keyword that are before any tag keyword friend_keywords_found = [ x for x in friend_keywords_found if x > min(tag_keywords_found) ] # Create all combinations between occurrences indexes = list(product(tag_keywords_found, friend_keywords_found)) # Find where the two keywords are closest closest_pair = [ x for x in sorted(indexes, key=lambda x: x[1] - x[0]) if x[1] - x[0] > 0 ] if not closest_pair: raise ValueError("Could not find substring") closest_pair = closest_pair[0] substring = text[closest_pair[0]:closest_pair[1]] # Split substring to words and remove empty substring = list(filter(None, substring.split(' '))) if len(substring) != 2: raise ValueError('Could not find how many tag are needed') amount = substring[1] for number, keywords in TwitterConfig.get( ).actions.tag_friend.number_keywords.items(): if amount in keywords: return number raise ValueError('Could not determinate how many tags are needed')
def test_filter_keywords_empty(self): TwitterConfig.get()['search']['filter']['blacklist']['enabled'] = True TwitterConfig.get()['search']['filter']['blacklist']['keywords'] = [] posts = { 1: create_post(id=1, full_text='should be fine'), 2: create_post(id=2, full_text='contains keyword'), } queue = PostQueue(posts) self.method.filter(queue) self.assertEqual(len(queue), 2)
def test_follow_with_remove_oldest(self): TwitterConfig.get()['actions']['follow']['keywords'] = [' follow '] post = ( {'id': 0, 'full_text': 'test follow tests', 'user': {'id': random.randint(1, 1000), 'screen_name': 'test'}, 'retweeted': False}) follows = [x for x in range(TwitterConfig.get()['actions']['follow']['max_following'] + 1)] self.client.get_friends_ids.return_value = follows self.action.process(post) self.client.follow.assert_called_once_with(post['user']['screen_name']) self.client.unfollow.assert_called_with(TwitterConfig.get()['actions']['follow']['max_following'])
def test_filter_queue(self): TwitterConfig.get()['search']['filter']['min_retweets']['enabled'] = True TwitterConfig.get()['search']['filter']['min_retweets']['number'] = 5 posts = dict() for i in range(10): post = create_post(retweets=i) posts[post['id']] = post queue = PostQueue(posts) queue.filter() for post in queue.values(): self.assertGreaterEqual(post['retweet_count'], 5)
def tag_needed(self, post): text = preprocess_text(post['full_text']) tag_keywords = create_keyword_mutations( *TwitterConfig.get().actions.tag_friend.tag_keywords) if not any(x in text for x in tag_keywords): return False friend_keywords = create_keyword_mutations( *TwitterConfig.get().actions.tag_friend.friend_keywords) if not any(x in text for x in friend_keywords): return False return True
def process(self, post): text = preprocess_text(post['full_text']) keywords = create_keyword_mutations( *TwitterConfig.get().actions.follow.keywords) if any(x in text.lower() for x in keywords): self.remove_oldest_follow() self.follow(post)
def process(self, post): text = preprocess_text(post['full_text']) keywords = create_keyword_mutations( *TwitterConfig.get().actions.favorite.keywords) if any(x in text.lower() for x in keywords): r = self.client.favorite(post['id']) logger.info("Favorite: {0}".format(post['id']))
def test_remove_oldest_follow_empty(self): follows = [ x for x in range( TwitterConfig.get()['actions']['follow']['max_following'] - 1) ] self.client.get_friends_ids.return_value = follows self.action.remove_oldest_follow() self.assertFalse(self.client.unfollow.called)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if TwitterConfig.get().actions.tag_friend.enabled: logger.warning( 'Experimental feature actions.tag_friend is enabled.') logger.warning( 'If unwanted behavior is observed, please open an issue on github along with the post id!' )
def test_no_follow(self): TwitterConfig.get()['actions']['follow']['keywords'] = [' follow '] post = ( {'id': 0, 'full_text': 'test tests', 'user': {'id': random.randint(1, 1000), 'screen_name': 'test'}, 'retweeted': False}) self.action.process(post) self.assertFalse(self.client.follow.called)
def user_is_blacklisted(self, post): user = post['user']['screen_name'].lower().strip() for blacklisted_user in TwitterConfig.get( ).search.filter.blacklist.users: if user == blacklisted_user.lower().strip(): logger.info( f"Skipping {post['id']} because user {user} is blacklisted" ) return True return False
def test_filter_users(self): TwitterConfig.get()['search']['filter']['blacklist']['enabled'] = True TwitterConfig.get()['search']['filter']['blacklist']['users'] = [ 'bad_user' ] posts = { 1: create_post(id=1, screen_name='good_user'), 2: create_post(id=2, screen_name='Bad_user'), } queue = PostQueue(posts) self.method.filter(queue) self.assertEqual(len(queue), 1) for post_id, post in queue.items(): self.assertNotEqual('bad_user', post['user']['screen_name'])
def test_filter_keywords(self): TwitterConfig.get()['search']['filter']['blacklist']['enabled'] = True TwitterConfig.get()['search']['filter']['blacklist']['keywords'] = [ 'keyword' ] posts = { 1: create_post(id=1, full_text='should be fine'), 2: create_post(id=2, full_text='contains keyword'), } queue = PostQueue(posts) self.method.filter(queue) self.assertEqual(len(queue), 1) for post_id, post in queue.items(): self.assertNotIn('keyword', post['full_text'])
def test_multiple_follows(self): TwitterConfig.get()['actions']['follow']['multiple'] = True post = get_fixture('post_multiple_mentions.json') self.action.process(post) self.assertEqual(self.client.follow.call_count, 2) for user in post['entities']['user_mentions']: self.client.follow.assert_any_call(user['screen_name'])
def remove_oldest_follow(self): """ If the follow limit is reached, unfollow the oldest follow """ follows = self.client.get_friends_ids() if len(follows) > TwitterConfig.get().actions.follow.max_following: r = self.client.unfollow(follows[-1]) logger.info('Unfollowed: {0}'.format(r['screen_name']))
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if TwitterConfig.get().actions.follow.multiple: logger.warning( 'Experimental feature actions.follow.multiple is enabled.') logger.warning('Will follow every mentioned user in a post.') logger.warning( 'If unwanted behavior is observed, please open an issue on github along with the post id!' )
def tag_friends(self, post, number): if len(TwitterConfig.get().actions.tag_friend.friends) < number: raise TagFriend.NotEnoughFriends('Not enough friends') # Copy friends list friends = list(TwitterConfig.get().actions.tag_friend.friends) # Randomize order random.shuffle(friends) text = '@{}\n'.format(post['user']['screen_name']) for friend in friends[:number]: text += '@{} '.format(friend) logger.info('Responding to {} with text:{}'.format( post['id'], text.replace('\n', ' '))) self.client.update(text, post['id'])
def test_filter(self): TwitterConfig.get( )['search']['filter']['min_retweets']['enabled'] = True TwitterConfig.get()['search']['filter']['min_retweets']['number'] = 10 posts = { 1: create_post(id=1, retweets=1), 2: create_post(id=2, retweets=5), 3: create_post(id=3, retweets=15), 4: create_post(id=4, retweets=20), } queue = PostQueue(posts) self.method.filter(queue) self.assertEqual(len(queue), 2) for post_id, post in queue.items(): self.assertGreaterEqual(post['retweet_count'], 10)
def contains_keyword(self, post): text = preprocess_text(post['full_text']) keywords = create_keyword_mutations( *TwitterConfig.get().search.filter.blacklist.keywords) for keyword in keywords: if count_keyword_in_text(keyword, text) > 0: logger.info( f"Skipping {post['id']} because it contains {keyword} keyword" ) return True return False
def test_favorite(self): self.action = Favorite(self.client) TwitterConfig.get()['actions']['favorite']['keywords'] = [' favorite '] post = ( {'id': 0, 'full_text': 'test favorite tests', 'user': {'id': random.randint(1, 1000), 'screen_name': 'test'}, 'retweeted': False}) self.action.process(post) self.client.favorite.assert_called_once_with(post['id'])
def test_get_keywords_rate(self): TwitterConfig.get()['search']['sort']['by_keywords']['enabled'] = True TwitterConfig.get()['search']['sort']['by_keywords']['keywords'] = [ "Test" ] posts = { 1: create_post(id=1, full_text="Test"), 2: create_post(id=2, full_text="test"), 3: create_post(id=3, full_text="norate"), 4: create_post(id=4, full_text="test test"), } queue = PostQueue(posts) rates = self.method.get_rating(queue) rates = {x.id: x.score for x in rates} self.assertEqual(rates[1], rates[2]) self.assertLess(rates[3], rates[2]) self.assertGreater(rates[4], rates[2])
def follow(self, post): users_followed = list() # If multiple users is enabled follow all users mentioned if TwitterConfig.get().actions.follow.multiple: for user in post['entities']['user_mentions']: self.client.follow(user['screen_name']) users_followed.append(user['screen_name']) logger.info("Follow: {0}".format(user['screen_name'])) # If op not already followed, follow if post['user']['screen_name'] not in users_followed: self.client.follow(post['user']['screen_name']) logger.info("Follow: {0}".format(post['user']['screen_name']))
def filter(self, queue): """ Removes all post that have less rewteets than min_retweets defined at config :param queue: Posts Queue """ ids_to_remove = list() for post_id, post in queue.items(): if post['retweet_count'] < TwitterConfig.get( ).search.filter.min_retweets.number: logger.info('Skipping {} because it has {} retweets'.format( post_id, post['retweet_count'])) ids_to_remove.append(post_id) for post_id in ids_to_remove: del queue[post_id]
def get_rating(self, queue): """ Gets a queue and returns the scores based on the keywords in text :param queue: The queue to score :return:A list of scores [Score] """ rates = [] # Find how many times each keyword appears in the text for post in queue.values(): rate = 0 text = post['full_text'].lower() for keyword in TwitterConfig.get( ).search.sort.by_keywords.keywords: keyword = keyword.lower() rate += count_keyword_in_text(keyword, text) rates.append(Score(post['id'], rate)) return self.normalize_scores(rates)