Esempio n. 1
0
class Bot:
    """
    Main class for the twitch bot
    """

    version = '1.32'
    date_fmt = '%H:%M'
    admin = None
    url_regex_str = r'\(?(?:(http|https):\/\/)?(?:((?:[^\W\s]|\.|-|[:]{1})+)@{1})?((?:www.)?(?:[^\W\s]|\.|-)+[\.][^\W\s]{2,4}|localhost(?=\/)|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?::(\d*))?([\/]?[^\s\?]*[\/]{1})*(?:\/?([^\s\n\?\[\]\{\}\#]*(?:(?=\.)){1}|[^\s\n\?\[\]\{\}\.\#]*)?([\.]{1}[^\s\?\#]*)?)?(?:\?{1}([^\s\n\#\[\]]*))?([\#][^\s\n]*)?\)?'

    last_ping = datetime.datetime.now()
    last_pong = datetime.datetime.now()

    def parse_args():
        parser = argparse.ArgumentParser()
        parser.add_argument('--config', '-c',
                            default='config.ini',
                            help='Specify which config file to use '
                                    '(default: config.ini)')
        parser.add_argument('--silent',
                            action='count',
                            help='Decides whether the bot should be '
                            'silent or not')
        # TODO: Add a log level argument.

        return parser.parse_args()

    def load_config(self, config):
        self.config = config

        pajbot.models.user.Config.se_sync_token = config['main'].get('se_sync_token', None)
        pajbot.models.user.Config.se_channel = config['main'].get('se_channel', None)

        self.domain = config['web'].get('domain', 'localhost')

        self.nickname = config['main'].get('nickname', 'pajbot')
        self.password = config['main'].get('password', 'abcdef')

        self.timezone = config['main'].get('timezone', 'UTC')
        os.environ['TZ'] = self.timezone

        if config['main'].getboolean('verified', False):
            TMI.promote_to_verified()

        self.trusted_mods = config.getboolean('main', 'trusted_mods')

        TimeManager.init_timezone(self.timezone)

        if 'streamer' in config['main']:
            self.streamer = config['main']['streamer']
            self.channel = '#' + self.streamer
        elif 'target' in config['main']:
            self.channel = config['main']['target']
            self.streamer = self.channel[1:]

        self.silent = False
        self.dev = False

        if 'flags' in config:
            self.silent = True if 'silent' in config['flags'] and config['flags']['silent'] == '1' else self.silent
            self.dev = True if 'dev' in config['flags'] and config['flags']['dev'] == '1' else self.dev

        DBManager.init(self.config['main']['db'])

        redis_options = {}
        if 'redis' in config:
            redis_options = config._sections['redis']

        RedisManager.init(**redis_options)

    def __init__(self, config, args=None):
        # Load various configuration variables from the given config object
        # The config object that should be passed through should
        # come from pajbot.utils.load_config
        self.load_config(config)

        # Update the database scheme if necessary using alembic
        # In case of errors, i.e. if the database is out of sync or the alembic
        # binary can't be called, we will shut down the bot.
        pajbot.utils.alembic_upgrade()

        # Actions in this queue are run in a separate thread.
        # This means actions should NOT access any database-related stuff.
        self.action_queue = ActionQueue()
        self.action_queue.start()

        self.reactor = irc.client.Reactor(self.on_connect)
        self.start_time = datetime.datetime.now()
        ActionParser.bot = self

        HandlerManager.init_handlers()

        self.socket_manager = SocketManager(self)
        self.stream_manager = StreamManager(self)

        StreamHelper.init_bot(self, self.stream_manager)
        ScheduleManager.init()

        self.users = UserManager()
        self.decks = DeckManager()
        self.module_manager = ModuleManager(self.socket_manager, bot=self).load()
        self.commands = CommandManager(
                socket_manager=self.socket_manager,
                module_manager=self.module_manager,
                bot=self).load()
        self.filters = FilterManager().reload()
        self.banphrase_manager = BanphraseManager(self).load()
        self.timer_manager = TimerManager(self).load()
        self.kvi = KVIManager()
        self.emotes = EmoteManager(self)
        self.twitter_manager = TwitterManager(self)

        HandlerManager.trigger('on_managers_loaded')

        # Reloadable managers
        self.reloadable = {
                'filters': self.filters,
                }

        # Commitable managers
        self.commitable = {
                'commands': self.commands,
                'filters': self.filters,
                'banphrases': self.banphrase_manager,
                }

        self.execute_every(10 * 60, self.commit_all)
        self.execute_every(1, self.do_tick)

        try:
            self.admin = self.config['main']['admin']
        except KeyError:
            log.warning('No admin user specified. See the [main] section in config.example.ini for its usage.')
        if self.admin:
            with self.users.get_user_context(self.admin) as user: pass
                # user.level = 2000

        self.parse_version()

        relay_host = self.config['main'].get('relay_host', None)
        relay_password = self.config['main'].get('relay_password', None)
        if relay_host is None or relay_password is None:
            self.irc = MultiIRCManager(self)
        else:
            self.irc = SingleIRCManager(self)

        self.reactor.add_global_handler('all_events', self.irc._dispatcher, -10)

        twitch_client_id = None
        twitch_oauth = None
        if 'twitchapi' in self.config:
            twitch_client_id = self.config['twitchapi'].get('client_id', None)
            twitch_oauth = self.config['twitchapi'].get('oauth', None)

        # A client ID is required for the bot to work properly now, give an error for now
        if twitch_client_id is None:
            log.error('MISSING CLIENT ID, SET "client_id" VALUE UNDER [twitchapi] SECTION IN CONFIG FILE')

        self.twitchapi = TwitchAPI(twitch_client_id, twitch_oauth)

        self.data = {}
        self.data_cb = {}
        self.url_regex = re.compile(self.url_regex_str, re.IGNORECASE)

        self.data['broadcaster'] = self.streamer
        self.data['version'] = self.version
        self.data['version_brief'] = self.version_brief
        self.data['bot_name'] = self.nickname
        self.data_cb['status_length'] = self.c_status_length
        self.data_cb['stream_status'] = self.c_stream_status
        self.data_cb['bot_uptime'] = self.c_uptime
        self.data_cb['current_time'] = self.c_current_time

        self.silent = True if args.silent else self.silent

        if self.silent:
            log.info('Silent mode enabled')

        """
        For actions that need to access the main thread,
        we can use the mainthread_queue.
        """
        self.mainthread_queue = ActionQueue()
        self.execute_every(1, self.mainthread_queue.parse_action)

        self.websocket_manager = WebSocketManager(self)

        try:
            if self.config['twitchapi']['update_subscribers'] == '1':
                self.execute_every(30 * 60,
                                   self.action_queue.add,
                                   (self.update_subscribers_stage1, ))
        except:
            pass

        # XXX: TEMPORARY UGLY CODE
        HandlerManager.add_handler('on_user_gain_tokens', self.on_user_gain_tokens)
        HandlerManager.add_handler('send_whisper', self.whisper)

    def on_connect(self, sock):
        return self.irc.on_connect(sock)

    def on_user_gain_tokens(self, user, tokens_gained):
        self.whisper(user.username, 'You finished todays quest! You have been awarded with {} tokens.'.format(tokens_gained))

    def update_subscribers_stage1(self):
        limit = 25
        offset = 0
        subscribers = []
        log.info('Starting stage1 subscribers update')

        try:
            retry_same = 0
            while True:
                log.debug('Beginning sub request {0} {1}'.format(limit, offset))
                subs, retry_same, error = self.twitchapi.get_subscribers(self.streamer, 0, offset, 0 if retry_same is False else retry_same)
                log.debug('got em!')

                if error is True:
                    log.error('Too many attempts, aborting')
                    return False

                if retry_same is False:
                    offset += limit
                    if len(subs) == 0:
                        # We don't need to retry, and the last query finished propery
                        # Break out of the loop and start fiddling with the subs!
                        log.debug('Done!')
                        break
                    else:
                        log.debug('Fetched {0} subs'.format(len(subs)))
                        subscribers.extend(subs)

                if retry_same is not False:
                    # In case the next attempt is a retry, wait for 3 seconds
                    log.debug('waiting for 3 seconds...')
                    time.sleep(3)
                    log.debug('waited enough!')
            log.debug('Finished with the while True loop!')
        except:
            log.exception('Caught an exception while trying to get subscribers')
            return

        log.info('Ended stage1 subscribers update')
        if len(subscribers) > 0:
            log.info('Got some subscribers, so we are pushing them to stage 2!')
            self.mainthread_queue.add(self.update_subscribers_stage2,
                                      args=[subscribers])
            log.info('Pushed them now.')

    def update_subscribers_stage2(self, subscribers):
        self.kvi['active_subs'].set(len(subscribers) - 1)

        self.users.reset_subs()

        self.users.update_subs(subscribers)

    def start(self):
        """Start the IRC client."""
        self.reactor.process_forever()

    def get_kvi_value(self, key, extra={}):
        return self.kvi[key].get()

    def get_last_tweet(self, key, extra={}):
        return self.twitter_manager.get_last_tweet(key)

    def get_emote_tm(self, key, extra={}):
        val = self.emotes.get_emote_epm(key)
        if not val:
            return None
        return '{0:,d}'.format(val)

    def get_emote_count(self, key, extra={}):
        val = self.emotes.get_emote_count(key)
        if not val:
            return None
        return '{0:,d}'.format(val)

    def get_emote_tm_record(self, key, extra={}):
        val = self.emotes.get_emote_epmrecord(key)
        if not val:
            return None
        return '{0:,d}'.format(val)

    def get_source_value(self, key, extra={}):
        try:
            return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_user_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_command_value(self, key, extra={}):
        try:
            return getattr(extra['command'].data, key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_usersource_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
            else:
                return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_time_value(self, key, extra={}):
        try:
            tz = timezone(key)
            return datetime.datetime.now(tz).strftime(self.date_fmt)
        except:
            log.exception('Unhandled exception in get_time_value')

        return None

    def get_current_song_value(self, key, extra={}):
        if self.stream_manager.online:
            current_song = PleblistManager.get_current_song(self.stream_manager.current_stream.id)
            inner_keys = key.split('.')
            val = current_song
            for inner_key in inner_keys:
                val = getattr(val, inner_key, None)
                if val is None:
                    return None
            if val is not None:
                return val
        return None

    def get_strictargs_value(self, key, extra={}):
        ret = self.get_args_value(key, extra)

        if len(ret) == 0:
            return None
        return ret

    def get_args_value(self, key, extra={}):
        range = None
        try:
            msg_parts = extra['message'].split(' ')
        except (KeyError, AttributeError):
            msg_parts = ['']

        try:
            if '-' in key:
                range_str = key.split('-')
                if len(range_str) == 2:
                    range = (int(range_str[0]), int(range_str[1]))

            if range is None:
                range = (int(key), len(msg_parts))
        except (TypeError, ValueError):
            range = (0, len(msg_parts))

        try:
            return ' '.join(msg_parts[range[0]:range[1]])
        except AttributeError:
            return ''
        except:
            log.exception('UNHANDLED ERROR IN get_args_value')
            return ''

    def get_notify_value(self, key, extra={}):
        payload = {
                'message': extra['message'] or '',
                'trigger': extra['trigger'],
                'user': extra['source'].username_raw,
                }
        self.websocket_manager.emit('notify', payload)

        return ''

    def get_value(self, key, extra={}):
        if key in extra:
            return extra[key]
        elif key in self.data:
            return self.data[key]
        elif key in self.data_cb:
            return self.data_cb[key]()

        log.warning('Unknown key passed to get_value: {0}'.format(key))
        return None

    def privmsg_arr(self, arr):
        for msg in arr:
            self.privmsg(msg)

    def privmsg_from_file(self, url, per_chunk=35, chunk_delay=30):
        try:
            r = requests.get(url)
            lines = r.text.split('\n')
            i = 0
            while len(lines) > 0:
                if i == 0:
                    self.privmsg_arr(lines[:per_chunk])
                else:
                    self.execute_delayed(chunk_delay * i, self.privmsg_arr, (lines[:per_chunk],))
                del lines[:per_chunk]
                i = i + 1
        except:
            log.exception('error in privmsg_from_file')

    # event is an event to clone and change the text from.
    def eval_from_file(self, event, url):
        try:
            r = requests.get(url)
            lines = r.text.splitlines()
            import copy
            for msg in lines:
                cloned_event = copy.deepcopy(event)
                cloned_event.arguments = [msg]
                # omit the source connectino as None (since its not used)
                self.on_pubmsg(None, cloned_event)
        except:
            log.exception('BabyRage')
            self.say('Exception')
        self.say('All lines done')

    def privmsg(self, message, channel=None, increase_message=True):
        if channel is None:
            channel = self.channel

        return self.irc.privmsg(message, channel, increase_message=increase_message)

    def c_uptime(self):
        return time_since(datetime.datetime.now().timestamp(), self.start_time.timestamp())

    def c_current_time(self):
        return datetime.datetime.now()

    @property
    def is_online(self):
        return self.stream_manager.online

    def c_stream_status(self):
        return 'online' if self.stream_manager.online else 'offline'

    def c_status_length(self):
        if self.stream_manager.online:
            return time_since(time.time(), self.stream_manager.current_stream.stream_start.timestamp())
        else:
            if self.stream_manager.last_stream is not None:
                return time_since(time.time(), self.stream_manager.last_stream.stream_end.timestamp())
            else:
                return 'No recorded stream FeelsBadMan '

    def execute_at(self, at, function, arguments=()):
        self.reactor.scheduler.execute_at(at, lambda: function(*arguments))

    def execute_delayed(self, delay, function, arguments=()):
        self.reactor.scheduler.execute_after(delay, lambda: function(*arguments))

    def execute_every(self, period, function, arguments=()):
        self.reactor.scheduler.execute_every(period, lambda: function(*arguments))

    def _ban(self, username, reason=''):
        self.privmsg('.ban {0} {1}'.format(username, reason), increase_message=False)

    def ban(self, username, reason=''):
        log.debug('Banning {}'.format(username))
        self._timeout(username, 30, reason)
        self.execute_delayed(1, self._ban, (username, reason))

    def ban_user(self, user, reason=''):
        self._timeout(user.username, 30, reason)
        self.execute_delayed(1, self._ban, (user.username, reason))

    def unban(self, username):
        self.privmsg('.unban {0}'.format(username), increase_message=False)

    def _timeout(self, username, duration, reason=''):
        self.privmsg('.timeout {0} {1} {2}'.format(username, duration, reason), increase_message=False)

    def timeout(self, username, duration, reason=''):
        log.debug('Timing out {} for {} seconds'.format(username, duration))
        self._timeout(username, duration, reason)
        # self.execute_delayed(1, self._timeout, (username, duration, reason))

    def timeout_warn(self, user, duration, reason=''):
        duration, punishment = user.timeout(duration, warning_module=self.module_manager['warning'])
        self.timeout(user.username, duration, reason)
        return (duration, punishment)

    def timeout_user(self, user, duration, reason=''):
        self._timeout(user.username, duration, reason)
        self.execute_delayed(1, self._timeout, (user.username, duration, reason))

    def _timeout_user(self, user, duration, reason=''):
        self._timeout(user.username, duration, reason)

    def whisper(self, username, *messages, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string as a whisper to username
        """

        if len(messages) < 0:
            return False

        message = separator.join(messages)

        return self.irc.whisper(username, message)

    def send_message_to_user(self, user, message, event, separator='. ', method='say'):
        if method == 'say':
            self.say(user.username + ', ' + lowercase_first_letter(message), separator=separator)
        elif method == 'whisper':
            self.whisper(user.username, message, separator=separator)
        elif method == 'me':
            self.me(message, separator=separator)
        elif method == 'reply':
            if event.type in ['action', 'pubmsg']:
                self.say(message, separator=separator)
            elif event.type == 'whisper':
                self.whisper(user.username, message, separator=separator)
        else:
            log.warning('Unknown send_message method: {}'.format(method))

    def safe_privmsg(self, message, channel=None, increase_message=True):
        # Check for banphrases
        res = self.banphrase_manager.check_message(message, None)
        if res is not False:
            self.privmsg('filtered message ({})'.format(res.id), channel, increase_message)
            return

        self.privmsg(message, channel, increase_message)

    def say(self, *messages, channel=None, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string to the given channel.
        """

        if len(messages) < 0:
            return False

        if not self.silent:
            message = separator.join(messages).strip()

            message = clean_up_message(message)
            if not message:
                return False

            # log.info('Sending message: {0}'.format(message))

            self.privmsg(message[:510], channel)

    def is_bad_message(self, message):
        return self.banphrase_manager.check_message(message, None) is not False

    def safe_me(self, message, channel=None):
        if not self.is_bad_message(message):
            self.me(message, channel)

    def me(self, message, channel=None):
        if not self.silent:
            self.say('.me ' + message[:500], channel=channel)

    def parse_version(self):
        self.version = self.version
        self.version_brief = self.version

        if self.dev:
            try:
                current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf8').strip()
                latest_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf8').strip()[:8]
                commit_number = subprocess.check_output(['git', 'rev-list', 'HEAD', '--count']).decode('utf8').strip()
                self.version = '{0} DEV ({1}, {2}, commit {3})'.format(self.version, current_branch, latest_commit, commit_number)
            except:
                pass

    def on_welcome(self, chatconn, event):
        return self.irc.on_welcome(chatconn, event)

    def connect(self):
        return self.irc.start()

    def on_disconnect(self, chatconn, event):
        self.irc.on_disconnect(chatconn, event)

    def parse_message(self, msg_raw, source, event, tags={}, whisper=False):
        msg_lower = msg_raw.lower()

        emote_tag = None

        for tag in tags:
            if tag['key'] == 'subscriber' and event.target == self.channel:
                source.subscriber = tag['value'] == '1'
            elif tag['key'] == 'emotes' and tag['value']:
                emote_tag = tag['value']
            elif tag['key'] == 'display-name' and tag['value']:
                source.username_raw = tag['value']
            elif tag['key'] == 'user-type':
                source.moderator = tag['value'] == 'mod' or source.username == self.streamer

        # source.num_lines += 1

        if source is None:
            log.error('No valid user passed to parse_message')
            return False

        if source.banned:
            self.ban(source.username)
            return False

        # If a user types when timed out, we assume he's been unbanned for a good reason and remove his flag.
        if source.timed_out is True:
            source.timed_out = False

        # Parse emotes in the message
        message_emotes = self.emotes.parse_message_twitch_emotes(source, msg_raw, emote_tag, whisper)

        urls = self.find_unique_urls(msg_raw)

        if whisper:
            self.whisper('datguy1', '{} said: {}'.format(source.username, msg_raw))
        # log.debug('{2}{0}: {1}'.format(source.username, msg_raw, '<w>' if whisper else ''))

        res = HandlerManager.trigger('on_message',
                source, msg_raw, message_emotes, whisper, urls, event,
                stop_on_false=True)
        if res is False:
            return False

        source.last_seen = datetime.datetime.now()
        source.last_active = datetime.datetime.now()

        if source.ignored:
            return False

        if msg_lower[:1] == '!':
            msg_lower_parts = msg_lower.split(' ')
            trigger = msg_lower_parts[0][1:]
            msg_raw_parts = msg_raw.split(' ')
            remaining_message = ' '.join(msg_raw_parts[1:]) if len(msg_raw_parts) > 1 else None
            if trigger in self.commands:
                command = self.commands[trigger]
                extra_args = {
                        'emotes': message_emotes,
                        'trigger': trigger,
                        }
                command.run(self, source, remaining_message, event=event, args=extra_args, whisper=whisper)

    @time_method
    def redis_test(self, username):
        try:
            pipeline = RedisManager.get().pipeline()
            pipeline.hget('pajlada:users:points', username)
            pipeline.hget('pajlada:users:ignored', username)
            pipeline.hget('pajlada:users:banned', username)
            pipeline.hget('pajlada:users:last_active', username)
        except:
            pipeline.reset()
        finally:
            b = pipeline.execute()
            log.info(b)

    @time_method
    def redis_test2(self, username):
        redis = RedisManager.get()
        values = [
                redis.hget('pajlada:users:points', username),
                redis.hget('pajlada:users:ignored', username),
                redis.hget('pajlada:users:banned', username),
                redis.hget('pajlada:users:last_active', username),
                ]
        return values

    def on_whisper(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        username = event.source.user.lower()

        with self.users.get_user_context(username) as source:
            self.parse_message(event.arguments[0], source, event, whisper=True, tags=event.tags)

    def on_ping(self, chatconn, event):
        # self.say('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        # log.info('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        self.last_ping = datetime.datetime.now()

    def on_pong(self, chatconn, event):
        # self.say('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        # log.info('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        self.last_pong = datetime.datetime.now()

    def on_pubnotice(self, chatconn, event):
        return
        type = 'whisper' if chatconn in self.whisper_manager else 'normal'
        log.debug('NOTICE {}@{}: {}'.format(type, event.target, event.arguments))

    def on_usernotice(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        tags = {}
        for d in event.tags:
            tags[d['key']] = d['value']

        if 'login' not in tags:
            return

        username = tags['login']

        with self.users.get_user_context(username) as source:
            msg = ''
            if len(event.arguments) > 0:
                msg = event.arguments[0]
            HandlerManager.trigger('on_usernotice',
                    source, msg, tags)

    def on_action(self, chatconn, event):
        self.on_pubmsg(chatconn, event)

    def on_pubmsg(self, chatconn, event):
        if event.source.user == self.nickname:
            return False

        username = event.source.user.lower()

        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        with self.users.get_user_context(username) as source:
            res = HandlerManager.trigger('on_pubmsg',
                    source, event.arguments[0],
                    stop_on_false=True)
            if res is False:
                return False
            self.parse_message(event.arguments[0], source, event, tags=event.tags)

    @time_method
    def reload_all(self):
        log.info('Reloading all...')
        for key, manager in self.reloadable.items():
            log.debug('Reloading {0}'.format(key))
            manager.reload()
            log.debug('Done with {0}'.format(key))
        log.info('ok!')

    @time_method
    def commit_all(self):
        log.info('Commiting all...')
        for key, manager in self.commitable.items():
            log.info('Commiting {0}'.format(key))
            manager.commit()
            log.info('Done with {0}'.format(key))
        log.info('ok!')

        HandlerManager.trigger('on_commit', stop_on_false=False)

    def do_tick(self):
        HandlerManager.trigger('on_tick')

    def quit(self, message, event, **options):
        quit_chub = self.config['main'].get('control_hub', None)
        quit_delay = 0

        if quit_chub is not None and event.target == ('#{}'.format(quit_chub)):
            quit_delay_random = 300
            try:
                if message is not None and int(message.split()[0]) >= 1:
                    quit_delay_random = int(message.split()[0])
            except (IndexError, ValueError, TypeError):
                pass
            quit_delay = random.randint(0, quit_delay_random)
            log.info('{} is restarting in {} seconds.'.format(self.nickname, quit_delay))

        self.execute_delayed(quit_delay, self.quit_bot)

    def quit_bot(self, **options):
        self.commit_all()
        quit = '{nickname} {version} shutting down...'
        phrase_data = {
                'nickname': self.nickname,
                'version': self.version,
                }

        try:
            ScheduleManager.base_scheduler.print_jobs()
            ScheduleManager.base_scheduler.shutdown(wait=False)
        except:
            log.exception('Error while shutting down the apscheduler')

        try:
            self.say('I have to leave PepeHands')
        except Exception:
            log.exception('Exception caught while trying to say quit phrase')

        self.twitter_manager.quit()
        self.socket_manager.quit()
        self.irc.quit()

        sys.exit(0)

    def apply_filter(self, resp, filter):
        available_filters = {
                'strftime': _filter_strftime,
                'lower': lambda var, args: var.lower(),
                'upper': lambda var, args: var.upper(),
                'time_since_minutes': lambda var, args: 'no time' if var == 0 else time_since(var * 60, 0, format='long'),
                'time_since': lambda var, args: 'no time' if var == 0 else time_since(var, 0, format='long'),
                'time_since_dt': _filter_time_since_dt,
                'urlencode': _filter_urlencode,
                'join': _filter_join,
                'number_format': _filter_number_format,
                'add': _filter_add,
                }
        if filter.name in available_filters:
            return available_filters[filter.name](resp, filter.arguments)
        return resp

    def find_unique_urls(self, message):
        from pajbot.modules.linkchecker import find_unique_urls
        return find_unique_urls(self.url_regex, message)
Esempio n. 2
0
class FollowAgeModule(BaseModule):

    ID = __name__.split(".")[-1]
    NAME = "Follow age"
    DESCRIPTION = "Makes two commands available: !followage and !followsince"
    CATEGORY = "Feature"
    SETTINGS = [
        ModuleSetting(
            key="action_followage",
            label="MessageAction for !followage",
            type="options",
            required=True,
            default="say",
            options=["say", "whisper", "reply"],
        ),
        ModuleSetting(
            key="action_followsince",
            label="MessageAction for !followsince",
            type="options",
            required=True,
            default="say",
            options=["say", "whisper", "reply"],
        ),
        ModuleSetting(
            key="global_cd",
            label="Global cooldown (seconds)",
            type="number",
            required=True,
            placeholder="",
            default=4,
            constraints={"min_value": 0, "max_value": 120},
        ),
        ModuleSetting(
            key="user_cd",
            label="Per-user cooldown (seconds)",
            type="number",
            required=True,
            placeholder="",
            default=8,
            constraints={"min_value": 0, "max_value": 240},
        ),
    ]

    def __init__(self, bot):
        super().__init__(bot)
        self.action_queue = ActionQueue()
        self.action_queue.start()

    def load_commands(self, **options):
        # TODO: Have delay modifiable in settings

        self.commands["followage"] = Command.raw_command(
            self.follow_age,
            delay_all=self.settings["global_cd"],
            delay_user=self.settings["user_cd"],
            description="Check your or someone elses follow age for a channel",
            can_execute_with_whisper=True,
            examples=[
                CommandExample(
                    None,
                    "Check your own follow age",
                    chat="user:!followage\n" "bot:pajlada, you have been following Karl_Kons for 4 months and 24 days",
                    description="Check how long you have been following the current streamer (Karl_Kons in this case)",
                ).parse(),
                CommandExample(
                    None,
                    "Check someone elses follow age",
                    chat="user:!followage NightNacht\n"
                    "bot:pajlada, NightNacht has been following Karl_Kons for 5 months and 4 days",
                    description="Check how long any user has been following the current streamer (Karl_Kons in this case)",
                ).parse(),
                CommandExample(
                    None,
                    "Check someones follow age for a certain streamer",
                    chat="user:!followage NightNacht forsenlol\n"
                    "bot:pajlada, NightNacht has been following forsenlol for 1 year and 4 months",
                    description="Check how long NightNacht has been following forsenlol",
                ).parse(),
                CommandExample(
                    None,
                    "Check your own follow age for a certain streamer",
                    chat="user:!followage pajlada forsenlol\n"
                    "bot:pajlada, you have been following forsenlol for 1 year and 3 months",
                    description="Check how long you have been following forsenlol",
                ).parse(),
            ],
        )

        self.commands["followsince"] = Command.raw_command(
            self.follow_since,
            delay_all=self.settings["global_cd"],
            delay_user=self.settings["user_cd"],
            description="Check from when you or someone else first followed a channel",
            can_execute_with_whisper=True,
            examples=[
                CommandExample(
                    None,
                    "Check your own follow since",
                    chat="user:!followsince\n"
                    "bot:pajlada, you have been following Karl_Kons since 04 March 2015, 07:02:01 UTC",
                    description="Check when you first followed the current streamer (Karl_Kons in this case)",
                ).parse(),
                CommandExample(
                    None,
                    "Check someone elses follow since",
                    chat="user:!followsince NightNacht\n"
                    "bot:pajlada, NightNacht has been following Karl_Kons since 03 July 2014, 04:12:42 UTC",
                    description="Check when NightNacht first followed the current streamer (Karl_Kons in this case)",
                ).parse(),
                CommandExample(
                    None,
                    "Check someone elses follow since for another streamer",
                    chat="user:!followsince NightNacht forsenlol\n"
                    "bot:pajlada, NightNacht has been following forsenlol since 13 June 2013, 13:10:51 UTC",
                    description="Check when NightNacht first followed the given streamer (forsenlol)",
                ).parse(),
                CommandExample(
                    None,
                    "Check your follow since for another streamer",
                    chat="user:!followsince pajlada forsenlol\n"
                    "bot:pajlada, you have been following forsenlol since 16 December 1990, 03:06:51 UTC",
                    description="Check when you first followed the given streamer (forsenlol)",
                ).parse(),
            ],
        )

    def check_follow_age(self, bot, source, username, streamer, event):
        streamer = bot.streamer if streamer is None else streamer.lower()
        age = bot.twitchapi.get_follow_relationship(username, streamer)
        is_self = source.username == username
        message = ""
        streamer = streamer.replace('admiralbulldog', 'Buldog')

        if age:
            # Following
            human_age = time_since(utils.now().timestamp() - age.timestamp(), 0)
            message = "{} has been following {} for {}".format(username, streamer, human_age)
        else:
            # Not following
            message = "{} is not following {}".format(username, streamer)

        bot.send_message_to_user(source, message, event, method=self.settings["action_followage"])

    def check_follow_since(self, bot, source, username, streamer, event):
        streamer = bot.streamer if streamer is None else streamer.lower()
        follow_since = bot.twitchapi.get_follow_relationship(username, streamer)
        is_self = source.username == username
        message = ""
        streamer = streamer.replace('admiralbulldog', 'Buldog')

        if follow_since:
            # Following
            human_age = follow_since.strftime("%d %B %Y, %X")
            message = "{} has been following {} since {} UTC".format(username, streamer, human_age)
        else:
            # Not following
            message = "{} is not following {}".format(username, streamer)

        bot.send_message_to_user(source, message, event, method=self.settings["action_followsince"])

    def follow_age(self, **options):
        source = options["source"]
        message = options["message"]
        bot = options["bot"]
        event = options["event"]

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_age, args=[bot, source, username, streamer, event])

    def follow_since(self, **options):
        bot = options["bot"]
        source = options["source"]
        message = options["message"]
        event = options["event"]

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_since, args=[bot, source, username, streamer, event])

    @staticmethod
    def parse_message(bot, source, message):
        username = source.username
        streamer = None
        if message is not None and len(message) > 0:
            message_split = message.split(" ")
            if len(message_split[0]) and message_split[0].replace("_", "").isalnum():
                username = message_split[0].lower()
            if len(message_split) > 1 and message_split[1].replace("_", "").isalnum():
                streamer = message_split[1]

        return username, streamer
Esempio n. 3
0
File: bot.py Progetto: jardg/pajbot
class Bot:
    """
    Main class for the twitch bot
    """

    version = '1.30'
    date_fmt = '%H:%M'
    admin = None
    url_regex_str = r'\(?(?:(http|https):\/\/)?(?:((?:[^\W\s]|\.|-|[:]{1})+)@{1})?((?:www.)?(?:[^\W\s]|\.|-)+[\.][^\W\s]{2,4}|localhost(?=\/)|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?::(\d*))?([\/]?[^\s\?]*[\/]{1})*(?:\/?([^\s\n\?\[\]\{\}\#]*(?:(?=\.)){1}|[^\s\n\?\[\]\{\}\.\#]*)?([\.]{1}[^\s\?\#]*)?)?(?:\?{1}([^\s\n\#\[\]]*))?([\#][^\s\n]*)?\)?'

    last_ping = datetime.datetime.now()
    last_pong = datetime.datetime.now()

    def parse_args():
        parser = argparse.ArgumentParser()
        parser.add_argument('--config', '-c',
                            default='config.ini',
                            help='Specify which config file to use '
                                    '(default: config.ini)')
        parser.add_argument('--silent',
                            action='count',
                            help='Decides whether the bot should be '
                            'silent or not')
        # TODO: Add a log level argument.

        return parser.parse_args()

    def load_config(self, config):
        self.config = config

        self.domain = config['web'].get('domain', 'localhost')

        self.nickname = config['main'].get('nickname', 'pajbot')
        self.password = config['main'].get('password', 'abcdef')

        self.timezone = config['main'].get('timezone', 'UTC')

        self.trusted_mods = config.getboolean('main', 'trusted_mods')

        TimeManager.init_timezone(self.timezone)

        if 'streamer' in config['main']:
            self.streamer = config['main']['streamer']
            self.channel = '#' + self.streamer
        elif 'target' in config['main']:
            self.channel = config['main']['target']
            self.streamer = self.channel[1:]

        self.wolfram = None
        if 'wolfram' in config['main']:
            try:
                import wolframalpha
                self.wolfram = wolframalpha.Client(config['main']['wolfram'])
            except ImportError:
                pass

        self.silent = False
        self.dev = False

        if 'flags' in config:
            self.silent = True if 'silent' in config['flags'] and config['flags']['silent'] == '1' else self.silent
            self.dev = True if 'dev' in config['flags'] and config['flags']['dev'] == '1' else self.dev

        DBManager.init(self.config['main']['db'])

        redis_options = {}
        if 'redis' in config:
            redis_options = config._sections['redis']

        RedisManager.init(**redis_options)

    def __init__(self, config, args=None):
        # Load various configuration variables from the given config object
        # The config object that should be passed through should
        # come from pajbot.utils.load_config
        self.load_config(config)

        # Update the database scheme if necessary using alembic
        # In case of errors, i.e. if the database is out of sync or the alembic
        # binary can't be called, we will shut down the bot.
        pajbot.utils.alembic_upgrade()

        # Actions in this queue are run in a separate thread.
        # This means actions should NOT access any database-related stuff.
        self.action_queue = ActionQueue()
        self.action_queue.start()

        self.reactor = irc.client.Reactor(self.on_connect)
        self.start_time = datetime.datetime.now()
        ActionParser.bot = self

        HandlerManager.init_handlers()

        self.socket_manager = SocketManager(self)
        self.stream_manager = StreamManager(self)

        StreamHelper.init_bot(self, self.stream_manager)
        ScheduleManager.init()

        self.users = UserManager()
        self.decks = DeckManager()
        self.module_manager = ModuleManager(self.socket_manager, bot=self).load()
        self.commands = CommandManager(
                socket_manager=self.socket_manager,
                module_manager=self.module_manager,
                bot=self).load()
        self.filters = FilterManager().reload()
        self.banphrase_manager = BanphraseManager(self).load()
        self.timer_manager = TimerManager(self).load()
        self.kvi = KVIManager()
        self.emotes = EmoteManager(self)
        self.twitter_manager = TwitterManager(self)

        HandlerManager.trigger('on_managers_loaded')

        # Reloadable managers
        self.reloadable = {
                'filters': self.filters,
                }

        # Commitable managers
        self.commitable = {
                'commands': self.commands,
                'filters': self.filters,
                'banphrases': self.banphrase_manager,
                }

        self.execute_every(10 * 60, self.commit_all)
        self.execute_every(1, self.do_tick)

        try:
            self.admin = self.config['main']['admin']
        except KeyError:
            log.warning('No admin user specified. See the [main] section in config.example.ini for its usage.')
        if self.admin:
            with self.users.get_user_context(self.admin) as user:
                user.level = 2000

        self.parse_version()

        relay_host = self.config['main'].get('relay_host', None)
        relay_password = self.config['main'].get('relay_password', None)
        if relay_host is None or relay_password is None:
            self.irc = MultiIRCManager(self)
        else:
            self.irc = SingleIRCManager(self)

        self.reactor.add_global_handler('all_events', self.irc._dispatcher, -10)

        twitch_client_id = None
        twitch_oauth = None
        if 'twitchapi' in self.config:
            twitch_client_id = self.config['twitchapi'].get('client_id', None)
            twitch_oauth = self.config['twitchapi'].get('oauth', None)

        # A client ID is required for the bot to work properly now, give an error for now
        if twitch_client_id is None:
            log.error('MISSING CLIENT ID, SET "client_id" VALUE UNDER [twitchapi] SECTION IN CONFIG FILE')

        self.twitchapi = TwitchAPI(twitch_client_id, twitch_oauth)

        self.data = {}
        self.data_cb = {}
        self.url_regex = re.compile(self.url_regex_str, re.IGNORECASE)

        self.data['broadcaster'] = self.streamer
        self.data['version'] = self.version
        self.data['version_brief'] = self.version_brief
        self.data['bot_name'] = self.nickname
        self.data_cb['status_length'] = self.c_status_length
        self.data_cb['stream_status'] = self.c_stream_status
        self.data_cb['bot_uptime'] = self.c_uptime
        self.data_cb['current_time'] = self.c_current_time

        self.silent = True if args.silent else self.silent

        if self.silent:
            log.info('Silent mode enabled')

        """
        For actions that need to access the main thread,
        we can use the mainthread_queue.
        """
        self.mainthread_queue = ActionQueue()
        self.execute_every(1, self.mainthread_queue.parse_action)

        self.websocket_manager = WebSocketManager(self)

        try:
            if self.config['twitchapi']['update_subscribers'] == '1':
                self.execute_every(30 * 60,
                                   self.action_queue.add,
                                   (self.update_subscribers_stage1, ))
        except:
            pass

        # XXX: TEMPORARY UGLY CODE
        HandlerManager.add_handler('on_user_gain_tokens', self.on_user_gain_tokens)
        HandlerManager.add_handler('send_whisper', self.whisper)

    def on_connect(self, sock):
        return self.irc.on_connect(sock)

    def on_user_gain_tokens(self, user, tokens_gained):
        self.whisper(user.username, 'You finished todays quest! You have been awarded with {} tokens.'.format(tokens_gained))

    def update_subscribers_stage1(self):
        limit = 100
        offset = 0
        subscribers = []
        log.info('Starting stage1 subscribers update')

        try:
            retry_same = 0
            while True:
                log.debug('Beginning sub request {0} {1}'.format(limit, offset))
                subs, retry_same, error = self.twitchapi.get_subscribers(self.streamer, limit, offset, 0 if retry_same is False else retry_same)
                log.debug('got em!')

                if error is True:
                    log.error('Too many attempts, aborting')
                    return False

                if retry_same is False:
                    offset += limit
                    if len(subs) == 0:
                        # We don't need to retry, and the last query finished propery
                        # Break out of the loop and start fiddling with the subs!
                        log.debug('Done!')
                        break
                    else:
                        log.debug('Fetched {0} subs'.format(len(subs)))
                        subscribers.extend(subs)

                if retry_same is not False:
                    # In case the next attempt is a retry, wait for 3 seconds
                    log.debug('waiting for 3 seconds...')
                    time.sleep(3)
                    log.debug('waited enough!')
            log.debug('Finished with the while True loop!')
        except:
            log.exception('Caught an exception while trying to get subscribers')
            return

        log.info('Ended stage1 subscribers update')
        if len(subscribers) > 0:
            log.info('Got some subscribers, so we are pushing them to stage 2!')
            self.mainthread_queue.add(self.update_subscribers_stage2,
                                      args=[subscribers])
            log.info('Pushed them now.')

    def update_subscribers_stage2(self, subscribers):
        self.kvi['active_subs'].set(len(subscribers) - 1)

        self.users.reset_subs()

        self.users.update_subs(subscribers)

    def start(self):
        """Start the IRC client."""
        self.reactor.process_forever()

    def get_kvi_value(self, key, extra={}):
        return self.kvi[key].get()

    def get_last_tweet(self, key, extra={}):
        return self.twitter_manager.get_last_tweet(key)

    def get_emote_tm(self, key, extra={}):
        val = self.emotes.get_emote_epm(key)
        if not val:
            return None
        return '{0:,d}'.format(val)

    def get_emote_count(self, key, extra={}):
        val = self.emotes.get_emote_count(key)
        if not val:
            return None
        return '{0:,d}'.format(val)

    def get_emote_tm_record(self, key, extra={}):
        val = self.emotes.get_emote_epmrecord(key)
        if not val:
            return None
        return '{0:,d}'.format(val)

    def get_source_value(self, key, extra={}):
        try:
            return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_user_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_command_value(self, key, extra={}):
        try:
            return getattr(extra['command'].data, key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_usersource_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
            else:
                return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_time_value(self, key, extra={}):
        try:
            tz = timezone(key)
            return datetime.datetime.now(tz).strftime(self.date_fmt)
        except:
            log.exception('Unhandled exception in get_time_value')

        return None

    def get_current_song_value(self, key, extra={}):
        if self.stream_manager.online:
            current_song = PleblistManager.get_current_song(self.stream_manager.current_stream.id)
            inner_keys = key.split('.')
            val = current_song
            for inner_key in inner_keys:
                val = getattr(val, inner_key, None)
                if val is None:
                    return None
            if val is not None:
                return val
        return None

    def get_strictargs_value(self, key, extra={}):
        ret = self.get_args_value(key, extra)

        if len(ret) == 0:
            return None
        return ret

    def get_args_value(self, key, extra={}):
        range = None
        try:
            msg_parts = extra['message'].split(' ')
        except (KeyError, AttributeError):
            msg_parts = ['']

        try:
            if '-' in key:
                range_str = key.split('-')
                if len(range_str) == 2:
                    range = (int(range_str[0]), int(range_str[1]))

            if range is None:
                range = (int(key), len(msg_parts))
        except (TypeError, ValueError):
            range = (0, len(msg_parts))

        try:
            return ' '.join(msg_parts[range[0]:range[1]])
        except AttributeError:
            return ''
        except:
            log.exception('UNHANDLED ERROR IN get_args_value')
            return ''

    def get_notify_value(self, key, extra={}):
        payload = {
                'message': extra['message'] or '',
                'trigger': extra['trigger'],
                'user': extra['source'].username_raw,
                }
        self.websocket_manager.emit('notify', payload)

        return ''

    def get_value(self, key, extra={}):
        if key in extra:
            return extra[key]
        elif key in self.data:
            return self.data[key]
        elif key in self.data_cb:
            return self.data_cb[key]()

        log.warning('Unknown key passed to get_value: {0}'.format(key))
        return None

    def privmsg_arr(self, arr):
        for msg in arr:
            self.privmsg(msg)

    def privmsg_from_file(self, url, per_chunk=35, chunk_delay=30):
        try:
            r = requests.get(url)
            lines = r.text.split('\n')
            i = 0
            while len(lines) > 0:
                if i == 0:
                    self.privmsg_arr(lines[:per_chunk])
                else:
                    self.execute_delayed(chunk_delay * i, self.privmsg_arr, (lines[:per_chunk],))

                del lines[:per_chunk]

                i = i + 1
        except:
            log.exception('error in privmsg_from_file')

    def privmsg(self, message, channel=None, increase_message=True):
        if channel is None:
            channel = self.channel

        return self.irc.privmsg(message, channel, increase_message=increase_message)

    def c_uptime(self):
        return time_since(datetime.datetime.now().timestamp(), self.start_time.timestamp())

    def c_current_time(self):
        return datetime.datetime.now()

    @property
    def is_online(self):
        return self.stream_manager.online

    def c_stream_status(self):
        return 'online' if self.stream_manager.online else 'offline'

    def c_status_length(self):
        if self.stream_manager.online:
            return time_since(time.time(), self.stream_manager.current_stream.stream_start.timestamp())
        else:
            if self.stream_manager.last_stream is not None:
                return time_since(time.time(), self.stream_manager.last_stream.stream_end.timestamp())
            else:
                return 'No recorded stream FeelsBadMan '

    def execute_at(self, at, function, arguments=()):
        self.reactor.scheduler.execute_at(at, lambda: function(*arguments))

    def execute_delayed(self, delay, function, arguments=()):
        self.reactor.scheduler.execute_after(delay, lambda: function(*arguments))

    def execute_every(self, period, function, arguments=()):
        self.reactor.scheduler.execute_every(period, lambda: function(*arguments))

    def _ban(self, username, reason=''):
        self.privmsg('.ban {0} {1}'.format(username, reason), increase_message=False)

    def ban(self, username, reason=''):
        log.debug('Banning {}'.format(username))
        self._timeout(username, 30, reason)
        self.execute_delayed(1, self._ban, (username, reason))

    def ban_user(self, user, reason=''):
        self._timeout(user.username, 30, reason)
        self.execute_delayed(1, self._ban, (user.username, reason))

    def unban(self, username):
        self.privmsg('.unban {0}'.format(username), increase_message=False)

    def _timeout(self, username, duration, reason=''):
        self.privmsg('.timeout {0} {1} {2}'.format(username, duration, reason), increase_message=False)

    def timeout(self, username, duration, reason=''):
        log.debug('Timing out {} for {} seconds'.format(username, duration))
        self._timeout(username, duration, reason)
        self.execute_delayed(1, self._timeout, (username, duration, reason))

    def timeout_warn(self, user, duration, reason=''):
        duration, punishment = user.timeout(duration, warning_module=self.module_manager['warning'])
        self.timeout(user.username, duration, reason)
        return (duration, punishment)

    def timeout_user(self, user, duration, reason=''):
        self._timeout(user.username, duration, reason)
        self.execute_delayed(1, self._timeout, (user.username, duration, reason))

    def whisper(self, username, *messages, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string as a whisper to username
        """

        if len(messages) < 0:
            return False

        message = separator.join(messages)

        return self.irc.whisper(username, message)

    def send_message_to_user(self, user, message, separator='. ', method='say'):
        if method == 'say':
            self.say(user.username + ', ' + lowercase_first_letter(message), separator=separator)
        elif method == 'whisper':
            self.whisper(user.username, message, separator=separator)
        else:
            log.warning('Unknown send_message method: {}'.format(method))

    def say(self, *messages, channel=None, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string to the given channel.
        """

        if len(messages) < 0:
            return False

        if not self.silent:
            message = separator.join(messages).strip()

            if len(message) >= 1:
                if (message[0] == '.' or message[0] == '/') and not message[1:3] == 'me':
                    log.warning('Message we attempted to send started with . or /, skipping.')
                    return

                log.info('Sending message: {0}'.format(message))

                self.privmsg(message[:510], channel)

    def me(self, message, channel=None):
        if not self.silent:
            message = message.strip()

            if len(message) >= 1:
                if message[0] == '.' or message[0] == '/':
                    log.warning('Message we attempted to send started with . or /, skipping.')
                    return

                log.info('Sending message: {0}'.format(message))

                self.privmsg('.me ' + message[:500], channel)

    def parse_version(self):
        self.version = self.version
        self.version_brief = self.version

        if self.dev:
            try:
                current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf8').strip()
                latest_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf8').strip()[:8]
                commit_number = subprocess.check_output(['git', 'rev-list', 'HEAD', '--count']).decode('utf8').strip()
                self.version = '{0} DEV ({1}, {2}, commit {3})'.format(self.version, current_branch, latest_commit, commit_number)
            except:
                pass

    def on_welcome(self, chatconn, event):
        return self.irc.on_welcome(chatconn, event)

    def connect(self):
        return self.irc.start()

    def on_disconnect(self, chatconn, event):
        self.irc.on_disconnect(chatconn, event)

    def parse_message(self, msg_raw, source, event, tags={}, whisper=False):
        msg_lower = msg_raw.lower()

        emote_tag = None

        for tag in tags:
            if tag['key'] == 'subscriber' and event.target == self.channel:
                source.subscriber = tag['value'] == '1'
            elif tag['key'] == 'emotes' and tag['value']:
                emote_tag = tag['value']
            elif tag['key'] == 'display-name' and tag['value']:
                source.username_raw = tag['value']
            elif tag['key'] == 'user-type':
                source.moderator = tag['value'] == 'mod' or source.username == self.streamer

        # source.num_lines += 1

        if source is None:
            log.error('No valid user passed to parse_message')
            return False

        if source.banned:
            self.ban(source.username)
            return False

        # If a user types when timed out, we assume he's been unbanned for a good reason and remove his flag.
        if source.timed_out is True:
            source.timed_out = False

        # Parse emotes in the message
        message_emotes = self.emotes.parse_message_twitch_emotes(source, msg_raw, emote_tag, whisper)

        urls = self.find_unique_urls(msg_raw)

        log.debug('{2}{0}: {1}'.format(source.username, msg_raw, '<w>' if whisper else ''))

        res = HandlerManager.trigger('on_message',
                source, msg_raw, message_emotes, whisper, urls, event,
                stop_on_false=True)
        if res is False:
            return False

        source.last_seen = datetime.datetime.now()
        source.last_active = datetime.datetime.now()

        """
        if self.streamer == 'forsenlol' and whisper is False:
            follow_time = self.twitchapi.get_follow_relationship2(source.username, self.streamer)

            if follow_time is False:
                self._timeout(source.username, 600, '2 years follow mode (or api is down?)')
                return

            follow_age = datetime.datetime.now() - follow_time
            log.debug(follow_age)

            if follow_age.days < 730:
                log.debug('followed less than 730 days LUL')
                self._timeout(source.username, 600, '2 years follow mode')
                """

        if source.ignored:
            return False

        if msg_lower[:1] == '!':
            msg_lower_parts = msg_lower.split(' ')
            trigger = msg_lower_parts[0][1:]
            msg_raw_parts = msg_raw.split(' ')
            remaining_message = ' '.join(msg_raw_parts[1:]) if len(msg_raw_parts) > 1 else None
            if trigger in self.commands:
                command = self.commands[trigger]
                extra_args = {
                        'emotes': message_emotes,
                        'trigger': trigger,
                        }
                command.run(self, source, remaining_message, event=event, args=extra_args, whisper=whisper)

    @time_method
    def redis_test(self, username):
        try:
            pipeline = RedisManager.get().pipeline()
            pipeline.hget('pajlada:users:points', username)
            pipeline.hget('pajlada:users:ignored', username)
            pipeline.hget('pajlada:users:banned', username)
            pipeline.hget('pajlada:users:last_active', username)
        except:
            pipeline.reset()
        finally:
            b = pipeline.execute()
            log.info(b)

    @time_method
    def redis_test2(self, username):
        redis = RedisManager.get()
        values = [
                redis.hget('pajlada:users:points', username),
                redis.hget('pajlada:users:ignored', username),
                redis.hget('pajlada:users:banned', username),
                redis.hget('pajlada:users:last_active', username),
                ]
        return values

    def on_whisper(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        username = event.source.user.lower()

        with self.users.get_user_context(username) as source:
            self.parse_message(event.arguments[0], source, event, whisper=True, tags=event.tags)

    def on_ping(self, chatconn, event):
        # self.say('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        log.info('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        self.last_ping = datetime.datetime.now()

    def on_pong(self, chatconn, event):
        # self.say('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        log.info('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        self.last_pong = datetime.datetime.now()

    def on_pubnotice(self, chatconn, event):
        return
        type = 'whisper' if chatconn in self.whisper_manager else 'normal'
        log.debug('NOTICE {}@{}: {}'.format(type, event.target, event.arguments))

    def on_usernotice(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        tags = {}
        for d in event.tags:
            tags[d['key']] = d['value']

        if 'login' not in tags:
            return

        username = tags['login']

        with self.users.get_user_context(username) as source:
            msg = ''
            if len(event.arguments) > 0:
                msg = event.arguments[0]
            HandlerManager.trigger('on_usernotice',
                    source, msg, tags)

    def on_action(self, chatconn, event):
        self.on_pubmsg(chatconn, event)

    def on_pubmsg(self, chatconn, event):
        if event.source.user == self.nickname:
            return False

        username = event.source.user.lower()


        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        with self.users.get_user_context(username) as source:
            res = HandlerManager.trigger('on_pubmsg',
                    source, event.arguments[0],
                    stop_on_false=True)
            if res is False:
                return False

            self.parse_message(event.arguments[0], source, event, tags=event.tags)

    @time_method
    def reload_all(self):
        log.info('Reloading all...')
        for key, manager in self.reloadable.items():
            log.debug('Reloading {0}'.format(key))
            manager.reload()
            log.debug('Done with {0}'.format(key))
        log.info('ok!')

    @time_method
    def commit_all(self):
        log.info('Commiting all...')
        for key, manager in self.commitable.items():
            log.info('Commiting {0}'.format(key))
            manager.commit()
            log.info('Done with {0}'.format(key))
        log.info('ok!')

        HandlerManager.trigger('on_commit', stop_on_false=False)

    def do_tick(self):
        HandlerManager.trigger('on_tick')

    def quit(self, message, event, **options):
        quit_chub = self.config['main'].get('control_hub', None)
        quit_delay = 0

        if quit_chub is not None and event.target == ('#{}'.format(quit_chub)):
            quit_delay_random = 300
            try:
                if message is not None and int(message.split()[0]) >= 1:
                    quit_delay_random = int(message.split()[0])
            except (IndexError, ValueError, TypeError):
                pass
            quit_delay = random.randint(0, quit_delay_random)
            log.info('{} is restarting in {} seconds.'.format(self.nickname, quit_delay))

        self.execute_delayed(quit_delay, self.quit_bot)

    def quit_bot(self, **options):
        self.commit_all()
        quit = '{nickname} {version} shutting down...'
        phrase_data = {
                'nickname': self.nickname,
                'version': self.version,
                }

        try:
            ScheduleManager.base_scheduler.print_jobs()
            ScheduleManager.base_scheduler.shutdown(wait=False)
        except:
            log.exception('Error while shutting down the apscheduler')

        try:
            self.say(quit.format(**phrase_data))
        except Exception:
            log.exception('Exception caught while trying to say quit phrase')

        self.twitter_manager.quit()
        self.socket_manager.quit()
        self.irc.quit()

        sys.exit(0)

    def apply_filter(self, resp, filter):
        available_filters = {
                'strftime': _filter_strftime,
                'lower': lambda var, args: var.lower(),
                'upper': lambda var, args: var.upper(),
                'time_since_minutes': lambda var, args: 'no time' if var == 0 else time_since(var * 60, 0, format='long'),
                'time_since': lambda var, args: 'no time' if var == 0 else time_since(var, 0, format='long'),
                'time_since_dt': _filter_time_since_dt,
                'urlencode': _filter_urlencode,
                'join': _filter_join,
                'number_format': _filter_number_format,
                }
        if filter.name in available_filters:
            return available_filters[filter.name](resp, filter.arguments)
        return resp

    def find_unique_urls(self, message):
        from pajbot.modules.linkchecker import find_unique_urls
        return find_unique_urls(self.url_regex, message)
Esempio n. 4
0
class LinkCheckerModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Link Checker'
    DESCRIPTION = 'Checks links if they\'re bad'
    ENABLED_DEFAULT = True
    CATEGORY = 'Filter'
    SETTINGS = [
        ModuleSetting(key='ban_pleb_links',
                      label='Disallow links from non-subscribers',
                      type='boolean',
                      required=True,
                      default=False)
    ]

    def __init__(self):
        super().__init__()
        self.db_session = None
        self.links = {}

        self.blacklisted_links = []
        self.whitelisted_links = []

        self.cache = LinkCheckerCache(
        )  # cache[url] = True means url is safe, False means the link is bad

        self.action_queue = ActionQueue()
        self.action_queue.start()

    def enable(self, bot):
        self.bot = bot
        pajbot.managers.handler.HandlerManager.add_handler('on_message',
                                                           self.on_message,
                                                           priority=100)
        pajbot.managers.handler.HandlerManager.add_handler(
            'on_commit', self.on_commit)
        if bot:
            self.run_later = bot.execute_delayed

            if 'safebrowsingapi' in bot.config['main']:
                # XXX: This should be loaded as a setting instead.
                # There needs to be a setting for settings to have them as "passwords"
                # so they're not displayed openly
                self.safeBrowsingAPI = SafeBrowsingAPI(
                    bot.config['main']['safebrowsingapi'], bot.nickname,
                    bot.version)
            else:
                self.safeBrowsingAPI = None

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
        self.db_session = DBManager.create_session()
        self.blacklisted_links = []
        for link in self.db_session.query(BlacklistedLink):
            self.blacklisted_links.append(link)

        self.whitelisted_links = []
        for link in self.db_session.query(WhitelistedLink):
            self.whitelisted_links.append(link)

    def disable(self, bot):
        pajbot.managers.handler.HandlerManager.remove_handler(
            'on_message', self.on_message)
        pajbot.managers.handler.HandlerManager.remove_handler(
            'on_commit', self.on_commit)

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
            self.blacklisted_links = []
            self.whitelisted_links = []

    def reload(self):

        log.info('Loaded {0} bad links and {1} good links'.format(
            len(self.blacklisted_links), len(self.whitelisted_links)))
        return self

    super_whitelist = ['pajlada.se', 'pajlada.com', 'forsen.tv', 'pajbot.com']

    def on_message(self, source, message, emotes, whisper, urls, event):
        if not whisper and source.level < 500 and source.moderator is False:
            if self.settings[
                    'ban_pleb_links'] is True and source.subscriber is False and len(
                        urls) > 0:
                # Check if the links are in our super-whitelist. i.e. on the pajlada.se domain o forsen.tv
                for url in urls:
                    parsed_url = Url(url)
                    if len(parsed_url.parsed.netloc.split('.')) < 2:
                        continue
                    whitelisted = False
                    for whitelist in self.super_whitelist:
                        if is_subdomain(parsed_url.parsed.netloc, whitelist):
                            whitelisted = True
                            break
                    if whitelisted is False:
                        self.bot.timeout(source.username,
                                         30,
                                         reason='Non-subs cannot post links')
                        if source.minutes_in_chat_online > 60:
                            self.bot.whisper(
                                source.username,
                                'You cannot post non-verified links in chat if you\'re not a subscriber.'
                            )
                        return False

            for url in urls:
                # Action which will be taken when a bad link is found
                action = Action(self.bot.timeout,
                                args=[source.username, 20],
                                kwargs={'reason': 'Banned link'})
                # First we perform a basic check
                if self.simple_check(url, action) == self.RET_FURTHER_ANALYSIS:
                    # If the basic check returns no relevant data, we queue up a proper check on the URL
                    self.action_queue.add(self.check_url, args=[url, action])

    def on_commit(self):
        if self.db_session is not None:
            self.db_session.commit()

    def delete_from_cache(self, url):
        if url in self.cache:
            log.debug('LinkChecker: Removing url {0} from cache'.format(url))
            del self.cache[url]

    def cache_url(self, url, safe):
        if url in self.cache and self.cache[url] == safe:
            return

        log.debug('LinkChecker: Caching url {0} as {1}'.format(
            url, 'SAFE' if safe is True else 'UNSAFE'))
        self.cache[url] = safe
        self.run_later(20, self.delete_from_cache, (url, ))

    def counteract_bad_url(self,
                           url,
                           action=None,
                           want_to_cache=True,
                           want_to_blacklist=False):
        log.debug('LinkChecker: BAD URL FOUND {0}'.format(url.url))
        if action:
            action.run()
        if want_to_cache:
            self.cache_url(url.url, False)
        if want_to_blacklist:
            self.blacklist_url(url.url, url.parsed)
            return True

    def blacklist_url(self, url, parsed_url=None, level=0):
        if not (url.lower().startswith('http://')
                or url.lower().startswith('https://')):
            url = 'http://' + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        if self.is_blacklisted(url, parsed_url):
            return False

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = BlacklistedLink(domain, path, level)
        self.db_session.add(link)
        self.blacklisted_links.append(link)
        self.db_session.commit()

    def whitelist_url(self, url, parsed_url=None):
        if not (url.lower().startswith('http://')
                or url.lower().startswith('https://')):
            url = 'http://' + url
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        if self.is_whitelisted(url, parsed_url):
            return

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = WhitelistedLink(domain, path)
        self.db_session.add(link)
        self.whitelisted_links.append(link)
        self.db_session.commit()

    def is_blacklisted(self, url, parsed_url=None, sublink=False):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.blacklisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    if not sublink:
                        return True
                    elif link.level >= 1:  # if it's a sublink, but the blacklisting level is 0, we don't consider it blacklisted
                        return True

        return False

    def is_whitelisted(self, url, parsed_url=None):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.whitelisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    return True

        return False

    RET_BAD_LINK = -1
    RET_FURTHER_ANALYSIS = 0
    RET_GOOD_LINK = 1

    def basic_check(self, url, action, sublink=False):
        """
        Check if the url is in the cache, or if it's
        Return values:
        1 = Link is OK
        -1 = Link is bad
        0 = Link needs further analysis
        """
        if url.url in self.cache:
            log.debug('LinkChecker: Url {0} found in cache'.format(url.url))
            if not self.cache[url.url]:  # link is bad
                self.counteract_bad_url(url, action, False, False)
                return self.RET_BAD_LINK
            return self.RET_GOOD_LINK

        log.info('Checking if link is blacklisted...')
        if self.is_blacklisted(url.url, url.parsed, sublink):
            log.debug('LinkChecker: Url {0} is blacklisted'.format(url.url))
            self.counteract_bad_url(url, action, want_to_blacklist=False)
            return self.RET_BAD_LINK

        log.info('Checking if link is whitelisted...')
        if self.is_whitelisted(url.url, url.parsed):
            log.debug('LinkChecker: Url {0} allowed by the whitelist'.format(
                url.url))
            self.cache_url(url.url, True)
            return self.RET_GOOD_LINK

        return self.RET_FURTHER_ANALYSIS

    def simple_check(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return self.RET_FURTHER_ANALYSIS

        return self.basic_check(url, action)

    def check_url(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return

        try:
            self._check_url(url, action)
        except:
            log.exception('LinkChecker unhanled exception while _check_url')

    def _check_url(self, url, action):
        log.debug('LinkChecker: Checking url {0}'.format(url.url))

        # XXX: The basic check is currently performed twice on links found in messages. Solve
        res = self.basic_check(url, action)
        if res == self.RET_GOOD_LINK:
            return
        elif res == self.RET_BAD_LINK:
            return

        connection_timeout = 2
        read_timeout = 1
        try:
            r = requests.head(url.url,
                              allow_redirects=True,
                              timeout=connection_timeout)
        except:
            self.cache_url(url.url, True)
            return

        checkcontenttype = ('content-type' in r.headers
                            and r.headers['content-type']
                            == 'application/octet-stream')
        checkdispotype = ('disposition-type' in r.headers
                          and r.headers['disposition-type'] == 'attachment')

        if checkcontenttype or checkdispotype:  # triggering a download not allowed
            self.counteract_bad_url(url, action)
            return

        redirected_url = Url(r.url)
        if is_same_url(url, redirected_url) is False:
            res = self.basic_check(redirected_url, action)
            if res == self.RET_GOOD_LINK:
                return
            elif res == self.RET_BAD_LINK:
                return

        if self.safeBrowsingAPI:
            if self.safeBrowsingAPI.check_url(
                    redirected_url.url):  # harmful url detected
                log.debug('Bad url because google api')
                self.counteract_bad_url(url, action, want_to_blacklist=False)
                self.counteract_bad_url(redirected_url,
                                        want_to_blacklist=False)
                return

        if 'content-type' not in r.headers or not r.headers[
                'content-type'].startswith('text/html'):
            return  # can't analyze non-html content
        maximum_size = 1024 * 1024 * 10  # 10 MB
        receive_timeout = 3

        html = ''
        try:
            response = requests.get(url=url.url,
                                    stream=True,
                                    timeout=(connection_timeout, read_timeout))

            content_length = response.headers.get('Content-Length')
            if content_length and int(
                    response.headers.get('Content-Length')) > maximum_size:
                log.error('This file is too big!')
                return

            size = 0
            start = time.time()

            for chunk in response.iter_content(1024):
                if time.time() - start > receive_timeout:
                    log.error('The site took too long to load')
                    return

                size += len(chunk)
                if size > maximum_size:
                    log.error('This file is too big! (fake header)')
                    return
                html += str(chunk)

        except requests.exceptions.ConnectTimeout:
            log.warning('Connection timed out while checking {0}'.format(
                url.url))
            self.cache_url(url.url, True)
            return
        except requests.exceptions.ReadTimeout:
            log.warning('Reading timed out while checking {0}'.format(url.url))
            self.cache_url(url.url, True)
            return
        except:
            log.exception('Unhandled exception')
            return

        try:
            soup = BeautifulSoup(html, 'html.parser')
        except:
            return

        original_url = url
        original_redirected_url = redirected_url
        urls = []
        for link in soup.find_all(
                'a'):  # get a list of links to external sites
            url = link.get('href')
            if url is None:
                continue
            if url.startswith('//'):
                urls.append('http:' + url)
            elif url.startswith('http://') or url.startswith('https://'):
                urls.append(url)

        for url in urls:  # check if the site links to anything dangerous
            url = Url(url)

            if is_subdomain(url.parsed.netloc, original_url.parsed.netloc):
                # log.debug('Skipping because internal link')
                continue

            log.debug('Checking sublink {0}'.format(url.url))
            res = self.basic_check(url, action, sublink=True)
            if res == self.RET_BAD_LINK:
                self.counteract_bad_url(url)
                self.counteract_bad_url(original_url, want_to_blacklist=False)
                self.counteract_bad_url(original_redirected_url,
                                        want_to_blacklist=False)
                return
            elif res == self.RET_GOOD_LINK:
                continue

            try:
                r = requests.head(url.url,
                                  allow_redirects=True,
                                  timeout=connection_timeout)
            except:
                continue

            redirected_url = Url(r.url)
            if not is_same_url(url, redirected_url):
                res = self.basic_check(redirected_url, action, sublink=True)
                if res == self.RET_BAD_LINK:
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(original_url,
                                            want_to_blacklist=False)
                    self.counteract_bad_url(original_redirected_url,
                                            want_to_blacklist=False)
                    return
                elif res == self.RET_GOOD_LINK:
                    continue

            if self.safeBrowsingAPI:
                if self.safeBrowsingAPI.check_url(
                        redirected_url.url):  # harmful url detected
                    log.debug('Evil sublink {0} by google API'.format(url))
                    self.counteract_bad_url(original_url, action)
                    self.counteract_bad_url(original_redirected_url)
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(redirected_url)
                    return

        # if we got here, the site is clean for our standards
        self.cache_url(original_url.url, True)
        self.cache_url(original_redirected_url.url, True)
        return

    def load_commands(self, **options):
        self.commands['add'] = pajbot.models.command.Command.multiaction_command(
            level=100,
            delay_all=0,
            delay_user=0,
            default=None,
            command='add',
            commands={
                'link':
                pajbot.models.command.Command.multiaction_command(
                    level=500,
                    delay_all=0,
                    delay_user=0,
                    default=None,
                    commands={
                        'blacklist':
                        pajbot.models.
                        command.Command.raw_command(
                            self
                            .add_link_blacklist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description=
                            'Blacklist a link',
                            examples=[
                                pajbot.
                                models.command.CommandExample(
                                    None,
                                    'Add a link to the blacklist for a shallow search',
                                    chat=
                                    'user:!add link blacklist --shallow scamlink.lonk/\n'
                                    'bot>user:Successfully added your links',
                                    description=
                                    'Added the link scamlink.lonk/ to the blacklist for a shallow search'
                                ).parse(),
                                pajbot.models.command.CommandExample(
                                    None,
                                    'Add a link to the blacklist for a deep search',
                                    chat=
                                    'user:!add link blacklist --deep scamlink.lonk/\n'
                                    'bot>user:Successfully added your links',
                                    description=
                                    'Added the link scamlink.lonk/ to the blacklist for a deep search'
                                ).parse(),
                            ]),
                        'whitelist':
                        pajbot.models.command.Command.raw_command(
                            self.add_link_whitelist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description='Whitelist a link',
                            examples=[
                                pajbot.models.command.CommandExample(
                                    None,
                                    'Add a link to the whitelist',
                                    chat=
                                    'user:!add link whitelink safelink.lonk/\n'
                                    'bot>user:Successfully added your links',
                                    description=
                                    'Added the link safelink.lonk/ to the whitelist'
                                ).parse(),
                            ]),
                    })
            })

        self.commands['remove'] = pajbot.models.command.Command.multiaction_command(
            level=100,
            delay_all=0,
            delay_user=0,
            default=None,
            command='remove',
            commands={
                'link':
                pajbot.models.command.Command.multiaction_command(
                    level=500,
                    delay_all=0,
                    delay_user=0,
                    default=None,
                    commands={
                        'blacklist':
                        pajbot.models.
                        command.Command.raw_command(
                            self
                            .remove_link_blacklist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description=
                            'Remove a link from the blacklist.',
                            examples=[
                                pajbot.
                                models.command.CommandExample(
                                    None,
                                    'Remove a link from the blacklist.',
                                    chat
                                    ='user:!remove link blacklist 20\n'
                                    'bot>user:Successfully removed blacklisted link with id 20',
                                    description=
                                    'Remove a link from the blacklist with an ID'
                                ).parse(),
                            ]),
                        'whitelist':
                        pajbot.models.command.Command.raw_command(
                            self
                            .remove_link_whitelist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description='Remove a link from the whitelist.',
                            examples=[
                                pajbot.models.command.CommandExample(
                                    None,
                                    'Remove a link from the whitelist.',
                                    chat='user:!remove link whitelist 12\n'
                                    'bot>user:Successfully removed blacklisted link with id 12',
                                    description=
                                    'Remove a link from the whitelist with an ID'
                                ).parse(),
                            ]),
                    }),
            })

    def add_link_blacklist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        options, new_links = self.parse_link_blacklist_arguments(message)

        if new_links:
            parts = new_links.split(' ')
            try:
                for link in parts:
                    if len(link) > 1:
                        self.blacklist_url(link, **options)
                        AdminLogManager.post('Blacklist link added', source,
                                             link)
                bot.whisper(source.username, 'Successfully added your links')
                return True
            except:
                log.exception('Unhandled exception in add_link_blacklist')
                bot.whisper(source.username,
                            'Some error occurred while adding your links')
                return False
        else:
            bot.whisper(source.username, 'Usage: !add link blacklist LINK')
            return False

    def add_link_whitelist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.whitelist_url(link)
                AdminLogManager.post('Whitelist link added', source, link)
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username,
                        'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully added your links')

    def remove_link_blacklist(self, **options):
        message = options['message']
        bot = options['bot']
        source = options['source']

        if message:
            id = None
            try:
                id = int(message)
            except ValueError:
                pass

            link = self.db_session.query(BlacklistedLink).filter_by(
                id=id).one_or_none()

            if link:
                self.blacklisted_links.remove(link)
                self.db_session.delete(link)
                self.db_session.commit()
            else:
                bot.whisper(source.username, 'No link with the given id found')
                return False

            AdminLogManager.post('Blacklist link removed', source, link.domain)
            bot.whisper(
                source.username,
                'Successfully removed blacklisted link with id {0}'.format(
                    link.id))
        else:
            bot.whisper(source.username, 'Usage: !remove link blacklist ID')
            return False

    def remove_link_whitelist(self, **options):
        message = options['message']
        bot = options['bot']
        source = options['source']

        if message:
            id = None
            try:
                id = int(message)
            except ValueError:
                pass

            link = self.db_session.query(WhitelistedLink).filter_by(
                id=id).one_or_none()

            if link:
                self.whitelisted_links.remove(link)
                self.db_session.delete(link)
                self.db_session.commit()
            else:
                bot.whisper(source.username, 'No link with the given id found')
                return False

            AdminLogManager.post('Whitelist link removed', source, link.domain)
            bot.whisper(
                source.username,
                'Successfully removed whitelisted link with id {0}'.format(
                    link.id))
        else:
            bot.whisper(source.username, 'Usage: !remove link whitelist ID')
            return False

    def parse_link_blacklist_arguments(self, message):
        parser = argparse.ArgumentParser()
        parser.add_argument('--deep', dest='level', action='store_true')
        parser.add_argument('--shallow', dest='level', action='store_false')
        parser.set_defaults(level=False)

        try:
            args, unknown = parser.parse_known_args(message.split())
        except SystemExit:
            return False, False
        except:
            log.exception('Unhandled exception in add_link_blacklist')
            return False, False

        # Strip options of any values that are set as None
        options = {k: v for k, v in vars(args).items() if v is not None}
        response = ' '.join(unknown)

        return options, response
Esempio n. 5
0
class LinkCheckerModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Link Checker'
    DESCRIPTION = 'Checks links if they\'re bad'
    ENABLED_DEFAULT = True
    SETTINGS = []

    def __init__(self):
        super().__init__()
        self.db_session = None
        self.links = {}

        self.blacklisted_links = []
        self.whitelisted_links = []

        self.cache = LinkCheckerCache()  # cache[url] = True means url is safe, False means the link is bad

        self.action_queue = ActionQueue()
        self.action_queue.start()

    def enable(self, bot):
        self.bot = bot
        if bot:
            bot.add_handler('on_message', self.on_message, priority=100)
            bot.add_handler('on_commit', self.on_commit)
            self.run_later = bot.execute_delayed

            if 'safebrowsingapi' in bot.config['main']:
                # XXX: This should be loaded as a setting instead.
                # There needs to be a setting for settings to have them as "passwords"
                # so they're not displayed openly
                self.safeBrowsingAPI = SafeBrowsingAPI(bot.config['main']['safebrowsingapi'], bot.nickname, bot.version)
            else:
                self.safeBrowsingAPI = None

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
        self.db_session = DBManager.create_session()
        self.blacklisted_links = []
        for link in self.db_session.query(BlacklistedLink):
            self.blacklisted_links.append(link)

        self.whitelisted_links = []
        for link in self.db_session.query(WhitelistedLink):
            self.whitelisted_links.append(link)

    def disable(self, bot):
        if bot:
            bot.remove_handler('on_message', self.on_message)
            bot.remove_handler('on_commit', self.on_commit)

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
            self.blacklisted_links = []
            self.whitelisted_links = []

    def reload(self):

        log.info('Loaded {0} bad links and {1} good links'.format(len(self.blacklisted_links), len(self.whitelisted_links)))
        return self

    def on_message(self, source, message, emotes, whisper, urls):
        if not whisper and source.level < 500 and source.moderator is False:
            for url in urls:
                # Action which will be taken when a bad link is found
                action = Action(self.bot.timeout, args=[source.username, 20])
                # First we perform a basic check
                if self.simple_check(url, action) == self.RET_FURTHER_ANALYSIS:
                    # If the basic check returns no relevant data, we queue up a proper check on the URL
                    self.action_queue.add(self.check_url, args=[url, action])

    def on_commit(self):
        if self.db_session is not None:
            self.db_session.commit()

    def delete_from_cache(self, url):
        if url in self.cache:
            log.debug("LinkChecker: Removing url {0} from cache".format(url))
            del self.cache[url]

    def cache_url(self, url, safe):
        if url in self.cache and self.cache[url] == safe:
            return

        log.debug("LinkChecker: Caching url {0} as {1}".format(url, 'SAFE' if safe is True else 'UNSAFE'))
        self.cache[url] = safe
        self.run_later(20, self.delete_from_cache, (url, ))

    def counteract_bad_url(self, url, action=None, want_to_cache=True, want_to_blacklist=True):
        log.debug("LinkChecker: BAD URL FOUND {0}".format(url.url))
        if action:
            action.run()
        if want_to_cache:
            self.cache_url(url.url, False)
        if want_to_blacklist:
            self.blacklist_url(url.url, url.parsed)

    def unlist_url(self, url, list_type, parsed_url=None):
        """ list_type is either 'blacklist' or 'whitelist' """
        if not (url.startswith('http://') or url.startswith('https://')):
            url = 'http://' + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        domain = parsed_url.netloc
        path = parsed_url.path

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        if list_type == 'blacklist':
            link = self.db_session.query(BlacklistedLink).filter_by(domain=domain, path=path).one_or_none()
            if link:
                self.blacklisted_links.remove(link)
                self.db_session.delete(link)
            else:
                log.warning('Unable to unlist {0}{1}'.format(domain, path))
        elif list_type == 'whitelist':
            link = self.db_session.query(WhitelistedLink).filter_by(domain=domain, path=path).one_or_none()
            if link:
                self.whitelisted_links.remove(link)
                self.db_session.delete(link)
            else:
                log.warning('Unable to unlist {0}{1}'.format(domain, path))

    def blacklist_url(self, url, parsed_url=None, level=1):
        if not (url.lower().startswith('http://') or url.lower().startswith('https://')):
            url = 'http://' + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        if self.is_blacklisted(url, parsed_url):
            return False

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = BlacklistedLink(domain, path, level)
        self.db_session.add(link)
        self.blacklisted_links.append(link)
        return True

    def whitelist_url(self, url, parsed_url=None):
        if not (url.lower().startswith('http://') or url.lower().startswith('https://')):
            url = 'http://' + url
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        if self.is_whitelisted(url, parsed_url):
            return

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = WhitelistedLink(domain, path)
        self.db_session.add(link)
        self.whitelisted_links.append(link)

    def is_blacklisted(self, url, parsed_url=None, sublink=False):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.blacklisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    if not sublink:
                        return True
                    elif link.level >= 1:  # if it's a sublink, but the blacklisting level is 0, we don't consider it blacklisted
                        return True

        return False

    def is_whitelisted(self, url, parsed_url=None):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.whitelisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    return True

        return False

    RET_BAD_LINK = -1
    RET_FURTHER_ANALYSIS = 0
    RET_GOOD_LINK = 1

    def basic_check(self, url, action, sublink=False):
        """
        Check if the url is in the cache, or if it's
        Return values:
        1 = Link is OK
        -1 = Link is bad
        0 = Link needs further analysis
        """
        if url.url in self.cache:
            log.debug("LinkChecker: Url {0} found in cache".format(url.url))
            if not self.cache[url.url]:  # link is bad
                self.counteract_bad_url(url, action, False, False)
                return self.RET_BAD_LINK
            return self.RET_GOOD_LINK

        log.info('Checking if link is blacklisted...')
        if self.is_blacklisted(url.url, url.parsed, sublink):
            log.debug("LinkChecker: Url {0} is blacklisted".format(url.url))
            self.counteract_bad_url(url, action, want_to_blacklist=False)
            return self.RET_BAD_LINK

        log.info('Checking if link is whitelisted...')
        if self.is_whitelisted(url.url, url.parsed):
            log.debug("LinkChecker: Url {0} allowed by the whitelist".format(url.url))
            self.cache_url(url.url, True)
            return self.RET_GOOD_LINK

        return self.RET_FURTHER_ANALYSIS

    def simple_check(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return self.RET_FURTHER_ANALYSIS

        return self.basic_check(url, action)

    def check_url(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return

        try:
            self._check_url(url, action)
        except:
            log.exception("LinkChecker unhanled exception while _check_url")

    def _check_url(self, url, action):
        log.debug("LinkChecker: Checking url {0}".format(url.url))

        # XXX: The basic check is currently performed twice on links found in messages. Solve
        res = self.basic_check(url, action)
        if res == self.RET_GOOD_LINK:
            return
        elif res == self.RET_BAD_LINK:
            return

        connection_timeout = 2
        read_timeout = 1
        try:
            r = requests.head(url.url, allow_redirects=True, timeout=connection_timeout)
        except:
            self.cache_url(url.url, True)
            return

        checkcontenttype = ('content-type' in r.headers and r.headers['content-type'] == 'application/octet-stream')
        checkdispotype = ('disposition-type' in r.headers and r.headers['disposition-type'] == 'attachment')

        if checkcontenttype or checkdispotype:  # triggering a download not allowed
            self.counteract_bad_url(url, action)
            return

        redirected_url = Url(r.url)
        if is_same_url(url, redirected_url) is False:
            res = self.basic_check(redirected_url, action)
            if res == self.RET_GOOD_LINK:
                return
            elif res == self.RET_BAD_LINK:
                return

        if self.safeBrowsingAPI:
            if self.safeBrowsingAPI.check_url(redirected_url.url):  # harmful url detected
                log.debug("Bad url because google api")
                self.counteract_bad_url(url, action)
                self.counteract_bad_url(redirected_url)
                return

        if 'content-type' not in r.headers or not r.headers['content-type'].startswith('text/html'):
            return  # can't analyze non-html content
        maximum_size = 1024 * 1024 * 10  # 10 MB
        receive_timeout = 3

        html = ''
        try:
            response = requests.get(url=url.url, stream=True, timeout=(connection_timeout, read_timeout))

            content_length = response.headers.get('Content-Length')
            if content_length and int(response.headers.get('Content-Length')) > maximum_size:
                log.error('This file is too big!')
                return

            size = 0
            start = time.time()

            for chunk in response.iter_content(1024):
                if time.time() - start > receive_timeout:
                    log.error('The site took too long to load')
                    return

                size += len(chunk)
                if size > maximum_size:
                    log.error('This file is too big! (fake header)')
                    return
                html += str(chunk)

        except requests.exceptions.ConnectTimeout:
            log.warning('Connection timed out while checking {0}'.format(url.url))
            self.cache_url(url.url, True)
            return
        except requests.exceptions.ReadTimeout:
            log.warning('Reading timed out while checking {0}'.format(url.url))
            self.cache_url(url.url, True)
            return
        except:
            log.exception('Unhandled exception')
            return

        try:
            soup = BeautifulSoup(html, 'html.parser')
        except:
            return

        original_url = url
        original_redirected_url = redirected_url
        urls = []
        for link in soup.find_all('a'):  # get a list of links to external sites
            url = link.get('href')
            if url is None:
                continue
            if url.startswith('//'):
                urls.append('http:' + url)
            elif url.startswith('http://') or url.startswith('https://'):
                urls.append(url)

        for url in urls:  # check if the site links to anything dangerous
            url = Url(url)

            if is_subdomain(url.parsed.netloc, original_url.parsed.netloc):
                # log.debug("Skipping because internal link")
                continue

            log.debug("Checking sublink {0}".format(url.url))
            res = self.basic_check(url, action, sublink=True)
            if res == self.RET_BAD_LINK:
                self.counteract_bad_url(url)
                self.counteract_bad_url(original_url, want_to_blacklist=False)
                self.counteract_bad_url(original_redirected_url, want_to_blacklist=False)
                return
            elif res == self.RET_GOOD_LINK:
                continue

            try:
                r = requests.head(url.url, allow_redirects=True, timeout=connection_timeout)
            except:
                continue

            redirected_url = Url(r.url)
            if not is_same_url(url, redirected_url):
                res = self.basic_check(redirected_url, action, sublink=True)
                if res == self.RET_BAD_LINK:
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(original_url, want_to_blacklist=False)
                    self.counteract_bad_url(original_redirected_url, want_to_blacklist=False)
                    return
                elif res == self.RET_GOOD_LINK:
                    continue

            if self.safeBrowsingAPI:
                if self.safeBrowsingAPI.check_url(redirected_url.url):  # harmful url detected
                    log.debug("Evil sublink {0} by google API".format(url))
                    self.counteract_bad_url(original_url, action)
                    self.counteract_bad_url(original_redirected_url)
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(redirected_url)
                    return

        # if we got here, the site is clean for our standards
        self.cache_url(original_url.url, True)
        self.cache_url(original_redirected_url.url, True)
        return

    def load_commands(self, **options):
        self.commands['add'] = Command.multiaction_command(
                level=100,
                delay_all=0,
                delay_user=0,
                default=None,
                command='add',
                commands={
                    'link': Command.multiaction_command(
                        level=500,
                        delay_all=0,
                        delay_user=0,
                        default=None,
                        commands={
                            'blacklist': Command.raw_command(self.add_link_blacklist,
                                level=500,
                                description='Blacklist a link',
                                examples=[
                                    CommandExample(None, 'Add a link to the blacklist for shallow search',
                                        chat='user:!add link blacklist 0 scamlink.lonk/\n'
                                        'bot>user:Successfully added your links',
                                        description='Added the link scamlink.lonk/ to the blacklist for a shallow search').parse(),
                                    CommandExample(None, 'Add a link to the blacklist for deep search',
                                        chat='user:!add link blacklist 1 scamlink.lonk/\n'
                                        'bot>user:Successfully added your links',
                                        description='Added the link scamlink.lonk/ to the blacklist for a deep search').parse(),
                                    ]),
                            'whitelist': Command.raw_command(self.add_link_whitelist,
                                level=500,
                                description='Whitelist a link',
                                examples=[
                                    CommandExample(None, 'Add a link to the whitelist',
                                        chat='user:!add link whitelink safelink.lonk/\n'
                                        'bot>user:Successfully added your links',
                                        description='Added the link safelink.lonk/ to the whitelist').parse(),
                                    ]),
                            }
                        )
                    }
                )

        self.commands['remove'] = Command.multiaction_command(
                level=100,
                delay_all=0,
                delay_user=0,
                default=None,
                command='remove',
                commands={
                    'link': Command.multiaction_command(
                        level=500,
                        delay_all=0,
                        delay_user=0,
                        default=None,
                        commands={
                            'blacklist': Command.raw_command(self.remove_link_blacklist,
                                level=500,
                                description='Unblacklist a link',
                                examples=[
                                    CommandExample(None, 'Remove a blacklist link',
                                        chat='user:!remove link blacklist scamtwitch.scam\n'
                                        'bot>user:Successfully removed your links',
                                        description='Removes scamtwitch.scam as a blacklisted link').parse(),
                                    ]),
                            'whitelist': Command.raw_command(self.remove_link_whitelist,
                                level=500,
                                description='Unwhitelist a link',
                                examples=[
                                    CommandExample(None, 'Remove a whitelist link',
                                        chat='user:!remove link whitelist twitch.safe\n'
                                        'bot>user:Successfully removed your links',
                                        description='Removes twitch.safe as a whitelisted link').parse(),
                                    ]),
                            }
                        ),
                    }
                )

    def add_link_blacklist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            if not parts[0].isnumeric():
                for link in parts:
                    self.blacklist_url(link)
            else:
                for link in parts[1:]:
                    self.blacklist_url(link, level=int(parts[0]))
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username, 'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully added your links')

    def add_link_whitelist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.whitelist_url(link)
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username, 'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully added your links')

    def remove_link_blacklist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.unlist_url(link, 'blacklist')
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username, 'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully removed your links')

    def remove_link_whitelist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.unlist_url(link, 'whitelist')
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username, 'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully removed your links')
Esempio n. 6
0
class LinkCheckerModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Link Checker'
    DESCRIPTION = 'Checks links if they\'re bad'
    ENABLED_DEFAULT = True
    CATEGORY = 'Filter'
    SETTINGS = [
            ModuleSetting(
                key='ban_pleb_links',
                label='Disallow links from non-subscribers',
                type='boolean',
                required=True,
                default=False)
            ]

    def __init__(self):
        super().__init__()
        self.db_session = None
        self.links = {}

        self.blacklisted_links = []
        self.whitelisted_links = []

        self.cache = LinkCheckerCache()  # cache[url] = True means url is safe, False means the link is bad

        self.action_queue = ActionQueue()
        self.action_queue.start()

    def enable(self, bot):
        self.bot = bot
        pajbot.managers.handler.HandlerManager.add_handler('on_message', self.on_message, priority=100)
        pajbot.managers.handler.HandlerManager.add_handler('on_commit', self.on_commit)
        if bot:
            self.run_later = bot.execute_delayed

            if 'safebrowsingapi' in bot.config['main']:
                # XXX: This should be loaded as a setting instead.
                # There needs to be a setting for settings to have them as "passwords"
                # so they're not displayed openly
                self.safeBrowsingAPI = SafeBrowsingAPI(bot.config['main']['safebrowsingapi'], bot.nickname, bot.version)
            else:
                self.safeBrowsingAPI = None

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
        self.db_session = DBManager.create_session()
        self.blacklisted_links = []
        for link in self.db_session.query(BlacklistedLink):
            self.blacklisted_links.append(link)

        self.whitelisted_links = []
        for link in self.db_session.query(WhitelistedLink):
            self.whitelisted_links.append(link)

    def disable(self, bot):
        pajbot.managers.handler.HandlerManager.remove_handler('on_message', self.on_message)
        pajbot.managers.handler.HandlerManager.remove_handler('on_commit', self.on_commit)

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
            self.blacklisted_links = []
            self.whitelisted_links = []

    def reload(self):

        log.info('Loaded {0} bad links and {1} good links'.format(len(self.blacklisted_links), len(self.whitelisted_links)))
        return self

    super_whitelist = ['pajlada.se', 'pajlada.com', 'forsen.tv', 'pajbot.com']

    def on_message(self, source, message, emotes, whisper, urls, event):
        if not whisper and source.level < 500 and source.moderator is False:
            if self.settings['ban_pleb_links'] is True and source.subscriber is False and len(urls) > 0:
                # Check if the links are in our super-whitelist. i.e. on the pajlada.se domain o forsen.tv
                for url in urls:
                    parsed_url = Url(url)
                    if len(parsed_url.parsed.netloc.split('.')) < 2:
                        continue
                    whitelisted = False
                    for whitelist in self.super_whitelist:
                        if is_subdomain(parsed_url.parsed.netloc, whitelist):
                            whitelisted = True
                            break
                    if whitelisted is False:
                        self.bot.timeout(source.username, 30, reason='Non-subs cannot post links')
                        if source.minutes_in_chat_online > 60:
                            self.bot.whisper(source.username, 'You cannot post non-verified links in chat if you\'re not a subscriber.')
                        return False

            for url in urls:
                # Action which will be taken when a bad link is found
                action = Action(self.bot.timeout, args=[source.username, 20], kwargs={'reason': 'Banned link'})
                # First we perform a basic check
                if self.simple_check(url, action) == self.RET_FURTHER_ANALYSIS:
                    # If the basic check returns no relevant data, we queue up a proper check on the URL
                    self.action_queue.add(self.check_url, args=[url, action])

    def on_commit(self):
        if self.db_session is not None:
            self.db_session.commit()

    def delete_from_cache(self, url):
        if url in self.cache:
            log.debug('LinkChecker: Removing url {0} from cache'.format(url))
            del self.cache[url]

    def cache_url(self, url, safe):
        if url in self.cache and self.cache[url] == safe:
            return

        log.debug('LinkChecker: Caching url {0} as {1}'.format(url, 'SAFE' if safe is True else 'UNSAFE'))
        self.cache[url] = safe
        self.run_later(20, self.delete_from_cache, (url, ))

    def counteract_bad_url(self, url, action=None, want_to_cache=True, want_to_blacklist=False):
        log.debug('LinkChecker: BAD URL FOUND {0}'.format(url.url))
        if action:
            action.run()
        if want_to_cache:
            self.cache_url(url.url, False)
        if want_to_blacklist:
            self.blacklist_url(url.url, url.parsed)
            return True

    def blacklist_url(self, url, parsed_url=None, level=0):
        if not (url.lower().startswith('http://') or url.lower().startswith('https://')):
            url = 'http://' + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        if self.is_blacklisted(url, parsed_url):
            return False

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = BlacklistedLink(domain, path, level)
        self.db_session.add(link)
        self.blacklisted_links.append(link)
        self.db_session.commit()

    def whitelist_url(self, url, parsed_url=None):
        if not (url.lower().startswith('http://') or url.lower().startswith('https://')):
            url = 'http://' + url
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        if self.is_whitelisted(url, parsed_url):
            return

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = WhitelistedLink(domain, path)
        self.db_session.add(link)
        self.whitelisted_links.append(link)
        self.db_session.commit()

    def is_blacklisted(self, url, parsed_url=None, sublink=False):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.blacklisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    if not sublink:
                        return True
                    elif link.level >= 1:  # if it's a sublink, but the blacklisting level is 0, we don't consider it blacklisted
                        return True

        return False

    def is_whitelisted(self, url, parsed_url=None):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.whitelisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    return True

        return False

    RET_BAD_LINK = -1
    RET_FURTHER_ANALYSIS = 0
    RET_GOOD_LINK = 1

    def basic_check(self, url, action, sublink=False):
        """
        Check if the url is in the cache, or if it's
        Return values:
        1 = Link is OK
        -1 = Link is bad
        0 = Link needs further analysis
        """
        if url.url in self.cache:
            log.debug('LinkChecker: Url {0} found in cache'.format(url.url))
            if not self.cache[url.url]:  # link is bad
                self.counteract_bad_url(url, action, False, False)
                return self.RET_BAD_LINK
            return self.RET_GOOD_LINK

        log.info('Checking if link is blacklisted...')
        if self.is_blacklisted(url.url, url.parsed, sublink):
            log.debug('LinkChecker: Url {0} is blacklisted'.format(url.url))
            self.counteract_bad_url(url, action, want_to_blacklist=False)
            return self.RET_BAD_LINK

        log.info('Checking if link is whitelisted...')
        if self.is_whitelisted(url.url, url.parsed):
            log.debug('LinkChecker: Url {0} allowed by the whitelist'.format(url.url))
            self.cache_url(url.url, True)
            return self.RET_GOOD_LINK

        return self.RET_FURTHER_ANALYSIS

    def simple_check(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return self.RET_FURTHER_ANALYSIS

        return self.basic_check(url, action)

    def check_url(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return

        try:
            self._check_url(url, action)
        except:
            log.exception('LinkChecker unhanled exception while _check_url')

    def _check_url(self, url, action):
        log.debug('LinkChecker: Checking url {0}'.format(url.url))

        # XXX: The basic check is currently performed twice on links found in messages. Solve
        res = self.basic_check(url, action)
        if res == self.RET_GOOD_LINK:
            return
        elif res == self.RET_BAD_LINK:
            return

        connection_timeout = 2
        read_timeout = 1
        try:
            r = requests.head(url.url, allow_redirects=True, timeout=connection_timeout)
        except:
            self.cache_url(url.url, True)
            return

        checkcontenttype = ('content-type' in r.headers and r.headers['content-type'] == 'application/octet-stream')
        checkdispotype = ('disposition-type' in r.headers and r.headers['disposition-type'] == 'attachment')

        if checkcontenttype or checkdispotype:  # triggering a download not allowed
            self.counteract_bad_url(url, action)
            return

        redirected_url = Url(r.url)
        if is_same_url(url, redirected_url) is False:
            res = self.basic_check(redirected_url, action)
            if res == self.RET_GOOD_LINK:
                return
            elif res == self.RET_BAD_LINK:
                return

        if self.safeBrowsingAPI:
            if self.safeBrowsingAPI.check_url(redirected_url.url):  # harmful url detected
                log.debug('Bad url because google api')
                self.counteract_bad_url(url, action, want_to_blacklist=False)
                self.counteract_bad_url(redirected_url, want_to_blacklist=False)
                return

        if 'content-type' not in r.headers or not r.headers['content-type'].startswith('text/html'):
            return  # can't analyze non-html content
        maximum_size = 1024 * 1024 * 10  # 10 MB
        receive_timeout = 3

        html = ''
        try:
            response = requests.get(url=url.url, stream=True, timeout=(connection_timeout, read_timeout))

            content_length = response.headers.get('Content-Length')
            if content_length and int(response.headers.get('Content-Length')) > maximum_size:
                log.error('This file is too big!')
                return

            size = 0
            start = time.time()

            for chunk in response.iter_content(1024):
                if time.time() - start > receive_timeout:
                    log.error('The site took too long to load')
                    return

                size += len(chunk)
                if size > maximum_size:
                    log.error('This file is too big! (fake header)')
                    return
                html += str(chunk)

        except requests.exceptions.ConnectTimeout:
            log.warning('Connection timed out while checking {0}'.format(url.url))
            self.cache_url(url.url, True)
            return
        except requests.exceptions.ReadTimeout:
            log.warning('Reading timed out while checking {0}'.format(url.url))
            self.cache_url(url.url, True)
            return
        except:
            log.exception('Unhandled exception')
            return

        try:
            soup = BeautifulSoup(html, 'html.parser')
        except:
            return

        original_url = url
        original_redirected_url = redirected_url
        urls = []
        for link in soup.find_all('a'):  # get a list of links to external sites
            url = link.get('href')
            if url is None:
                continue
            if url.startswith('//'):
                urls.append('http:' + url)
            elif url.startswith('http://') or url.startswith('https://'):
                urls.append(url)

        for url in urls:  # check if the site links to anything dangerous
            url = Url(url)

            if is_subdomain(url.parsed.netloc, original_url.parsed.netloc):
                # log.debug('Skipping because internal link')
                continue

            log.debug('Checking sublink {0}'.format(url.url))
            res = self.basic_check(url, action, sublink=True)
            if res == self.RET_BAD_LINK:
                self.counteract_bad_url(url)
                self.counteract_bad_url(original_url, want_to_blacklist=False)
                self.counteract_bad_url(original_redirected_url, want_to_blacklist=False)
                return
            elif res == self.RET_GOOD_LINK:
                continue

            try:
                r = requests.head(url.url, allow_redirects=True, timeout=connection_timeout)
            except:
                continue

            redirected_url = Url(r.url)
            if not is_same_url(url, redirected_url):
                res = self.basic_check(redirected_url, action, sublink=True)
                if res == self.RET_BAD_LINK:
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(original_url, want_to_blacklist=False)
                    self.counteract_bad_url(original_redirected_url, want_to_blacklist=False)
                    return
                elif res == self.RET_GOOD_LINK:
                    continue

            if self.safeBrowsingAPI:
                if self.safeBrowsingAPI.check_url(redirected_url.url):  # harmful url detected
                    log.debug('Evil sublink {0} by google API'.format(url))
                    self.counteract_bad_url(original_url, action)
                    self.counteract_bad_url(original_redirected_url)
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(redirected_url)
                    return

        # if we got here, the site is clean for our standards
        self.cache_url(original_url.url, True)
        self.cache_url(original_redirected_url.url, True)
        return

    def load_commands(self, **options):
        self.commands['add'] = pajbot.models.command.Command.multiaction_command(
                level=100,
                delay_all=0,
                delay_user=0,
                default=None,
                command='add',
                commands={
                    'link': pajbot.models.command.Command.multiaction_command(
                        level=500,
                        delay_all=0,
                        delay_user=0,
                        default=None,
                        commands={
                            'blacklist': pajbot.models.command.Command.raw_command(self.add_link_blacklist,
                                level=500,
                                delay_all=0,
                                delay_user=0,
                                description='Blacklist a link',
                                examples=[
                                    pajbot.models.command.CommandExample(None, 'Add a link to the blacklist for a shallow search',
                                        chat='user:!add link blacklist --shallow scamlink.lonk/\n'
                                        'bot>user:Successfully added your links',
                                        description='Added the link scamlink.lonk/ to the blacklist for a shallow search').parse(),
                                    pajbot.models.command.CommandExample(None, 'Add a link to the blacklist for a deep search',
                                        chat='user:!add link blacklist --deep scamlink.lonk/\n'
                                        'bot>user:Successfully added your links',
                                        description='Added the link scamlink.lonk/ to the blacklist for a deep search').parse(),
                                    ]),
                            'whitelist': pajbot.models.command.Command.raw_command(self.add_link_whitelist,
                                level=500,
                                delay_all=0,
                                delay_user=0,
                                description='Whitelist a link',
                                examples=[
                                    pajbot.models.command.CommandExample(None, 'Add a link to the whitelist',
                                        chat='user:!add link whitelink safelink.lonk/\n'
                                        'bot>user:Successfully added your links',
                                        description='Added the link safelink.lonk/ to the whitelist').parse(),
                                    ]),
                            }
                        )
                    }
                )

        self.commands['remove'] = pajbot.models.command.Command.multiaction_command(
                level=100,
                delay_all=0,
                delay_user=0,
                default=None,
                command='remove',
                commands={
                    'link': pajbot.models.command.Command.multiaction_command(
                        level=500,
                        delay_all=0,
                        delay_user=0,
                        default=None,
                        commands={
                            'blacklist': pajbot.models.command.Command.raw_command(self.remove_link_blacklist,
                                level=500,
                                delay_all=0,
                                delay_user=0,
                                description='Remove a link from the blacklist.',
                                examples=[
                                    pajbot.models.command.CommandExample(None, 'Remove a link from the blacklist.',
                                        chat='user:!remove link blacklist 20\n'
                                        'bot>user:Successfully removed blacklisted link with id 20',
                                        description='Remove a link from the blacklist with an ID').parse(),
                                    ]),
                            'whitelist': pajbot.models.command.Command.raw_command(self.remove_link_whitelist,
                                level=500,
                                delay_all=0,
                                delay_user=0,
                                description='Remove a link from the whitelist.',
                                examples=[
                                    pajbot.models.command.CommandExample(None, 'Remove a link from the whitelist.',
                                        chat='user:!remove link whitelist 12\n'
                                        'bot>user:Successfully removed blacklisted link with id 12',
                                        description='Remove a link from the whitelist with an ID').parse(),
                                    ]),
                            }
                        ),
                    }
                )

    def add_link_blacklist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        options, new_links = self.parse_link_blacklist_arguments(message)

        if new_links:
            parts = new_links.split(' ')
            try:
                for link in parts:
                    if len(link) > 1:
                        self.blacklist_url(link, **options)
                        AdminLogManager.post('Blacklist link added', source, link)
                bot.whisper(source.username, 'Successfully added your links')
                return True
            except:
                log.exception('Unhandled exception in add_link_blacklist')
                bot.whisper(source.username, 'Some error occurred while adding your links')
                return False
        else:
            bot.whisper(source.username, 'Usage: !add link blacklist LINK')
            return False

    def add_link_whitelist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.whitelist_url(link)
                AdminLogManager.post('Whitelist link added', source, link)
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username, 'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully added your links')

    def remove_link_blacklist(self, **options):
        message = options['message']
        bot = options['bot']
        source = options['source']

        if message:
            id = None
            try:
                id = int(message)
            except ValueError:
                pass

            link = self.db_session.query(BlacklistedLink).filter_by(id=id).one_or_none()

            if link:
                self.blacklisted_links.remove(link)
                self.db_session.delete(link)
                self.db_session.commit()
            else:
                bot.whisper(source.username, 'No link with the given id found')
                return False

            AdminLogManager.post('Blacklist link removed', source, link.domain)
            bot.whisper(source.username, 'Successfully removed blacklisted link with id {0}'.format(link.id))
        else:
            bot.whisper(source.username, 'Usage: !remove link blacklist ID')
            return False

    def remove_link_whitelist(self, **options):
        message = options['message']
        bot = options['bot']
        source = options['source']

        if message:
            id = None
            try:
                id = int(message)
            except ValueError:
                pass

            link = self.db_session.query(WhitelistedLink).filter_by(id=id).one_or_none()

            if link:
                self.whitelisted_links.remove(link)
                self.db_session.delete(link)
                self.db_session.commit()
            else:
                bot.whisper(source.username, 'No link with the given id found')
                return False

            AdminLogManager.post('Whitelist link removed', source, link.domain)
            bot.whisper(source.username, 'Successfully removed whitelisted link with id {0}'.format(link.id))
        else:
            bot.whisper(source.username, 'Usage: !remove link whitelist ID')
            return False

    def parse_link_blacklist_arguments(self, message):
        parser = argparse.ArgumentParser()
        parser.add_argument('--deep', dest='level', action='store_true')
        parser.add_argument('--shallow', dest='level', action='store_false')
        parser.set_defaults(level=False)

        try:
            args, unknown = parser.parse_known_args(message.split())
        except SystemExit:
            return False, False
        except:
            log.exception('Unhandled exception in add_link_blacklist')
            return False, False

        # Strip options of any values that are set as None
        options = {k: v for k, v in vars(args).items() if v is not None}
        response = ' '.join(unknown)

        return options, response
Esempio n. 7
0
class LinkCheckerModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Link Checker'
    DESCRIPTION = 'Checks links if they\'re bad'
    ENABLED_DEFAULT = True
    SETTINGS = []

    def __init__(self):
        super().__init__()
        self.db_session = None
        self.links = {}

        self.blacklisted_links = []
        self.whitelisted_links = []

        self.cache = LinkCheckerCache(
        )  # cache[url] = True means url is safe, False means the link is bad

        self.action_queue = ActionQueue()
        self.action_queue.start()

    def enable(self, bot):
        self.bot = bot
        if bot:
            bot.add_handler('on_message', self.on_message, priority=100)
            bot.add_handler('on_commit', self.on_commit)
            self.run_later = bot.execute_delayed

            if 'safebrowsingapi' in bot.config['main']:
                # XXX: This should be loaded as a setting instead.
                # There needs to be a setting for settings to have them as "passwords"
                # so they're not displayed openly
                self.safeBrowsingAPI = SafeBrowsingAPI(
                    bot.config['main']['safebrowsingapi'], bot.nickname,
                    bot.version)
            else:
                self.safeBrowsingAPI = None

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
        self.db_session = DBManager.create_session()
        self.blacklisted_links = []
        for link in self.db_session.query(BlacklistedLink):
            self.blacklisted_links.append(link)

        self.whitelisted_links = []
        for link in self.db_session.query(WhitelistedLink):
            self.whitelisted_links.append(link)

    def disable(self, bot):
        if bot:
            bot.remove_handler('on_message', self.on_message)
            bot.remove_handler('on_commit', self.on_commit)

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
            self.blacklisted_links = []
            self.whitelisted_links = []

    def reload(self):

        log.info('Loaded {0} bad links and {1} good links'.format(
            len(self.blacklisted_links), len(self.whitelisted_links)))
        return self

    def on_message(self, source, message, emotes, whisper, urls):
        if not whisper and source.level < 500 and source.moderator is False:
            for url in urls:
                # Action which will be taken when a bad link is found
                action = Action(self.bot.timeout, args=[source.username, 20])
                # First we perform a basic check
                if self.simple_check(url, action) == self.RET_FURTHER_ANALYSIS:
                    # If the basic check returns no relevant data, we queue up a proper check on the URL
                    self.action_queue.add(self.check_url, args=[url, action])

    def on_commit(self):
        if self.db_session is not None:
            self.db_session.commit()

    def delete_from_cache(self, url):
        if url in self.cache:
            log.debug("LinkChecker: Removing url {0} from cache".format(url))
            del self.cache[url]

    def cache_url(self, url, safe):
        if url in self.cache and self.cache[url] == safe:
            return

        log.debug("LinkChecker: Caching url {0} as {1}".format(
            url, 'SAFE' if safe is True else 'UNSAFE'))
        self.cache[url] = safe
        self.run_later(20, self.delete_from_cache, (url, ))

    def counteract_bad_url(self,
                           url,
                           action=None,
                           want_to_cache=True,
                           want_to_blacklist=True):
        log.debug("LinkChecker: BAD URL FOUND {0}".format(url.url))
        if action:
            action.run()
        if want_to_cache:
            self.cache_url(url.url, False)
        if want_to_blacklist:
            self.blacklist_url(url.url, url.parsed)

    def unlist_url(self, url, list_type, parsed_url=None):
        """ list_type is either 'blacklist' or 'whitelist' """
        if not (url.startswith('http://') or url.startswith('https://')):
            url = 'http://' + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        domain = parsed_url.netloc
        path = parsed_url.path

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        if list_type == 'blacklist':
            link = self.db_session.query(BlacklistedLink).filter_by(
                domain=domain, path=path).one_or_none()
            if link:
                self.blacklisted_links.remove(link)
                self.db_session.delete(link)
            else:
                log.warning('Unable to unlist {0}{1}'.format(domain, path))
        elif list_type == 'whitelist':
            link = self.db_session.query(WhitelistedLink).filter_by(
                domain=domain, path=path).one_or_none()
            if link:
                self.whitelisted_links.remove(link)
                self.db_session.delete(link)
            else:
                log.warning('Unable to unlist {0}{1}'.format(domain, path))

    def blacklist_url(self, url, parsed_url=None, level=1):
        if not (url.lower().startswith('http://')
                or url.lower().startswith('https://')):
            url = 'http://' + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        if self.is_blacklisted(url, parsed_url):
            return False

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = BlacklistedLink(domain, path, level)
        self.db_session.add(link)
        self.blacklisted_links.append(link)
        return True

    def whitelist_url(self, url, parsed_url=None):
        if not (url.lower().startswith('http://')
                or url.lower().startswith('https://')):
            url = 'http://' + url
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        if self.is_whitelisted(url, parsed_url):
            return

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith('www.'):
            domain = domain[4:]
        if path.endswith('/'):
            path = path[:-1]
        if path == '':
            path = '/'

        link = WhitelistedLink(domain, path)
        self.db_session.add(link)
        self.whitelisted_links.append(link)

    def is_blacklisted(self, url, parsed_url=None, sublink=False):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.blacklisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    if not sublink:
                        return True
                    elif link.level >= 1:  # if it's a sublink, but the blacklisting level is 0, we don't consider it blacklisted
                        return True

        return False

    def is_whitelisted(self, url, parsed_url=None):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == '':
            path = '/'

        domain_split = domain.split('.')
        if len(domain_split) < 2:
            return False

        for link in self.whitelisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    return True

        return False

    RET_BAD_LINK = -1
    RET_FURTHER_ANALYSIS = 0
    RET_GOOD_LINK = 1

    def basic_check(self, url, action, sublink=False):
        """
        Check if the url is in the cache, or if it's
        Return values:
        1 = Link is OK
        -1 = Link is bad
        0 = Link needs further analysis
        """
        if url.url in self.cache:
            log.debug("LinkChecker: Url {0} found in cache".format(url.url))
            if not self.cache[url.url]:  # link is bad
                self.counteract_bad_url(url, action, False, False)
                return self.RET_BAD_LINK
            return self.RET_GOOD_LINK

        log.info('Checking if link is blacklisted...')
        if self.is_blacklisted(url.url, url.parsed, sublink):
            log.debug("LinkChecker: Url {0} is blacklisted".format(url.url))
            self.counteract_bad_url(url, action, want_to_blacklist=False)
            return self.RET_BAD_LINK

        log.info('Checking if link is whitelisted...')
        if self.is_whitelisted(url.url, url.parsed):
            log.debug("LinkChecker: Url {0} allowed by the whitelist".format(
                url.url))
            self.cache_url(url.url, True)
            return self.RET_GOOD_LINK

        return self.RET_FURTHER_ANALYSIS

    def simple_check(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return self.RET_FURTHER_ANALYSIS

        return self.basic_check(url, action)

    def check_url(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split('.')) < 2:
            # The URL is broken, ignore it
            return

        try:
            self._check_url(url, action)
        except:
            log.exception("LinkChecker unhanled exception while _check_url")

    def _check_url(self, url, action):
        log.debug("LinkChecker: Checking url {0}".format(url.url))

        # XXX: The basic check is currently performed twice on links found in messages. Solve
        res = self.basic_check(url, action)
        if res == self.RET_GOOD_LINK:
            return
        elif res == self.RET_BAD_LINK:
            return

        connection_timeout = 2
        read_timeout = 1
        try:
            r = requests.head(url.url,
                              allow_redirects=True,
                              timeout=connection_timeout)
        except:
            self.cache_url(url.url, True)
            return

        checkcontenttype = ('content-type' in r.headers
                            and r.headers['content-type']
                            == 'application/octet-stream')
        checkdispotype = ('disposition-type' in r.headers
                          and r.headers['disposition-type'] == 'attachment')

        if checkcontenttype or checkdispotype:  # triggering a download not allowed
            self.counteract_bad_url(url, action)
            return

        redirected_url = Url(r.url)
        if is_same_url(url, redirected_url) is False:
            res = self.basic_check(redirected_url, action)
            if res == self.RET_GOOD_LINK:
                return
            elif res == self.RET_BAD_LINK:
                return

        if self.safeBrowsingAPI:
            if self.safeBrowsingAPI.check_url(
                    redirected_url.url):  # harmful url detected
                log.debug("Bad url because google api")
                self.counteract_bad_url(url, action)
                self.counteract_bad_url(redirected_url)
                return

        if 'content-type' not in r.headers or not r.headers[
                'content-type'].startswith('text/html'):
            return  # can't analyze non-html content
        maximum_size = 1024 * 1024 * 10  # 10 MB
        receive_timeout = 3

        html = ''
        try:
            response = requests.get(url=url.url,
                                    stream=True,
                                    timeout=(connection_timeout, read_timeout))

            content_length = response.headers.get('Content-Length')
            if content_length and int(
                    response.headers.get('Content-Length')) > maximum_size:
                log.error('This file is too big!')
                return

            size = 0
            start = time.time()

            for chunk in response.iter_content(1024):
                if time.time() - start > receive_timeout:
                    log.error('The site took too long to load')
                    return

                size += len(chunk)
                if size > maximum_size:
                    log.error('This file is too big! (fake header)')
                    return
                html += str(chunk)

        except requests.exceptions.ConnectTimeout:
            log.warning('Connection timed out while checking {0}'.format(
                url.url))
            self.cache_url(url.url, True)
            return
        except requests.exceptions.ReadTimeout:
            log.warning('Reading timed out while checking {0}'.format(url.url))
            self.cache_url(url.url, True)
            return
        except:
            log.exception('Unhandled exception')
            return

        try:
            soup = BeautifulSoup(html, 'html.parser')
        except:
            return

        original_url = url
        original_redirected_url = redirected_url
        urls = []
        for link in soup.find_all(
                'a'):  # get a list of links to external sites
            url = link.get('href')
            if url is None:
                continue
            if url.startswith('//'):
                urls.append('http:' + url)
            elif url.startswith('http://') or url.startswith('https://'):
                urls.append(url)

        for url in urls:  # check if the site links to anything dangerous
            url = Url(url)

            if is_subdomain(url.parsed.netloc, original_url.parsed.netloc):
                # log.debug("Skipping because internal link")
                continue

            log.debug("Checking sublink {0}".format(url.url))
            res = self.basic_check(url, action, sublink=True)
            if res == self.RET_BAD_LINK:
                self.counteract_bad_url(url)
                self.counteract_bad_url(original_url, want_to_blacklist=False)
                self.counteract_bad_url(original_redirected_url,
                                        want_to_blacklist=False)
                return
            elif res == self.RET_GOOD_LINK:
                continue

            try:
                r = requests.head(url.url,
                                  allow_redirects=True,
                                  timeout=connection_timeout)
            except:
                continue

            redirected_url = Url(r.url)
            if not is_same_url(url, redirected_url):
                res = self.basic_check(redirected_url, action, sublink=True)
                if res == self.RET_BAD_LINK:
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(original_url,
                                            want_to_blacklist=False)
                    self.counteract_bad_url(original_redirected_url,
                                            want_to_blacklist=False)
                    return
                elif res == self.RET_GOOD_LINK:
                    continue

            if self.safeBrowsingAPI:
                if self.safeBrowsingAPI.check_url(
                        redirected_url.url):  # harmful url detected
                    log.debug("Evil sublink {0} by google API".format(url))
                    self.counteract_bad_url(original_url, action)
                    self.counteract_bad_url(original_redirected_url)
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(redirected_url)
                    return

        # if we got here, the site is clean for our standards
        self.cache_url(original_url.url, True)
        self.cache_url(original_redirected_url.url, True)
        return

    def load_commands(self, **options):
        self.commands['add'] = Command.multiaction_command(
            level=100,
            delay_all=0,
            delay_user=0,
            default=None,
            command='add',
            commands={
                'link':
                Command.multiaction_command(
                    level=500,
                    delay_all=0,
                    delay_user=0,
                    default=None,
                    commands={
                        'blacklist':
                        Command.raw_command(
                            self.add_link_blacklist,
                            level=500,
                            description='Blacklist a link',
                            examples=[
                                CommandExample(
                                    None,
                                    'Add a link to the blacklist for shallow search',
                                    chat=
                                    'user:!add link blacklist 0 scamlink.lonk/\n'
                                    'bot>user:Successfully added your links',
                                    description=
                                    'Added the link scamlink.lonk/ to the blacklist for a shallow search'
                                ).parse(),
                                CommandExample(
                                    None,
                                    'Add a link to the blacklist for deep search',
                                    chat=
                                    'user:!add link blacklist 1 scamlink.lonk/\n'
                                    'bot>user:Successfully added your links',
                                    description=
                                    'Added the link scamlink.lonk/ to the blacklist for a deep search'
                                ).parse(),
                            ]),
                        'whitelist':
                        Command.raw_command(
                            self.add_link_whitelist,
                            level=500,
                            description='Whitelist a link',
                            examples=[
                                CommandExample(
                                    None,
                                    'Add a link to the whitelist',
                                    chat=
                                    'user:!add link whitelink safelink.lonk/\n'
                                    'bot>user:Successfully added your links',
                                    description=
                                    'Added the link safelink.lonk/ to the whitelist'
                                ).parse(),
                            ]),
                    })
            })

        self.commands['remove'] = Command.multiaction_command(
            level=100,
            delay_all=0,
            delay_user=0,
            default=None,
            command='remove',
            commands={
                'link':
                Command.multiaction_command(
                    level=500,
                    delay_all=0,
                    delay_user=0,
                    default=None,
                    commands={
                        'blacklist':
                        Command.raw_command(
                            self.remove_link_blacklist,
                            level=500,
                            description='Unblacklist a link',
                            examples=[
                                CommandExample(
                                    None,
                                    'Remove a blacklist link',
                                    chat=
                                    'user:!remove link blacklist scamtwitch.scam\n'
                                    'bot>user:Successfully removed your links',
                                    description=
                                    'Removes scamtwitch.scam as a blacklisted link'
                                ).parse(),
                            ]),
                        'whitelist':
                        Command.raw_command(
                            self.remove_link_whitelist,
                            level=500,
                            description='Unwhitelist a link',
                            examples=[
                                CommandExample(
                                    None,
                                    'Remove a whitelist link',
                                    chat=
                                    'user:!remove link whitelist twitch.safe\n'
                                    'bot>user:Successfully removed your links',
                                    description=
                                    'Removes twitch.safe as a whitelisted link'
                                ).parse(),
                            ]),
                    }),
            })

    def add_link_blacklist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            if not parts[0].isnumeric():
                for link in parts:
                    self.blacklist_url(link)
            else:
                for link in parts[1:]:
                    self.blacklist_url(link, level=int(parts[0]))
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username,
                        'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully added your links')

    def add_link_whitelist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.whitelist_url(link)
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username,
                        'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully added your links')

    def remove_link_blacklist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.unlist_url(link, 'blacklist')
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username,
                        'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully removed your links')

    def remove_link_whitelist(self, **options):
        bot = options['bot']
        message = options['message']
        source = options['source']

        parts = message.split(' ')
        try:
            for link in parts:
                self.unlist_url(link, 'whitelist')
        except:
            log.exception('Unhandled exception in add_link')
            bot.whisper(source.username,
                        'Some error occurred white adding your links')
            return False

        bot.whisper(source.username, 'Successfully removed your links')
Esempio n. 8
0
class FollowAgeModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Follow age'
    DESCRIPTION = 'Makes two commands available: !followage and !followsince'
    CATEGORY = 'Feature'

    def __init__(self):
        super().__init__()
        self.action_queue = ActionQueue()
        self.action_queue.start()

    def load_commands(self, **options):
        # TODO: Have delay modifiable in settings

        self.commands['followage'] = pajbot.models.command.Command.raw_command(self.follow_age,
                delay_all=4,
                delay_user=8,
                description='Check your or someone elses follow age for a channel',
                examples=[
                    pajbot.models.command.CommandExample(None, 'Check your own follow age',
                        chat='user:!followage\n'
                        'bot:pajlada, you have been following Karl_Kons for 4 months and 24 days',
                        description='Check how long you have been following the current streamer (Karl_Kons in this case)').parse(),
                    pajbot.models.command.CommandExample(None, 'Check someone elses follow age',
                        chat='user:!followage NightNacht\n'
                        'bot:pajlada, NightNacht has been following Karl_Kons for 5 months and 4 days',
                        description='Check how long any user has been following the current streamer (Karl_Kons in this case)').parse(),
                    pajbot.models.command.CommandExample(None, 'Check someones follow age for a certain streamer',
                        chat='user:!followage NightNacht forsenlol\n'
                        'bot:pajlada, NightNacht has been following forsenlol for 1 year and 4 months',
                        description='Check how long NightNacht has been following forsenlol').parse(),
                    pajbot.models.command.CommandExample(None, 'Check your own follow age for a certain streamer',
                        chat='user:!followage pajlada forsenlol\n'
                        'bot:pajlada, you have been following forsenlol for 1 year and 3 months',
                        description='Check how long you have been following forsenlol').parse(),
                    ],
                )

        self.commands['followsince'] = pajbot.models.command.Command.raw_command(self.follow_since,
                delay_all=4,
                delay_user=8,
                description='Check from when you or someone else first followed a channel',
                examples=[
                    pajbot.models.command.CommandExample(None, 'Check your own follow since',
                        chat='user:!followsince\n'
                        'bot:pajlada, you have been following Karl_Kons since 04 March 2015, 07:02:01 UTC',
                        description='Check when you first followed the current streamer (Karl_Kons in this case)').parse(),
                    pajbot.models.command.CommandExample(None, 'Check someone elses follow since',
                        chat='user:!followsince NightNacht\n'
                        'bot:pajlada, NightNacht has been following Karl_Kons since 03 July 2014, 04:12:42 UTC',
                        description='Check when NightNacht first followed the current streamer (Karl_Kons in this case)').parse(),
                    pajbot.models.command.CommandExample(None, 'Check someone elses follow since for another streamer',
                        chat='user:!followsince NightNacht forsenlol\n'
                        'bot:pajlada, NightNacht has been following forsenlol since 13 June 2013, 13:10:51 UTC',
                        description='Check when NightNacht first followed the given streamer (forsenlol)').parse(),
                    pajbot.models.command.CommandExample(None, 'Check your follow since for another streamer',
                        chat='user:!followsince pajlada forsenlol\n'
                        'bot:pajlada, you have been following forsenlol since 16 December 1990, 03:06:51 UTC',
                        description='Check when you first followed the given streamer (forsenlol)').parse(),
                    ],
                )

    def check_follow_age(self, bot, source, username, streamer):
        streamer = bot.streamer if streamer is None else streamer.lower()
        age = bot.twitchapi.get_follow_relationship(username, streamer)
        if source.username == username:
            if age is False:
                bot.say('{}, you are not following {}'.format(source.username_raw, streamer))
            else:
                bot.say('{}, you have been following {} for {}'.format(source.username_raw, streamer, time_since(datetime.datetime.now().timestamp() - age.timestamp(), 0)))
        else:
            if age is False:
                bot.say('{}, {} is not following {}'.format(source.username_raw, username, streamer))
            else:
                bot.say('{}, {} has been following {} for {}'.format(
                    source.username_raw,
                    username,
                    streamer,
                    time_since(datetime.datetime.now().timestamp() - age.timestamp(), 0)))

    def check_follow_since(self, bot, source, username, streamer):
        streamer = bot.streamer if streamer is None else streamer.lower()
        follow_since = bot.twitchapi.get_follow_relationship(username, streamer)
        if source.username == username:
            if follow_since is False:
                bot.say('{}, you are not following {}'.format(source.username_raw, streamer))
            else:
                bot.say('{}, you have been following {} since {} UTC'.format(
                    source.username_raw,
                    streamer,
                    follow_since.strftime('%d %B %Y, %X')))
        else:
            if follow_since is False:
                bot.say('{}, {} is not following {}'.format(source.username_raw, username, streamer))
            else:
                bot.say('{}, {} has been following {} since {} UTC'.format(
                    source.username_raw,
                    username,
                    streamer,
                    follow_since.strftime('%d %B %Y, %X')))

    def follow_age(self, **options):
        source = options['source']
        message = options['message']
        bot = options['bot']

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_age, args=[bot, source, username, streamer])

    def follow_since(self, **options):
        bot = options['bot']
        source = options['source']
        message = options['message']

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_since, args=[bot, source, username, streamer])

    def parse_message(self, bot, source, message):
        username = source.username
        streamer = None
        if message is not None and len(message) > 0:
            message_split = message.split(' ')
            if len(message_split[0]) and message_split[0].replace('_', '').isalnum():
                username = message_split[0].lower()
            if len(message_split) > 1 and message_split[1].replace('_', '').isalnum():
                streamer = message_split[1]

        return username, streamer
Esempio n. 9
0
class LinkCheckerModule(BaseModule):

    ID = __name__.split(".")[-1]
    NAME = "Link Checker"
    DESCRIPTION = "Checks links if they're bad"
    ENABLED_DEFAULT = True
    CATEGORY = "Filter"
    SETTINGS = [
        ModuleSetting(
            key="ban_pleb_links",
            label="Disallow links from non-subscribers",
            type="boolean",
            required=True,
            default=False,
        ),
        ModuleSetting(key="ban_sub_links",
                      label="Disallow links from subscribers",
                      type="boolean",
                      required=True,
                      default=False),
        ModuleSetting(
            key="timeout_length",
            label="Timeout length",
            type="number",
            required=True,
            placeholder="Timeout length in seconds",
            default=60,
            constraints={
                "min_value": 1,
                "max_value": 3600
            },
        ),
    ]

    def __init__(self, bot):
        super().__init__(bot)
        self.db_session = None
        self.links = {}

        self.blacklisted_links = []
        self.whitelisted_links = []

        self.cache = LinkCheckerCache(
        )  # cache[url] = True means url is safe, False means the link is bad

        self.action_queue = ActionQueue()
        self.action_queue.start()

    def enable(self, bot):
        HandlerManager.add_handler("on_message", self.on_message, priority=100)
        HandlerManager.add_handler("on_commit", self.on_commit)
        if bot:
            self.run_later = bot.execute_delayed

            if "safebrowsingapi" in bot.config["main"]:
                # XXX: This should be loaded as a setting instead.
                # There needs to be a setting for settings to have them as "passwords"
                # so they're not displayed openly
                self.safeBrowsingAPI = SafeBrowsingAPI(
                    bot.config["main"]["safebrowsingapi"], bot.nickname,
                    bot.version)
            else:
                self.safeBrowsingAPI = None

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
        self.db_session = DBManager.create_session()
        self.blacklisted_links = []
        for link in self.db_session.query(BlacklistedLink):
            self.blacklisted_links.append(link)

        self.whitelisted_links = []
        for link in self.db_session.query(WhitelistedLink):
            self.whitelisted_links.append(link)

    def disable(self, bot):
        pajbot.managers.handler.HandlerManager.remove_handler(
            "on_message", self.on_message)
        pajbot.managers.handler.HandlerManager.remove_handler(
            "on_commit", self.on_commit)

        if self.db_session is not None:
            self.db_session.commit()
            self.db_session.close()
            self.db_session = None
            self.blacklisted_links = []
            self.whitelisted_links = []

    def reload(self):

        log.info("Loaded {0} bad links and {1} good links".format(
            len(self.blacklisted_links), len(self.whitelisted_links)))
        return self

    super_whitelist = ["pajlada.se", "pajlada.com", "forsen.tv", "pajbot.com"]

    def on_message(self, source, whisper, urls, **rest):
        if whisper:
            return

        if source.level >= 500 or source.moderator is True:
            return

        if len(urls) > 0:
            do_timeout = False
            ban_reason = "You are not allowed to post links in chat"
            whisper_reason = "??? KKona"

            if self.settings[
                    "ban_pleb_links"] is True and source.subscriber is False:
                do_timeout = True
                whisper_reason = "You cannot post non-verified links in chat if you're not a subscriber."
            elif self.settings[
                    "ban_sub_links"] is True and source.subscriber is True:
                do_timeout = True
                whisper_reason = "You cannot post non-verified links in chat."

            if do_timeout is True:
                # Check if the links are in our super-whitelist. i.e. on the pajlada.se domain o forsen.tv
                for url in urls:
                    parsed_url = Url(url)
                    if len(parsed_url.parsed.netloc.split(".")) < 2:
                        continue
                    whitelisted = False
                    for whitelist in self.super_whitelist:
                        if is_subdomain(parsed_url.parsed.netloc, whitelist):
                            whitelisted = True
                            break
                    if whitelisted is False:
                        self.bot.timeout(source.username,
                                         30,
                                         reason=ban_reason)
                        if source.minutes_in_chat_online > 60:
                            self.bot.whisper(source.username, whisper_reason)
                        return False

        for url in urls:
            # Action which will be taken when a bad link is found
            action = Action(
                self.bot.timeout,
                args=[source.username, self.settings["timeout_length"]],
                kwargs={"reason": "Banned link"},
            )
            # First we perform a basic check
            if self.simple_check(url, action) == self.RET_FURTHER_ANALYSIS:
                # If the basic check returns no relevant data, we queue up a proper check on the URL
                self.action_queue.add(self.check_url, args=[url, action])

    def on_commit(self, **rest):
        if self.db_session is not None:
            self.db_session.commit()

    def delete_from_cache(self, url):
        if url in self.cache:
            log.debug("LinkChecker: Removing url {0} from cache".format(url))
            del self.cache[url]

    def cache_url(self, url, safe):
        if url in self.cache and self.cache[url] == safe:
            return

        log.debug("LinkChecker: Caching url {0} as {1}".format(
            url, "SAFE" if safe is True else "UNSAFE"))
        self.cache[url] = safe
        self.run_later(20, self.delete_from_cache, (url, ))

    def counteract_bad_url(self,
                           url,
                           action=None,
                           want_to_cache=True,
                           want_to_blacklist=False):
        log.debug("LinkChecker: BAD URL FOUND {0}".format(url.url))
        if action:
            action.run()
        if want_to_cache:
            self.cache_url(url.url, False)
        if want_to_blacklist:
            self.blacklist_url(url.url, url.parsed)
            return True

    def blacklist_url(self, url, parsed_url=None, level=0):
        if not (url.lower().startswith("http://")
                or url.lower().startswith("https://")):
            url = "http://" + url

        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)

        if self.is_blacklisted(url, parsed_url):
            return False

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith("www."):
            domain = domain[4:]
        if path.endswith("/"):
            path = path[:-1]
        if path == "":
            path = "/"

        link = BlacklistedLink(domain, path, level)
        self.db_session.add(link)
        self.blacklisted_links.append(link)
        self.db_session.commit()

    def whitelist_url(self, url, parsed_url=None):
        if not (url.lower().startswith("http://")
                or url.lower().startswith("https://")):
            url = "http://" + url
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        if self.is_whitelisted(url, parsed_url):
            return

        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()

        if domain.startswith("www."):
            domain = domain[4:]
        if path.endswith("/"):
            path = path[:-1]
        if path == "":
            path = "/"

        link = WhitelistedLink(domain, path)
        self.db_session.add(link)
        self.whitelisted_links.append(link)
        self.db_session.commit()

    def is_blacklisted(self, url, parsed_url=None, sublink=False):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == "":
            path = "/"

        domain_split = domain.split(".")
        if len(domain_split) < 2:
            return False

        for link in self.blacklisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    if not sublink:
                        return True
                    elif (
                            link.level >= 1
                    ):  # if it's a sublink, but the blacklisting level is 0, we don't consider it blacklisted
                        return True

        return False

    def is_whitelisted(self, url, parsed_url=None):
        if parsed_url is None:
            parsed_url = urllib.parse.urlparse(url)
        domain = parsed_url.netloc.lower()
        path = parsed_url.path.lower()
        if path == "":
            path = "/"

        domain_split = domain.split(".")
        if len(domain_split) < 2:
            return False

        for link in self.whitelisted_links:
            if link.is_subdomain(domain):
                if link.is_subpath(path):
                    return True

        return False

    RET_BAD_LINK = -1
    RET_FURTHER_ANALYSIS = 0
    RET_GOOD_LINK = 1

    def basic_check(self, url, action, sublink=False):
        """
        Check if the url is in the cache, or if it's
        Return values:
        1 = Link is OK
        -1 = Link is bad
        0 = Link needs further analysis
        """
        if url.url in self.cache:
            log.debug("LinkChecker: Url {0} found in cache".format(url.url))
            if not self.cache[url.url]:  # link is bad
                self.counteract_bad_url(url, action, False, False)
                return self.RET_BAD_LINK
            return self.RET_GOOD_LINK

        log.info("Checking if link is blacklisted...")
        if self.is_blacklisted(url.url, url.parsed, sublink):
            log.debug("LinkChecker: Url {0} is blacklisted".format(url.url))
            self.counteract_bad_url(url, action, want_to_blacklist=False)
            return self.RET_BAD_LINK

        log.info("Checking if link is whitelisted...")
        if self.is_whitelisted(url.url, url.parsed):
            log.debug("LinkChecker: Url {0} allowed by the whitelist".format(
                url.url))
            self.cache_url(url.url, True)
            return self.RET_GOOD_LINK

        return self.RET_FURTHER_ANALYSIS

    def simple_check(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split(".")) < 2:
            # The URL is broken, ignore it
            return self.RET_FURTHER_ANALYSIS

        return self.basic_check(url, action)

    def check_url(self, url, action):
        url = Url(url)
        if len(url.parsed.netloc.split(".")) < 2:
            # The URL is broken, ignore it
            return

        try:
            self._check_url(url, action)
        except:
            log.exception("LinkChecker unhanled exception while _check_url")

    def _check_url(self, url, action):
        log.debug("LinkChecker: Checking url {0}".format(url.url))

        # XXX: The basic check is currently performed twice on links found in messages. Solve
        res = self.basic_check(url, action)
        if res == self.RET_GOOD_LINK:
            return
        elif res == self.RET_BAD_LINK:
            return

        connection_timeout = 2
        read_timeout = 1
        try:
            r = requests.head(url.url,
                              allow_redirects=True,
                              timeout=connection_timeout)
        except:
            self.cache_url(url.url, True)
            return

        checkcontenttype = "content-type" in r.headers and r.headers[
            "content-type"] == "application/octet-stream"
        checkdispotype = "disposition-type" in r.headers and r.headers[
            "disposition-type"] == "attachment"

        if checkcontenttype or checkdispotype:  # triggering a download not allowed
            self.counteract_bad_url(url, action)
            return

        redirected_url = Url(r.url)
        if is_same_url(url, redirected_url) is False:
            res = self.basic_check(redirected_url, action)
            if res == self.RET_GOOD_LINK:
                return
            elif res == self.RET_BAD_LINK:
                return

        if self.safeBrowsingAPI:
            if self.safeBrowsingAPI.check_url(
                    redirected_url.url):  # harmful url detected
                log.debug("Bad url because google api")
                self.counteract_bad_url(url, action, want_to_blacklist=False)
                self.counteract_bad_url(redirected_url,
                                        want_to_blacklist=False)
                return

        if "content-type" not in r.headers or not r.headers[
                "content-type"].startswith("text/html"):
            return  # can't analyze non-html content
        maximum_size = 1024 * 1024 * 10  # 10 MB
        receive_timeout = 3

        html = ""
        try:
            response = requests.get(url=url.url,
                                    stream=True,
                                    timeout=(connection_timeout, read_timeout))

            content_length = response.headers.get("Content-Length")
            if content_length and int(
                    response.headers.get("Content-Length")) > maximum_size:
                log.error("This file is too big!")
                return

            size = 0
            start = pajbot.utils.now().timestamp()

            for chunk in response.iter_content(1024):
                if pajbot.utils.now().timestamp() - start > receive_timeout:
                    log.error("The site took too long to load")
                    return

                size += len(chunk)
                if size > maximum_size:
                    log.error("This file is too big! (fake header)")
                    return
                html += str(chunk)

        except requests.exceptions.ConnectTimeout:
            log.warning("Connection timed out while checking {0}".format(
                url.url))
            self.cache_url(url.url, True)
            return
        except requests.exceptions.ReadTimeout:
            log.warning("Reading timed out while checking {0}".format(url.url))
            self.cache_url(url.url, True)
            return
        except:
            log.exception("Unhandled exception")
            return

        try:
            soup = BeautifulSoup(html, "html.parser")
        except:
            return

        original_url = url
        original_redirected_url = redirected_url
        urls = []
        for link in soup.find_all(
                "a"):  # get a list of links to external sites
            url = link.get("href")
            if url is None:
                continue
            if url.startswith("//"):
                urls.append("http:" + url)
            elif url.startswith("http://") or url.startswith("https://"):
                urls.append(url)

        for url in urls:  # check if the site links to anything dangerous
            url = Url(url)

            if is_subdomain(url.parsed.netloc, original_url.parsed.netloc):
                # log.debug('Skipping because internal link')
                continue

            log.debug("Checking sublink {0}".format(url.url))
            res = self.basic_check(url, action, sublink=True)
            if res == self.RET_BAD_LINK:
                self.counteract_bad_url(url)
                self.counteract_bad_url(original_url, want_to_blacklist=False)
                self.counteract_bad_url(original_redirected_url,
                                        want_to_blacklist=False)
                return
            elif res == self.RET_GOOD_LINK:
                continue

            try:
                r = requests.head(url.url,
                                  allow_redirects=True,
                                  timeout=connection_timeout)
            except:
                continue

            redirected_url = Url(r.url)
            if not is_same_url(url, redirected_url):
                res = self.basic_check(redirected_url, action, sublink=True)
                if res == self.RET_BAD_LINK:
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(original_url,
                                            want_to_blacklist=False)
                    self.counteract_bad_url(original_redirected_url,
                                            want_to_blacklist=False)
                    return
                elif res == self.RET_GOOD_LINK:
                    continue

            if self.safeBrowsingAPI:
                if self.safeBrowsingAPI.check_url(
                        redirected_url.url):  # harmful url detected
                    log.debug("Evil sublink {0} by google API".format(url))
                    self.counteract_bad_url(original_url, action)
                    self.counteract_bad_url(original_redirected_url)
                    self.counteract_bad_url(url)
                    self.counteract_bad_url(redirected_url)
                    return

        # if we got here, the site is clean for our standards
        self.cache_url(original_url.url, True)
        self.cache_url(original_redirected_url.url, True)
        return

    def load_commands(self, **options):
        self.commands["add"] = Command.multiaction_command(
            level=100,
            delay_all=0,
            delay_user=0,
            default=None,
            command="add",
            commands={
                "link":
                Command.multiaction_command(
                    level=500,
                    delay_all=0,
                    delay_user=0,
                    default=None,
                    commands={
                        "blacklist":
                        Command.raw_command(
                            self.add_link_blacklist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description="Blacklist a link",
                            examples=[
                                CommandExample(
                                    None,
                                    "Add a link to the blacklist for a shallow search",
                                    chat=
                                    "user:!add link blacklist --shallow scamlink.lonk/\n"
                                    "bot>user:Successfully added your links",
                                    description=
                                    "Added the link scamlink.lonk/ to the blacklist for a shallow search",
                                ).parse(),
                                CommandExample(
                                    None,
                                    "Add a link to the blacklist for a deep search",
                                    chat=
                                    "user:!add link blacklist --deep scamlink.lonk/\n"
                                    "bot>user:Successfully added your links",
                                    description=
                                    "Added the link scamlink.lonk/ to the blacklist for a deep search",
                                ).parse(),
                            ],
                        ),
                        "whitelist":
                        Command.raw_command(
                            self.add_link_whitelist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description="Whitelist a link",
                            examples=[
                                CommandExample(
                                    None,
                                    "Add a link to the whitelist",
                                    chat=
                                    "user:!add link whitelink safelink.lonk/\n"
                                    "bot>user:Successfully added your links",
                                    description=
                                    "Added the link safelink.lonk/ to the whitelist",
                                ).parse()
                            ],
                        ),
                    },
                )
            },
        )

        self.commands["remove"] = Command.multiaction_command(
            level=100,
            delay_all=0,
            delay_user=0,
            default=None,
            command="remove",
            commands={
                "link":
                Command.multiaction_command(
                    level=500,
                    delay_all=0,
                    delay_user=0,
                    default=None,
                    commands={
                        "blacklist":
                        Command.raw_command(
                            self.remove_link_blacklist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description="Remove a link from the blacklist.",
                            examples=[
                                CommandExample(
                                    None,
                                    "Remove a link from the blacklist.",
                                    chat="user:!remove link blacklist 20\n"
                                    "bot>user:Successfully removed blacklisted link with id 20",
                                    description=
                                    "Remove a link from the blacklist with an ID",
                                ).parse()
                            ],
                        ),
                        "whitelist":
                        Command.raw_command(
                            self.remove_link_whitelist,
                            level=500,
                            delay_all=0,
                            delay_user=0,
                            description="Remove a link from the whitelist.",
                            examples=[
                                CommandExample(
                                    None,
                                    "Remove a link from the whitelist.",
                                    chat="user:!remove link whitelist 12\n"
                                    "bot>user:Successfully removed blacklisted link with id 12",
                                    description=
                                    "Remove a link from the whitelist with an ID",
                                ).parse()
                            ],
                        ),
                    },
                )
            },
        )

    def add_link_blacklist(self, **options):
        bot = options["bot"]
        message = options["message"]
        source = options["source"]

        options, new_links = self.parse_link_blacklist_arguments(message)

        if new_links:
            parts = new_links.split(" ")
            try:
                for link in parts:
                    if len(link) > 1:
                        self.blacklist_url(link, **options)
                        AdminLogManager.post("Blacklist link added", source,
                                             link)
                bot.whisper(source.username, "Successfully added your links")
                return True
            except:
                log.exception("Unhandled exception in add_link_blacklist")
                bot.whisper(source.username,
                            "Some error occurred while adding your links")
                return False
        else:
            bot.whisper(source.username, "Usage: !add link blacklist LINK")
            return False

    def add_link_whitelist(self, **options):
        bot = options["bot"]
        message = options["message"]
        source = options["source"]

        parts = message.split(" ")
        try:
            for link in parts:
                self.whitelist_url(link)
                AdminLogManager.post("Whitelist link added", source, link)
        except:
            log.exception("Unhandled exception in add_link")
            bot.whisper(source.username,
                        "Some error occurred white adding your links")
            return False

        bot.whisper(source.username, "Successfully added your links")

    def remove_link_blacklist(self, **options):
        message = options["message"]
        bot = options["bot"]
        source = options["source"]

        if message:
            id = None
            try:
                id = int(message)
            except ValueError:
                pass

            link = self.db_session.query(BlacklistedLink).filter_by(
                id=id).one_or_none()

            if link:
                self.blacklisted_links.remove(link)
                self.db_session.delete(link)
                self.db_session.commit()
            else:
                bot.whisper(source.username, "No link with the given id found")
                return False

            AdminLogManager.post("Blacklist link removed", source, link.domain)
            bot.whisper(
                source.username,
                "Successfully removed blacklisted link with id {0}".format(
                    link.id))
        else:
            bot.whisper(source.username, "Usage: !remove link blacklist ID")
            return False

    def remove_link_whitelist(self, **options):
        message = options["message"]
        bot = options["bot"]
        source = options["source"]

        if message:
            id = None
            try:
                id = int(message)
            except ValueError:
                pass

            link = self.db_session.query(WhitelistedLink).filter_by(
                id=id).one_or_none()

            if link:
                self.whitelisted_links.remove(link)
                self.db_session.delete(link)
                self.db_session.commit()
            else:
                bot.whisper(source.username, "No link with the given id found")
                return False

            AdminLogManager.post("Whitelist link removed", source, link.domain)
            bot.whisper(
                source.username,
                "Successfully removed whitelisted link with id {0}".format(
                    link.id))
        else:
            bot.whisper(source.username, "Usage: !remove link whitelist ID")
            return False

    @staticmethod
    def parse_link_blacklist_arguments(message):
        parser = argparse.ArgumentParser()
        parser.add_argument("--deep", dest="level", action="store_true")
        parser.add_argument("--shallow", dest="level", action="store_false")
        parser.set_defaults(level=False)

        try:
            args, unknown = parser.parse_known_args(message.split())
        except SystemExit:
            return False, False
        except:
            log.exception("Unhandled exception in add_link_blacklist")
            return False, False

        # Strip options of any values that are set as None
        options = {k: v for k, v in vars(args).items() if v is not None}
        response = " ".join(unknown)

        return options, response
Esempio n. 10
0
class FollowAgeModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Follow age'
    DESCRIPTION = 'Makes two commands available: !followage and !followsince'
    CATEGORY = 'Feature'

    def __init__(self):
        super().__init__()
        self.action_queue = ActionQueue()
        self.action_queue.start()

    def load_commands(self, **options):
        # TODO: Have delay modifiable in settings

        self.commands['followage'] = pajbot.models.command.Command.raw_command(
            self.follow_age,
            delay_all=4,
            delay_user=8,
            description='Check your or someone elses follow age for a channel',
            examples=[
                pajbot.models.command.CommandExample(
                    None,
                    'Check your own follow age',
                    chat='user:!followage\n'
                    'bot:pajlada, you have been following Karl_Kons for 4 months and 24 days',
                    description=
                    'Check how long you have been following the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someone elses follow age',
                    chat='user:!followage NightNacht\n'
                    'bot:pajlada, NightNacht has been following Karl_Kons for 5 months and 4 days',
                    description=
                    'Check how long any user has been following the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someones follow age for a certain streamer',
                    chat='user:!followage NightNacht forsenlol\n'
                    'bot:pajlada, NightNacht has been following forsenlol for 1 year and 4 months',
                    description=
                    'Check how long NightNacht has been following forsenlol').
                parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check your own follow age for a certain streamer',
                    chat='user:!followage pajlada forsenlol\n'
                    'bot:pajlada, you have been following forsenlol for 1 year and 3 months',
                    description=
                    'Check how long you have been following forsenlol').parse(
                    ),
            ],
        )

        self.commands['followsince'] = pajbot.models.command.Command.raw_command(
            self.follow_since,
            delay_all=4,
            delay_user=8,
            description=
            'Check from when you or someone else first followed a channel',
            examples=[
                pajbot.models.command.CommandExample(
                    None,
                    'Check your own follow since',
                    chat='user:!followsince\n'
                    'bot:pajlada, you have been following Karl_Kons since 04 March 2015, 07:02:01 UTC',
                    description=
                    'Check when you first followed the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someone elses follow since',
                    chat='user:!followsince NightNacht\n'
                    'bot:pajlada, NightNacht has been following Karl_Kons since 03 July 2014, 04:12:42 UTC',
                    description=
                    'Check when NightNacht first followed the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someone elses follow since for another streamer',
                    chat='user:!followsince NightNacht forsenlol\n'
                    'bot:pajlada, NightNacht has been following forsenlol since 13 June 2013, 13:10:51 UTC',
                    description=
                    'Check when NightNacht first followed the given streamer (forsenlol)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check your follow since for another streamer',
                    chat='user:!followsince pajlada forsenlol\n'
                    'bot:pajlada, you have been following forsenlol since 16 December 1990, 03:06:51 UTC',
                    description=
                    'Check when you first followed the given streamer (forsenlol)'
                ).parse(),
            ],
        )

    def check_follow_age(self, bot, source, username, streamer):
        streamer = bot.streamer if streamer is None else streamer.lower()
        age = bot.twitchapi.get_follow_relationship(username, streamer)
        if source.username == username:
            if age is False:
                bot.say('{}, you are not following {}'.format(
                    source.username_raw, streamer))
            else:
                bot.say('{}, you have been following {} for {}'.format(
                    source.username_raw, streamer,
                    time_since(
                        datetime.datetime.now().timestamp() - age.timestamp(),
                        0)))
        else:
            if age is False:
                bot.say('{}, {} is not following {}'.format(
                    source.username_raw, username, streamer))
            else:
                bot.say('{}, {} has been following {} for {}'.format(
                    source.username_raw, username, streamer,
                    time_since(
                        datetime.datetime.now().timestamp() - age.timestamp(),
                        0)))

    def check_follow_since(self, bot, source, username, streamer):
        streamer = bot.streamer if streamer is None else streamer.lower()
        follow_since = bot.twitchapi.get_follow_relationship(
            username, streamer)
        if source.username == username:
            if follow_since is False:
                bot.say('{}, you are not following {}'.format(
                    source.username_raw, streamer))
            else:
                bot.say('{}, you have been following {} since {} UTC'.format(
                    source.username_raw, streamer,
                    follow_since.strftime('%d %B %Y, %X')))
        else:
            if follow_since is False:
                bot.say('{}, {} is not following {}'.format(
                    source.username_raw, username, streamer))
            else:
                bot.say('{}, {} has been following {} since {} UTC'.format(
                    source.username_raw, username, streamer,
                    follow_since.strftime('%d %B %Y, %X')))

    def follow_age(self, **options):
        source = options['source']
        message = options['message']
        bot = options['bot']

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_age,
                              args=[bot, source, username, streamer])

    def follow_since(self, **options):
        bot = options['bot']
        source = options['source']
        message = options['message']

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_since,
                              args=[bot, source, username, streamer])

    def parse_message(self, bot, source, message):
        username = source.username
        streamer = None
        if message is not None and len(message) > 0:
            message_split = message.split(' ')
            if len(message_split[0]) and message_split[0].replace(
                    '_', '').isalnum():
                username = message_split[0].lower()
            if len(message_split) > 1 and message_split[1].replace(
                    '_', '').isalnum():
                streamer = message_split[1]

        return username, streamer
Esempio n. 11
0
class Bot:
    """
    Main class for the twitch bot
    """

    version = "1.35"
    date_fmt = "%H:%M"
    admin = None
    url_regex_str = r"\(?(?:(http|https):\/\/)?(?:((?:[^\W\s]|\.|-|[:]{1})+)@{1})?((?:www.)?(?:[^\W\s]|\.|-)+[\.][^\W\s]{2,4}|localhost(?=\/)|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?::(\d*))?([\/]?[^\s\?]*[\/]{1})*(?:\/?([^\s\n\?\[\]\{\}\#]*(?:(?=\.)){1}|[^\s\n\?\[\]\{\}\.\#]*)?([\.]{1}[^\s\?\#]*)?)?(?:\?{1}([^\s\n\#\[\]]*))?([\#][^\s\n]*)?\)?"

    last_ping = pajbot.utils.now()
    last_pong = pajbot.utils.now()

    @staticmethod
    def parse_args():
        parser = argparse.ArgumentParser()
        parser.add_argument(
            "--config",
            "-c",
            default="config.ini",
            help="Specify which config file to use (default: config.ini)")
        parser.add_argument(
            "--silent",
            action="count",
            help="Decides whether the bot should be silent or not")
        # TODO: Add a log level argument.

        return parser.parse_args()

    bot_token = None

    @property
    def password(self):
        if "password" in self.config["main"]:
            log.warning(
                "DEPRECATED - Using bot password/oauth token from file. "
                "You should authenticate in web gui using route /bot_login "
                "and remove password from config file")
            return self.config["main"]["password"]

        if self.bot_token is None:
            self.bot_token = BotToken(self.config)

        t = self.bot_token.access_token()
        return "oauth:{}".format(t)

    def load_config(self, config):
        self.config = config

        DBManager.init(self.config["main"]["db"])

        redis_options = {}
        if "redis" in config:
            redis_options = dict(config.items("redis"))

        RedisManager.init(**redis_options)

        pajbot.models.user.Config.se_sync_token = config["main"].get(
            "se_sync_token", None)
        pajbot.models.user.Config.se_channel = config["main"].get(
            "se_channel", None)

        self.domain = config["web"].get("domain", "localhost")

        self.nickname = config["main"].get("nickname", "pajbot")

        self.timezone = config["main"].get("timezone", "UTC")

        if config["main"].getboolean("verified", False):
            TMI.promote_to_verified()

        self.trusted_mods = config.getboolean("main", "trusted_mods")

        self.phrases = {
            "welcome": ["{nickname} {version} running!"],
            "quit": ["{nickname} {version} shutting down..."]
        }

        if "phrases" in config:
            phrases = config["phrases"]
            if "welcome" in phrases:
                self.phrases["welcome"] = phrases["welcome"].splitlines()
            if "quit" in phrases:
                self.phrases["quit"] = phrases["quit"].splitlines()

        TimeManager.init_timezone(self.timezone)

        if "streamer" in config["main"]:
            self.streamer = config["main"]["streamer"]
            self.channel = "#" + self.streamer
        elif "target" in config["main"]:
            self.channel = config["main"]["target"]
            self.streamer = self.channel[1:]

        StreamHelper.init_streamer(self.streamer)

        self.silent = False
        self.dev = False

        if "flags" in config:
            self.silent = True if "silent" in config["flags"] and config[
                "flags"]["silent"] == "1" else self.silent
            self.dev = True if "dev" in config["flags"] and config["flags"][
                "dev"] == "1" else self.dev

    def __init__(self, config, args=None):
        # Load various configuration variables from the given config object
        # The config object that should be passed through should
        # come from pajbot.utils.load_config
        self.load_config(config)
        log.debug("Loaded config")

        # streamer is additionally initialized here so streamer can be accessed by the DB migrations
        # before StreamHelper.init_bot() is called later (which depends on an upgraded DB because
        # StreamManager accesses the DB)
        StreamHelper.init_streamer(self.streamer)

        # Update the database (and partially redis) scheme if necessary using alembic
        # In case of errors, i.e. if the database is out of sync or the alembic
        # binary can't be called, we will shut down the bot.
        pajbot.utils.alembic_upgrade()
        log.debug("ran db upgrade")

        # Actions in this queue are run in a separate thread.
        # This means actions should NOT access any database-related stuff.
        self.action_queue = ActionQueue()
        self.action_queue.start()

        self.reactor = irc.client.Reactor(self.on_connect)
        self.start_time = pajbot.utils.now()
        ActionParser.bot = self

        HandlerManager.init_handlers()

        self.socket_manager = SocketManager(self.streamer)
        self.stream_manager = StreamManager(self)

        StreamHelper.init_bot(self, self.stream_manager)
        ScheduleManager.init()

        self.users = UserManager()
        self.decks = DeckManager()
        self.banphrase_manager = BanphraseManager(self).load()
        self.timer_manager = TimerManager(self).load()
        self.kvi = KVIManager()

        twitch_client_id = None
        twitch_oauth = None
        if "twitchapi" in self.config:
            twitch_client_id = self.config["twitchapi"].get("client_id", None)
            twitch_oauth = self.config["twitchapi"].get("oauth", None)

        # A client ID is required for the bot to work properly now, give an error for now
        if twitch_client_id is None:
            log.error(
                'MISSING CLIENT ID, SET "client_id" VALUE UNDER [twitchapi] SECTION IN CONFIG FILE'
            )

        self.twitchapi = TwitchAPI(twitch_client_id, twitch_oauth)
        self.emote_manager = EmoteManager(twitch_client_id)
        self.epm_manager = EpmManager()
        self.ecount_manager = EcountManager()
        self.twitter_manager = TwitterManager(self)

        self.module_manager = ModuleManager(self.socket_manager,
                                            bot=self).load()
        self.commands = CommandManager(socket_manager=self.socket_manager,
                                       module_manager=self.module_manager,
                                       bot=self).load()

        HandlerManager.trigger("on_managers_loaded")

        # Reloadable managers
        self.reloadable = {}

        # Commitable managers
        self.commitable = {
            "commands": self.commands,
            "banphrases": self.banphrase_manager
        }

        self.execute_every(10 * 60, self.commit_all)
        self.execute_every(1, self.do_tick)

        try:
            self.admin = self.config["main"]["admin"]
        except KeyError:
            log.warning(
                "No admin user specified. See the [main] section in config.example.ini for its usage."
            )
        if self.admin:
            with self.users.get_user_context(self.admin) as user:
                user.level = 2000

        self.parse_version()

        relay_host = self.config["main"].get("relay_host", None)
        relay_password = self.config["main"].get("relay_password", None)
        if relay_host is None or relay_password is None:
            self.irc = MultiIRCManager(self)
        else:
            self.irc = SingleIRCManager(self)

        self.reactor.add_global_handler("all_events", self.irc._dispatcher,
                                        -10)

        self.data = {}
        self.data_cb = {}
        self.url_regex = re.compile(self.url_regex_str, re.IGNORECASE)

        self.data["broadcaster"] = self.streamer
        self.data["version"] = self.version
        self.data["version_brief"] = self.version_brief
        self.data["bot_name"] = self.nickname
        self.data_cb["status_length"] = self.c_status_length
        self.data_cb["stream_status"] = self.c_stream_status
        self.data_cb["bot_uptime"] = self.c_uptime
        self.data_cb["current_time"] = self.c_current_time

        self.silent = True if args.silent else self.silent

        if self.silent:
            log.info("Silent mode enabled")
        """
        For actions that need to access the main thread,
        we can use the mainthread_queue.
        """
        self.mainthread_queue = ActionQueue()
        self.execute_every(1, self.mainthread_queue.parse_action)

        self.websocket_manager = WebSocketManager(self)

        try:
            if self.config["twitchapi"]["update_subscribers"] == "1":
                self.execute_every(30 * 60, self.action_queue.add,
                                   (self.update_subscribers_stage1, ))
        except:
            pass

    def on_connect(self, sock):
        return self.irc.on_connect(sock)

    def update_subscribers_stage1(self):
        limit = 25
        offset = 0
        subscribers = []
        log.info("Starting stage1 subscribers update")

        try:
            retry_same = 0
            while True:
                log.debug("Beginning sub request limit=%s offset=%s", limit,
                          offset)
                subs, retry_same, error = self.twitchapi.get_subscribers(
                    self.streamer, 0, offset,
                    0 if retry_same is False else retry_same)
                log.debug("got em!")

                if error is True:
                    log.error("Too many attempts, aborting")
                    return False

                if retry_same is False:
                    offset += limit
                    if not subs:
                        # We don't need to retry, and the last query finished propery
                        # Break out of the loop and start fiddling with the subs!
                        log.debug("Done!")
                        break

                    log.debug("Fetched %d subs", len(subs))
                    subscribers.extend(subs)

                if retry_same is not False:
                    # In case the next attempt is a retry, wait for 3 seconds
                    log.debug("waiting for 3 seconds...")
                    time.sleep(3)
                    log.debug("waited enough!")
            log.debug("Finished with the while True loop!")
        except:
            log.exception(
                "Caught an exception while trying to get subscribers")
            return None

        log.info("Ended stage1 subscribers update")
        if subscribers:
            log.info(
                "Got some subscribers, so we are pushing them to stage 2!")
            self.mainthread_queue.add(self.update_subscribers_stage2,
                                      args=[subscribers])
            log.info("Pushed them now.")

    def update_subscribers_stage2(self, subscribers):
        self.kvi["active_subs"].set(len(subscribers) - 1)

        self.users.reset_subs()

        self.users.update_subs(subscribers)

    def start(self):
        """Start the IRC client."""
        self.reactor.process_forever()

    def get_kvi_value(self, key, extra={}):
        return self.kvi[key].get()

    def get_last_tweet(self, key, extra={}):
        return self.twitter_manager.get_last_tweet(key)

    def get_emote_epm(self, key, extra={}):
        val = self.epm_manager.get_emote_epm(key)
        if val is None:
            return None
        # formats the number with grouping (e.g. 112,556) and zero decimal places
        return "{0:,.0f}".format(val)

    def get_emote_epm_record(self, key, extra={}):
        val = self.epm_manager.get_emote_epm_record(key)
        if val is None:
            return None
        # formats the number with grouping (e.g. 112,556) and zero decimal places
        return "{0:,.0f}".format(val)

    def get_emote_count(self, key, extra={}):
        val = self.ecount_manager.get_emote_count(key)
        if val is None:
            return None
        # formats the number with grouping (e.g. 112,556) and zero decimal places
        return "{0:,.0f}".format(val)

    @staticmethod
    def get_source_value(key, extra={}):
        try:
            return getattr(extra["source"], key)
        except:
            log.exception("Caught exception in get_source_value")

        return None

    def get_user_value(self, key, extra={}):
        try:
            user = self.users.find(extra["argument"])
            if user:
                return getattr(user, key)
        except:
            log.exception("Caught exception in get_source_value")

        return None

    @staticmethod
    def get_command_value(key, extra={}):
        try:
            return getattr(extra["command"].data, key)
        except:
            log.exception("Caught exception in get_source_value")

        return None

    def get_usersource_value(self, key, extra={}):
        try:
            user = self.users.find(extra["argument"])
            if user:
                return getattr(user, key)

            return getattr(extra["source"], key)
        except:
            log.exception("Caught exception in get_source_value")

        return None

    def get_time_value(self, key, extra={}):
        try:
            tz = timezone(key)
            return datetime.datetime.now(tz).strftime(self.date_fmt)
        except:
            log.exception("Unhandled exception in get_time_value")

        return None

    def get_current_song_value(self, key, extra={}):
        if self.stream_manager.online:
            current_song = PleblistManager.get_current_song(
                self.stream_manager.current_stream.id)
            inner_keys = key.split(".")
            val = current_song
            for inner_key in inner_keys:
                val = getattr(val, inner_key, None)
                if val is None:
                    return None
            if val is not None:
                return val
        return None

    def get_strictargs_value(self, key, extra={}):
        ret = self.get_args_value(key, extra)

        if not ret:
            return None

        return ret

    @staticmethod
    def get_args_value(key, extra={}):
        r = None
        try:
            msg_parts = extra["message"].split(" ")
        except (KeyError, AttributeError):
            msg_parts = [""]

        try:
            if "-" in key:
                range_str = key.split("-")
                if len(range_str) == 2:
                    r = (int(range_str[0]), int(range_str[1]))

            if r is None:
                r = (int(key), len(msg_parts))
        except (TypeError, ValueError):
            r = (0, len(msg_parts))

        try:
            return " ".join(msg_parts[r[0]:r[1]])
        except AttributeError:
            return ""
        except:
            log.exception("UNHANDLED ERROR IN get_args_value")
            return ""

    def get_notify_value(self, key, extra={}):
        payload = {
            "message": extra["message"] or "",
            "trigger": extra["trigger"],
            "user": extra["source"].username_raw
        }
        self.websocket_manager.emit("notify", payload)

        return ""

    def get_value(self, key, extra={}):
        if key in extra:
            return extra[key]

        if key in self.data:
            return self.data[key]

        if key in self.data_cb:
            return self.data_cb[key]()

        log.warning("Unknown key passed to get_value: %s", key)
        return None

    def privmsg_arr(self, arr, target=None):
        for msg in arr:
            self.privmsg(msg, target)

    def privmsg_from_file(self,
                          url,
                          per_chunk=35,
                          chunk_delay=30,
                          target=None):
        try:
            r = requests.get(url)
            r.raise_for_status()

            content_type = r.headers["Content-Type"]
            if content_type is not None and cgi.parse_header(
                    content_type)[0] != "text/plain":
                log.error(
                    "privmsg_from_file should be fed with a text/plain URL. Refusing to send."
                )
                return

            lines = r.text.splitlines()
            i = 0
            while lines:
                if i == 0:
                    self.privmsg_arr(lines[:per_chunk], target)
                else:
                    self.execute_delayed(chunk_delay * i, self.privmsg_arr,
                                         (lines[:per_chunk], target))

                del lines[:per_chunk]

                i = i + 1
        except:
            log.exception("error in privmsg_from_file")

    # event is an event to clone and change the text from.
    # Usage: !eval bot.eval_from_file(event, 'https://pastebin.com/raw/LhCt8FLh')
    def eval_from_file(self, event, url):
        try:
            r = requests.get(url)
            r.raise_for_status()

            content_type = r.headers["Content-Type"]
            if content_type is not None and cgi.parse_header(
                    content_type)[0] != "text/plain":
                log.error(
                    "privmsg_from_file should be fed with a text/plain URL. Refusing to send."
                )
                return

            lines = r.text.splitlines()
            import copy

            for msg in lines:
                cloned_event = copy.deepcopy(event)
                cloned_event.arguments = [msg]
                # omit the source connection as None (since its not used)
                self.on_pubmsg(None, cloned_event)
            self.whisper(event.source.user.lower(),
                         "Successfully evaluated {0} lines".format(len(lines)))
        except:
            log.exception("BabyRage")
            self.whisper(event.source.user.lower(), "Exception BabyRage")

    def privmsg(self, message, channel=None, increase_message=True):
        if channel is None:
            channel = self.channel

        return self.irc.privmsg(message,
                                channel,
                                increase_message=increase_message)

    def c_uptime(self):
        return time_ago(self.start_time)

    @staticmethod
    def c_current_time():
        return pajbot.utils.now()

    @property
    def is_online(self):
        return self.stream_manager.online

    def c_stream_status(self):
        return "online" if self.stream_manager.online else "offline"

    def c_status_length(self):
        if self.stream_manager.online:
            return time_ago(self.stream_manager.current_stream.stream_start)

        if self.stream_manager.last_stream is not None:
            return time_ago(self.stream_manager.last_stream.stream_end)

        return "No recorded stream FeelsBadMan "

    def execute_at(self, at, function, arguments=()):
        self.reactor.scheduler.execute_at(at, lambda: function(*arguments))

    def execute_delayed(self, delay, function, arguments=()):
        self.reactor.scheduler.execute_after(delay,
                                             lambda: function(*arguments))

    def execute_every(self, period, function, arguments=()):
        self.reactor.scheduler.execute_every(period,
                                             lambda: function(*arguments))

    def _ban(self, username, reason=""):
        self.privmsg(".ban {0} {1}".format(username, reason),
                     increase_message=False)

    def ban(self, username, reason=""):
        log.debug("Banning %s", username)
        self._timeout(username, 30, reason)
        self.execute_delayed(1, self._ban, (username, reason))

    def ban_user(self, user, reason=""):
        self._timeout(user.username, 30, reason)
        self.execute_delayed(1, self._ban, (user.username, reason))

    def unban(self, username):
        self.privmsg(".unban {0}".format(username), increase_message=False)

    def _timeout(self, username, duration, reason=""):
        self.privmsg(".timeout {0} {1} {2}".format(username, duration, reason),
                     increase_message=False)

    def timeout(self, username, duration, reason=""):
        # log.debug("Timing out %s for %d seconds", username, duration)
        self._timeout(username, duration, reason)

    def timeout_warn(self, user, duration, reason=""):
        duration, punishment = user.timeout(
            duration, warning_module=self.module_manager["warning"])
        self.timeout(user.username, duration, reason)
        return (duration, punishment)

    def timeout_user(self, user, duration, reason=""):
        self._timeout(user.username, duration, reason)

    def _timeout_user(self, user, duration, reason=""):
        self._timeout(user.username, duration, reason)

    def whisper(self, username, *messages, separator=". ", **rest):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string as a whisper to username
        """

        if len(messages) < 0:
            return False

        message = separator.join(messages)

        return self.irc.whisper(username, message)

    def send_message_to_user(self,
                             user,
                             message,
                             event,
                             separator=". ",
                             method="say"):
        if method == "say":
            self.say(user.username + ", " + lowercase_first_letter(message),
                     separator=separator)
        elif method == "whisper":
            self.whisper(user.username, message, separator=separator)
        elif method == "me":
            self.me(message)
        elif method == "reply":
            if event.type in ["action", "pubmsg"]:
                self.say(message, separator=separator)
            elif event.type == "whisper":
                self.whisper(user.username, message, separator=separator)
        else:
            log.warning("Unknown send_message method: %s", method)

    def safe_privmsg(self, message, channel=None, increase_message=True):
        # Check for banphrases
        res = self.banphrase_manager.check_message(message, None)
        if res is not False:
            self.privmsg("filtered message ({})".format(res.id), channel,
                         increase_message)
            return

        self.privmsg(message, channel, increase_message)

    def say(self, *messages, channel=None, separator=". "):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string to the given channel.
        """

        if len(messages) < 0:
            return False

        if not self.silent:
            message = separator.join(messages).strip()

            message = clean_up_message(message)
            if not message:
                return False

            # log.info("Sending message: %s", message)

            self.privmsg(message[:510], channel)

    def is_bad_message(self, message):
        return self.banphrase_manager.check_message(message, None) is not False

    def safe_me(self, message, channel=None):
        if not self.is_bad_message(message):
            self.me(message, channel)

    def me(self, message, channel=None):
        self.say(".me " + message[:500], channel=channel)

    def parse_version(self):
        self.version = self.version
        self.version_brief = self.version

        if self.dev:
            try:
                current_branch = (subprocess.check_output(
                    ["git", "rev-parse", "--abbrev-ref",
                     "HEAD"]).decode("utf8").strip())
                latest_commit = subprocess.check_output(
                    ["git", "rev-parse", "HEAD"]).decode("utf8").strip()[:8]
                commit_number = subprocess.check_output(
                    ["git", "rev-list", "HEAD",
                     "--count"]).decode("utf8").strip()
                self.version = "{0} DEV ({1}, {2}, commit {3})".format(
                    self.version, current_branch, latest_commit, commit_number)
            except:
                log.exception("hmm")

    def on_welcome(self, chatconn, event):
        return self.irc.on_welcome(chatconn, event)

    def connect(self):
        return self.irc.start()

    def on_disconnect(self, chatconn, event):
        self.irc.on_disconnect(chatconn, event)

    def parse_message(self, message, source, event, tags={}, whisper=False):
        msg_lower = message.lower()

        emote_tag = None

        for tag in tags:
            if tag["key"] == "subscriber" and event.target == self.channel:
                source.subscriber = tag["value"] == "1"
            elif tag["key"] == "emotes":
                emote_tag = tag["value"]
            elif tag["key"] == "display-name" and tag["value"]:
                source.username_raw = tag["value"]
            elif tag["key"] == "user-type":
                source.moderator = tag[
                    "value"] == "mod" or source.username == self.streamer

        # source.num_lines += 1

        if source is None:
            log.error("No valid user passed to parse_message")
            return False

        if source.banned:
            self.ban(source.username)
            return False

        # If a user types when timed out, we assume he's been unbanned for a good reason and remove his flag.
        if source.timed_out is True:
            source.timed_out = False

        # Parse emotes in the message
        emote_instances, emote_counts = self.emote_manager.parse_all_emotes(
            message, emote_tag)

        self.epm_manager.handle_emotes(emote_counts)
        self.ecount_manager.handle_emotes(emote_counts)

        urls = self.find_unique_urls(message)

        # log.debug("{2}{0}: {1}".format(source.username, message, "<w>" if whisper else ""))

        res = HandlerManager.trigger(
            "on_message",
            source=source,
            message=message,
            emote_instances=emote_instances,
            emote_counts=emote_counts,
            whisper=whisper,
            urls=urls,
            event=event,
        )
        if res is False:
            return False

        source.last_seen = pajbot.utils.now()
        source.last_active = pajbot.utils.now()

        if source.ignored:
            return False

        if whisper:
            self.whisper('datguy1',
                         '{} said: {}'.format(source.username_raw, message))

        if msg_lower[:1] == "!":
            msg_lower_parts = msg_lower.split(" ")
            trigger = msg_lower_parts[0][1:]
            msg_raw_parts = message.split(" ")
            remaining_message = " ".join(
                msg_raw_parts[1:]) if len(msg_raw_parts) > 1 else None
            if trigger in self.commands:
                command = self.commands[trigger]
                extra_args = {
                    "emote_instances": emote_instances,
                    "emote_counts": emote_counts,
                    "trigger": trigger
                }
                command.run(self,
                            source,
                            remaining_message,
                            event=event,
                            args=extra_args,
                            whisper=whisper)

    def on_whisper(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        username = event.source.user.lower()

        with self.users.get_user_context(username) as source:
            self.parse_message(event.arguments[0],
                               source,
                               event,
                               whisper=True,
                               tags=event.tags)

    def on_ping(self, chatconn, event):
        # self.say('Received a ping. Last ping received {} ago'.format(time_since(pajbot.utils.now().timestamp(), self.last_ping.timestamp())))
        log.info("Received a ping. Last ping received %s ago",
                 time_ago(self.last_ping))
        self.last_ping = pajbot.utils.now()

    def on_pong(self, chatconn, event):
        # self.say('Received a pong. Last pong received {} ago'.format(time_since(pajbot.utils.now().timestamp(), self.last_pong.timestamp())))
        log.info("Received a pong. Last pong received %s ago",
                 time_ago(self.last_pong))
        self.last_pong = pajbot.utils.now()

    def on_pubnotice(self, chatconn, event):
        return
        type = "whisper" if chatconn in self.whisper_manager else "normal"
        log.debug("NOTICE {}@{}: {}".format(type, event.target,
                                            event.arguments))

    def on_usernotice(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        tags = {}
        for d in event.tags:
            tags[d["key"]] = d["value"]

        if "login" not in tags:
            return

        username = tags["login"]

        with self.users.get_user_context(username) as source:
            msg = ""
            if event.arguments:
                msg = event.arguments[0]
            HandlerManager.trigger("on_usernotice",
                                   source=source,
                                   message=msg,
                                   tags=tags)

    def on_action(self, chatconn, event):
        self.on_pubmsg(chatconn, event)

    def on_pubmsg(self, chatconn, event):
        if event.source.user == self.nickname:
            return False

        username = event.source.user.lower()

        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        with self.users.get_user_context(username) as source:
            res = HandlerManager.trigger("on_pubmsg",
                                         source=source,
                                         message=event.arguments[0])
            if res is False:
                return False

            self.parse_message(event.arguments[0],
                               source,
                               event,
                               tags=event.tags)

    @time_method
    def reload_all(self):
        log.info("Reloading all...")
        for key, manager in self.reloadable.items():
            log.debug("Reloading %s", key)
            manager.reload()
            log.debug("Done with %s", key)
        log.info("ok!")

    @time_method
    def commit_all(self):
        # log.info("Commiting all...")
        for key, manager in self.commitable.items():
            # log.info("Commiting %s", key)
            manager.commit()
            # log.info("Done with %s", key)
        # log.info("ok!")

        HandlerManager.trigger("on_commit", stop_on_false=False)

    @staticmethod
    def do_tick():
        HandlerManager.trigger("on_tick")

    def quit(self, message, event, **options):
        quit_chub = self.config["main"].get("control_hub", None)
        quit_delay = 0

        if quit_chub is not None and event.target == ("#{}".format(quit_chub)):
            quit_delay_random = 300
            try:
                if message is not None and int(message.split()[0]) >= 1:
                    quit_delay_random = int(message.split()[0])
            except (IndexError, ValueError, TypeError):
                pass
            quit_delay = random.randint(0, quit_delay_random)
            log.info("%s is restarting in %d seconds.", self.nickname,
                     quit_delay)

        self.execute_delayed(quit_delay, self.quit_bot)

    def quit_bot(self, **options):
        HandlerManager.trigger("on_quit")
        self.commit_all()
        phrase_data = {"nickname": self.nickname, "version": self.version}

        try:
            ScheduleManager.base_scheduler.print_jobs()
            ScheduleManager.base_scheduler.shutdown(wait=False)
        except:
            log.exception("Error while shutting down the apscheduler")

        try:
            for p in self.phrases["quit"]:
                if not self.silent:
                    self.privmsg(p.format(**phrase_data))
        except Exception:
            log.exception("Exception caught while trying to say quit phrase")

        self.twitter_manager.quit()
        self.socket_manager.quit()
        self.irc.quit()

        sys.exit(0)

    @staticmethod
    def apply_filter(resp, f):
        available_filters = {
            "strftime":
            _filter_strftime,
            "lower":
            lambda var, args: var.lower(),
            "upper":
            lambda var, args: var.upper(),
            "time_since_minutes":
            lambda var, args: "no time"
            if var == 0 else time_since(var * 60, 0, time_format="long"),
            "time_since":
            lambda var, args: "no time"
            if var == 0 else time_since(var, 0, time_format="long"),
            "time_since_dt":
            _filter_time_since_dt,
            "urlencode":
            _filter_urlencode,
            "join":
            _filter_join,
            "number_format":
            _filter_number_format,
            "add":
            _filter_add,
        }
        if f.name in available_filters:
            return available_filters[f.name](resp, f.arguments)
        return resp

    def find_unique_urls(self, message):
        from pajbot.modules.linkchecker import find_unique_urls

        return find_unique_urls(self.url_regex, message)
Esempio n. 12
0
class Bot:
    """
    Main class for the twitch bot
    """

    version = '2.6.3'
    date_fmt = '%H:%M'
    update_chatters_interval = 5
    admin = None
    url_regex_str = r'\(?(?:(http|https):\/\/)?(?:((?:[^\W\s]|\.|-|[:]{1})+)@{1})?((?:www.)?(?:[^\W\s]|\.|-)+[\.][^\W\s]{2,4}|localhost(?=\/)|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?::(\d*))?([\/]?[^\s\?]*[\/]{1})*(?:\/?([^\s\n\?\[\]\{\}\#]*(?:(?=\.)){1}|[^\s\n\?\[\]\{\}\.\#]*)?([\.]{1}[^\s\?\#]*)?)?(?:\?{1}([^\s\n\#\[\]]*))?([\#][^\s\n]*)?\)?'

    def parse_args():
        parser = argparse.ArgumentParser()
        parser.add_argument('--config', '-c',
                            default='config.ini',
                            help='Specify which config file to use '
                                    '(default: config.ini)')
        parser.add_argument('--silent',
                            action='count',
                            help='Decides whether the bot should be '
                            'silent or not')
        # TODO: Add a log level argument.

        return parser.parse_args()

    def load_default_phrases(self):
        default_phrases = {
                'welcome': False,
                'quit': False,
                'nl': '{username} has typed {num_lines} messages in this channel!',
                'nl_0': '{username} has not typed any messages in this channel BibleThump',
                'nl_pos': '{username} is rank {nl_pos} line-farmer in this channel!',
                'new_sub': 'Sub hype! {username} just subscribed PogChamp',
                'resub': 'Resub hype! {username} just subscribed, {num_months} months in a row PogChamp <3 PogChamp',
                'point_pos': '{username_w_verb} rank {point_pos} point-hoarder in this channel with {points} points.',
                }
        if 'phrases' in self.config:
            self.phrases = {}

            for phrase_key, phrase_value in self.config['phrases'].items():
                if len(phrase_value.strip()) <= 0:
                    self.phrases[phrase_key] = False
                else:
                    self.phrases[phrase_key] = phrase_value

            for phrase_key, phrase_value in default_phrases.items():
                if phrase_key not in self.phrases:
                    self.phrases[phrase_key] = phrase_value
        else:
            self.phrases = default_phrases

    def load_config(self, config):
        self.config = config

        self.nickname = config['main'].get('nickname', 'pajbot')
        self.password = config['main'].get('password', 'abcdef')

        self.timezone = config['main'].get('timezone', 'UTC')

        self.trusted_mods = config.getboolean('main', 'trusted_mods')

        TimeManager.init_timezone(self.timezone)

        if 'streamer' in config['main']:
            self.streamer = config['main']['streamer']
            self.channel = '#' + self.streamer
        elif 'target' in config['main']:
            self.channel = config['main']['target']
            self.streamer = self.channel[1:]

        self.wolfram = None
        try:
            if 'wolfram' in config['main']:
                import wolframalpha
                self.wolfram = wolframalpha.Client(config['main']['wolfram'])
        except:
            pass

        self.silent = False
        self.dev = False

        if 'flags' in config:
            self.silent = True if 'silent' in config['flags'] and config['flags']['silent'] == '1' else self.silent
            self.dev = True if 'dev' in config['flags'] and config['flags']['dev'] == '1' else self.dev

        DBManager.init(self.config['main']['db'])

        redis_options = {}
        if 'redis' in config:
            log.info(config._sections['redis'])
            redis_options = config._sections['redis']

        RedisManager.init(**redis_options)

    def __init__(self, config, args=None):
        self.load_config(config)
        self.last_ping = datetime.datetime.now()
        self.last_pong = datetime.datetime.now()

        self.load_default_phrases()

        self.db_session = DBManager.create_session()

        try:
            subprocess.check_call(['alembic', 'upgrade', 'head'] + ['--tag="{0}"'.format(' '.join(sys.argv[1:]))])
        except subprocess.CalledProcessError:
            log.exception('aaaa')
            log.error('Unable to call `alembic upgrade head`, this means the database could be out of date. Quitting.')
            sys.exit(1)
        except PermissionError:
            log.error('No permission to run `alembic upgrade head`. This means your user probably doesn\'t have execution rights on the `alembic` binary.')
            log.error('The error can also occur if it can\'t find `alembic` in your PATH, and instead tries to execute the alembic folder.')
            sys.exit(1)
        except FileNotFoundError:
            log.error('Could not found an installation of alembic. Please install alembic to continue.')
            sys.exit(1)
        except:
            log.exception('Unhandled exception when calling db update')
            sys.exit(1)

        # Actions in this queue are run in a separate thread.
        # This means actions should NOT access any database-related stuff.
        self.action_queue = ActionQueue()
        self.action_queue.start()

        self.reactor = irc.client.Reactor()
        self.start_time = datetime.datetime.now()
        ActionParser.bot = self

        HandlerManager.init_handlers()

        self.socket_manager = SocketManager(self)
        self.stream_manager = StreamManager(self)

        StreamHelper.init_bot(self, self.stream_manager)

        self.users = UserManager()
        self.decks = DeckManager().reload()
        self.module_manager = ModuleManager(self.socket_manager, bot=self).load()
        self.commands = CommandManager(
                socket_manager=self.socket_manager,
                module_manager=self.module_manager,
                bot=self).load()
        self.filters = FilterManager().reload()
        self.banphrase_manager = BanphraseManager(self).load()
        self.timer_manager = TimerManager(self).load()
        self.kvi = KVIManager().reload()
        self.emotes = EmoteManager(self).reload()
        self.twitter_manager = TwitterManager(self).reload()
        self.duel_manager = DuelManager(self)

        HandlerManager.trigger('on_managers_loaded')

        # Reloadable managers
        self.reloadable = {
                'filters': self.filters,
                'kvi': self.kvi,
                'emotes': self.emotes,
                'twitter': self.twitter_manager,
                'decks': self.decks,
                }

        # Commitable managers
        self.commitable = {
                'commands': self.commands,
                'filters': self.filters,
                'kvi': self.kvi,
                'emotes': self.emotes,
                'twitter': self.twitter_manager,
                'decks': self.decks,
                'users': self.users,
                'banphrases': self.banphrase_manager,
                }

        self.execute_every(10 * 60, self.commit_all)
        self.execute_every(30, lambda: self.connection_manager.get_main_conn().ping('tmi.twitch.tv'))

        try:
            self.admin = self.config['main']['admin']
        except KeyError:
            log.warning('No admin user specified. See the [main] section in config.example.ini for its usage.')
        if self.admin:
            self.users[self.admin].level = 2000

        self.parse_version()

        self.connection_manager = ConnectionManager(self.reactor, self, TMI.message_limit, streamer=self.streamer)
        chub = self.config['main'].get('control_hub', None)
        if chub is not None:
            self.control_hub = ConnectionManager(self.reactor, self, TMI.message_limit, streamer=chub, backup_conns=1)
            log.info('start pls')
        else:
            self.control_hub = None

        twitch_client_id = None
        twitch_oauth = None
        if 'twitchapi' in self.config:
            twitch_client_id = self.config['twitchapi'].get('client_id', None)
            twitch_oauth = self.config['twitchapi'].get('oauth', None)

        self.twitchapi = TwitchAPI(twitch_client_id, twitch_oauth)

        self.reactor.add_global_handler('all_events', self._dispatcher, -10)

        self.whisper_manager = WhisperConnectionManager(self.reactor, self, self.streamer, TMI.whispers_message_limit, TMI.whispers_limit_interval)
        self.whisper_manager.start(accounts=[{'username': self.nickname, 'oauth': self.password, 'can_send_whispers': self.config.getboolean('main', 'add_self_as_whisper_account')}])

        self.ascii_timeout_duration = 120
        self.msg_length_timeout_duration = 120

        self.data = {}
        self.data_cb = {}
        self.url_regex = re.compile(self.url_regex_str, re.IGNORECASE)

        self.data['broadcaster'] = self.streamer
        self.data['version'] = self.version
        self.data_cb['status_length'] = self.c_status_length
        self.data_cb['stream_status'] = self.c_stream_status
        self.data_cb['bot_uptime'] = self.c_uptime
        self.data_cb['current_time'] = self.c_current_time

        self.silent = True if args.silent else self.silent

        if self.silent:
            log.info('Silent mode enabled')

        self.reconnection_interval = 5

        """
        For actions that need to access the main thread,
        we can use the mainthread_queue.
        """
        self.mainthread_queue = ActionQueue()
        self.execute_every(1, self.mainthread_queue.parse_action)

        self.websocket_manager = WebSocketManager(self)

        """
        Update chatters every `update_chatters_interval' minutes.
        By default, this is set to run every 5 minutes.
        """
        self.execute_every(self.update_chatters_interval * 60,
                           self.action_queue.add,
                           (self.update_chatters_stage1, ))

        try:
            if self.config['twitchapi']['update_subscribers'] == '1':
                self.execute_every(30 * 60,
                                   self.action_queue.add,
                                   (self.update_subscribers_stage1, ))
        except:
            pass

        # XXX: TEMPORARY UGLY CODE
        HandlerManager.add_handler('on_user_gain_tokens', self.on_user_gain_tokens)

    def on_user_gain_tokens(self, user, tokens_gained):
        self.whisper(user.username, 'You finished todays quest! You have been awarded with {} tokens.'.format(tokens_gained))

    def update_subscribers_stage1(self):
        limit = 100
        offset = 0
        subscribers = []
        log.info('Starting stage1 subscribers update')

        try:
            retry_same = 0
            while True:
                log.debug('Beginning sub request {0} {1}'.format(limit, offset))
                subs, retry_same, error = self.twitchapi.get_subscribers(self.streamer, limit, offset, 0 if retry_same is False else retry_same)
                log.debug('got em!')

                if error is True:
                    log.error('Too many attempts, aborting')
                    return False

                if retry_same is False:
                    offset += limit
                    if len(subs) == 0:
                        # We don't need to retry, and the last query finished propery
                        # Break out of the loop and start fiddling with the subs!
                        log.debug('Done!')
                        break
                    else:
                        log.debug('Fetched {0} subs'.format(len(subs)))
                        subscribers.extend(subs)

                if retry_same is not False:
                    # In case the next attempt is a retry, wait for 3 seconds
                    log.debug('waiting for 3 seconds...')
                    time.sleep(3)
                    log.debug('waited enough!')
            log.debug('Finished with the while True loop!')
        except:
            log.exception('Caught an exception while trying to get subscribers')
            return

        log.info('Ended stage1 subscribers update')
        if len(subscribers) > 0:
            log.info('Got some subscribers, so we are pushing them to stage 2!')
            self.mainthread_queue.add(self.update_subscribers_stage2,
                                      args=[subscribers])
            log.info('Pushed them now.')

    def update_subscribers_stage2(self, subscribers):
        log.debug('begiunning stage 2 of update subs')
        self.kvi['active_subs'].set(len(subscribers) - 1)

        log.debug('Bulk loading subs...')
        loaded_subscribers = self.users.bulk_load(subscribers)
        log.debug('ok!')

        log.debug('settings all loaded users as non-subs')
        self.users.reset_subs()
        """
        for username, user in self.users.items():
            if user.subscriber:
                user.subscriber = False
                """

        log.debug('ok!, setting loaded subs as subs')
        for user in loaded_subscribers:
            user.subscriber = True

        log.debug('end of stage 2 of update subs')

    def update_chatters_stage1(self):
        chatters = self.twitchapi.get_chatters(self.streamer)
        if len(chatters) > 0:
            self.mainthread_queue.add(self.update_chatters_stage2, args=[chatters])

    def update_chatters_stage2(self, chatters):
        points = 1 if self.is_online else 0

        log.debug('Updating {0} chatters'.format(len(chatters)))

        u_chatters = self.users.bulk_load(chatters)

        for user in u_chatters:
            if self.is_online:
                user.minutes_in_chat_online += self.update_chatters_interval
            else:
                user.minutes_in_chat_offline += self.update_chatters_interval
            num_points = points
            if user.subscriber:
                num_points *= 5
            if self.streamer == 'forsenlol' and 'trump_sub' in user.tags:
                num_points *= 0.5
            user.touch(num_points)

    def _dispatcher(self, connection, event):
        if connection == self.connection_manager.get_main_conn() or connection in self.whisper_manager or (self.control_hub is not None and connection == self.control_hub.get_main_conn()):
            method = getattr(self, 'on_' + event.type, do_nothing)
            method(connection, event)

    def start(self):
        """Start the IRC client."""
        self.reactor.process_forever()

    def get_kvi_value(self, key, extra={}):
        if key in self.kvi.data:
            # We check if the value exists first.
            # We don't want to create a bunch of unneccesary KVIData's
            return self.kvi[key].get()
        return 0

    def get_last_tweet(self, key, extra={}):
        return self.twitter_manager.get_last_tweet(key)

    def get_emote_tm(self, key, extra={}):
        emote = self.emotes.find(key)
        if emote:
            return emote.tm
        return None

    def get_emote_count(self, key, extra={}):
        emote = self.emotes.find(key)
        if emote:
            return '{0:,d}'.format(emote.count)
        return None

    def get_emote_tm_record(self, key, extra={}):
        emote = self.emotes.find(key)
        if emote:
            return '{0:,d}'.format(emote.tm_record)
        return None

    def get_source_value(self, key, extra={}):
        try:
            return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_user_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_usersource_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
            else:
                return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_time_value(self, key, extra={}):
        try:
            tz = timezone(key)
            return datetime.datetime.now(tz).strftime(self.date_fmt)
        except:
            log.exception('Unhandled exception in get_time_value')

        return None

    def get_current_song_value(self, key, extra={}):
        if self.stream_manager.online:
            current_song = PleblistManager.get_current_song(self.stream_manager.current_stream.id)
            inner_keys = key.split('.')
            val = current_song
            for inner_key in inner_keys:
                val = getattr(val, inner_key, None)
                if val is None:
                    return None
            if val is not None:
                return val
        return None

    def get_strictargs_value(self, key, extra={}):
        ret = self.get_args_value(key, extra)

        if len(ret) == 0:
            return None
        return ret

    def get_args_value(self, key, extra={}):
        range = None
        try:
            msg_parts = extra['message'].split(' ')
        except (KeyError, AttributeError):
            msg_parts = ['']

        try:
            if '-' in key:
                range_str = key.split('-')
                if len(range_str) == 2:
                    range = (int(range_str[0]), int(range_str[1]))

            if range is None:
                range = (int(key), len(msg_parts))
        except (TypeError, ValueError):
            range = (0, len(msg_parts))

        try:
            return ' '.join(msg_parts[range[0]:range[1]])
        except AttributeError:
            return ''
        except:
            log.exception('UNHANDLED ERROR IN get_args_value')
            return ''

    def get_notify_value(self, key, extra={}):
        payload = {
                'message': extra['message'] or '',
                'trigger': extra['trigger'],
                'user': extra['source'].username_raw,
                }
        self.websocket_manager.emit('notify', payload)

        return ''

    def get_value(self, key, extra={}):
        if key in extra:
            return extra[key]
        elif key in self.data:
            return self.data[key]
        elif key in self.data_cb:
            return self.data_cb[key]()

        log.warning('Unknown key passed to get_value: {0}'.format(key))
        return None

    def privmsg(self, message, channel=None, increase_message=True):
        try:
            if channel is None:
                channel = self.channel

            if self.control_hub is not None and self.control_hub.channel == channel:
                self.control_hub.privmsg(channel, message)
            else:
                self.connection_manager.privmsg(channel, message, increase_message=increase_message)
        except Exception:
            log.exception('Exception caught while sending privmsg')

    def c_uptime(self):
        return time_since(datetime.datetime.now().timestamp(), self.start_time.timestamp())

    def c_current_time(self):
        return datetime.datetime.now()

    @property
    def is_online(self):
        return self.stream_manager.online

    def c_stream_status(self):
        return 'online' if self.stream_manager.online else 'offline'

    def c_status_length(self):
        if self.stream_manager.online:
            return time_since(time.time(), self.stream_manager.current_stream.stream_start.timestamp())
        else:
            if self.stream_manager.last_stream is not None:
                return time_since(time.time(), self.stream_manager.last_stream.stream_end.timestamp())
            else:
                return 'No recorded stream FeelsBadMan '

    def _ban(self, username):
        self.privmsg('.ban {0}'.format(username), increase_message=False)

    def execute_at(self, at, function, arguments=()):
        self.reactor.execute_at(at, function, arguments)

    def execute_delayed(self, delay, function, arguments=()):
        self.reactor.execute_delayed(delay, function, arguments)

    def execute_every(self, period, function, arguments=()):
        self.reactor.execute_every(period, function, arguments)

    def ban(self, username):
        log.debug('Banning {}'.format(username))
        self._timeout(username, 30)
        self.execute_delayed(1, self._ban, (username, ))

    def ban_user(self, user):
        if not user.ban_immune:
            self._timeout(user.username, 30)
            self.execute_delayed(1, self._ban, (user.username, ))

    def unban(self, username):
        self.privmsg('.unban {0}'.format(username), increase_message=False)

    def _timeout(self, username, duration):
        self.privmsg('.timeout {0} {1}'.format(username, duration), increase_message=False)

    def timeout(self, username, duration):
        log.debug('Timing out {} for {} seconds'.format(username, duration))
        self._timeout(username, duration)
        self.execute_delayed(1, self._timeout, (username, duration))

    def timeout_warn(self, user, duration):
        duration, punishment = user.timeout(duration, warning_module=self.module_manager['warning'])
        if not user.ban_immune:
            self.timeout(user.username, duration)
            return (duration, punishment)
        return (0, punishment)

    def timeout_user(self, user, duration):
        if not user.ban_immune:
            self._timeout(user.username, duration)
            self.execute_delayed(1, self._timeout, (user.username, duration))

    def whisper(self, username, *messages, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string as a whisper to username
        """

        if len(messages) < 0:
            return False

        if self.whisper_manager:
            self.whisper_manager.whisper(username, separator.join(messages))
        else:
            log.debug('No whisper conn set up.')

    def say(self, *messages, channel=None, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string to the given channel.
        """

        if len(messages) < 0:
            return False

        if not self.silent:
            message = separator.join(messages).strip()

            if len(message) >= 1:
                if (message[0] == '.' or message[0] == '/') and not message[1:3] == 'me':
                    log.warning('Message we attempted to send started with . or /, skipping.')
                    return

                log.info('Sending message: {0}'.format(message))

                self.privmsg(message[:510], channel)

    def me(self, message, channel=None):
        if not self.silent:
            message = message.strip()

            if len(message) >= 1:
                if message[0] == '.' or message[0] == '/':
                    log.warning('Message we attempted to send started with . or /, skipping.')
                    return

                log.info('Sending message: {0}'.format(message))

                self.privmsg('.me ' + message[:500], channel)

    def parse_version(self):
        self.version = self.version

        if self.dev:
            try:
                current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf8').strip()
                latest_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf8').strip()[:8]
                commit_number = subprocess.check_output(['git', 'rev-list', 'HEAD', '--count']).decode('utf8').strip()
                self.version = '{0} DEV ({1}, {2}, commit {3})'.format(self.version, current_branch, latest_commit, commit_number)
            except:
                pass

    def on_welcome(self, chatconn, event):
        if chatconn in self.whisper_manager:
            log.debug('Connected to Whisper server.')
        else:
            log.debug('Connected to IRC server.')

    def connect(self):
        return self.connection_manager.start()

    def on_disconnect(self, chatconn, event):
        if chatconn in self.whisper_manager:
            log.debug('Whispers: Disconnecting from Whisper server')
            self.whisper_manager.on_disconnect(chatconn)
        else:
            log.debug('Disconnected from IRC server')
            self.connection_manager.on_disconnect(chatconn)

    def parse_message(self, msg_raw, source, event, tags={}, whisper=False):
        msg_lower = msg_raw.lower()

        if source is None:
            log.error('No valid user passed to parse_message')
            return False

        if source.banned:
            self.ban(source.username)
            return False

        # If a user types when timed out, we assume he's been unbanned for a good reason and remove his flag.
        if source.timed_out is True:
            source.timed_out = False

        message_emotes = []
        for tag in tags:
            if tag['key'] == 'subscriber' and event.target == self.channel:
                if source.subscriber and tag['value'] == '0':
                    source.subscriber = False
                elif not source.subscriber and tag['value'] == '1':
                    source.subscriber = True
            elif tag['key'] == 'emotes' and tag['value']:
                emote_data = tag['value'].split('/')
                for emote in emote_data:
                    try:
                        emote_id, emote_occurrence = emote.split(':')
                        emote_indices = emote_occurrence.split(',')
                        emote_count = len(emote_indices)
                        emote = self.emotes[int(emote_id)]
                        first_index, last_index = emote_indices[0].split('-')
                        first_index = int(first_index)
                        last_index = int(last_index)
                        emote_code = msg_raw[first_index:last_index + 1]
                        if emote_code[0] == ':':
                            emote_code = emote_code.upper()
                        message_emotes.append({
                            'code': emote_code,
                            'twitch_id': emote_id,
                            'start': first_index,
                            'end': last_index,
                            })

                        tag_as = None
                        if emote_code.startswith('trump'):
                            tag_as = 'trump_sub'
                        elif emote_code.startswith('eloise'):
                            tag_as = 'eloise_sub'
                        elif emote_code.startswith('forsen'):
                            tag_as = 'forsen_sub'
                        elif emote_code.startswith('nostam'):
                            tag_as = 'nostam_sub'
                        elif emote_code.startswith('reynad'):
                            tag_as = 'reynad_sub'
                        elif emote_code.startswith('athene'):
                            tag_as = 'athene_sub'
                        elif emote_id in [12760, 35600, 68498, 54065, 59411, 59412, 59413, 62683, 70183, 70181, 68499, 70429, 70432, 71432, 71433]:
                            tag_as = 'massan_sub'

                        if tag_as is not None:
                            if source.tag_as(tag_as) is True:
                                self.execute_delayed(60 * 60 * 24, source.remove_tag, (tag_as, ))

                        if emote.id is None and emote.code is None:
                            # The emote we just detected is new, set its code.
                            emote.code = emote_code
                            if emote.code not in self.emotes:
                                self.emotes[emote.code] = emote

                        emote.add(emote_count, self.reactor)
                    except:
                        log.exception('Exception caught while splitting emote data')
                        log.error('Emote data: {}'.format(emote_data))
                        log.error('msg_raw: {}'.format(msg_raw))
            elif tag['key'] == 'display-name' and tag['value']:
                try:
                    source.update_username(tag['value'])
                except:
                    log.exception('Exception caught while updating a users username')
            elif tag['key'] == 'user-type':
                source.moderator = tag['value'] == 'mod' or source.username == self.streamer

        for emote in self.emotes.custom_data:
            num = 0
            for match in emote.regex.finditer(msg_raw):
                num += 1
                message_emotes.append({
                    'code': emote.code,
                    'bttv_hash': emote.emote_hash,
                    'start': match.span()[0],
                    'end': match.span()[1] - 1,  # don't ask me
                    })
            if num > 0:
                emote.add(num, self.reactor)

        urls = self.find_unique_urls(msg_raw)

        log.debug('{2}{0}: {1}'.format(source.username, msg_raw, '<w>' if whisper else ''))

        res = HandlerManager.trigger('on_message',
                source, msg_raw, message_emotes, whisper, urls, event,
                stop_on_false=True)
        if res is False:
            return False

        source.last_seen = datetime.datetime.now()
        source.last_active = datetime.datetime.now()

        if source.ignored:
            return False

        if msg_lower[:1] == '!':
            msg_lower_parts = msg_lower.split(' ')
            trigger = msg_lower_parts[0][1:]
            msg_raw_parts = msg_raw.split(' ')
            remaining_message = ' '.join(msg_raw_parts[1:]) if len(msg_raw_parts) > 1 else None
            if trigger in self.commands:
                command = self.commands[trigger]
                extra_args = {
                        'emotes': message_emotes,
                        'trigger': trigger,
                        }
                command.run(self, source, remaining_message, event=event, args=extra_args, whisper=whisper)

    def on_whisper(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        source = self.users[event.source.user.lower()]
        self.parse_message(event.arguments[0], source, event, whisper=True, tags=event.tags)

    def on_ping(self, chatconn, event):
        # self.say('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        log.info('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        self.last_ping = datetime.datetime.now()

    def on_pong(self, chatconn, event):
        # self.say('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        log.info('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        self.last_pong = datetime.datetime.now()

    def on_pubnotice(self, chatconn, event):
        type = 'whisper' if chatconn in self.whisper_manager else 'normal'
        log.debug('NOTICE {}@{}: {}'.format(type, event.target, event.arguments))

    def on_action(self, chatconn, event):
        self.on_pubmsg(chatconn, event)

    def on_pubmsg(self, chatconn, event):
        if event.source.user == self.nickname:
            return False

        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        source = self.users[event.source.user.lower()]

        res = HandlerManager.trigger('on_pubmsg',
                source, event.arguments[0],
                stop_on_false=True)
        if res is False:
            return False

        self.parse_message(event.arguments[0], source, event, tags=event.tags)

    @time_method
    def reload_all(self):
        log.info('Reloading all...')
        for key, manager in self.reloadable.items():
            log.debug('Reloading {0}'.format(key))
            manager.reload()
            log.debug('Done with {0}'.format(key))
        log.info('ok!')

    @time_method
    def commit_all(self):
        log.info('Commiting all...')
        for key, manager in self.commitable.items():
            log.info('Commiting {0}'.format(key))
            manager.commit()
            log.info('Done with {0}'.format(key))
        log.info('ok!')

        HandlerManager.trigger('on_commit', stop_on_false=False)

    def quit(self, message, event, **options):
        quit_chub = self.config['main'].get('control_hub', None)
        quit_delay = 1

        if quit_chub is not None and event.target == ('#{}'.format(quit_chub)):
            quit_delay_random = 300
            try:
                if message is not None and int(message.split()[0]) >= 1:
                    quit_delay_random = int(message.split()[0])
            except (IndexError, ValueError, TypeError):
                pass
            quit_delay = random.randint(0, quit_delay_random)
            log.info('{} is restarting in {} seconds.'.format(self.nickname, quit_delay))

        self.execute_delayed(quit_delay, self.quit_bot)

    def quit_bot(self, **options):
        self.commit_all()
        if self.phrases['quit']:
            phrase_data = {
                    'nickname': self.nickname,
                    'version': self.version,
                    }

            try:
                self.say(self.phrases['quit'].format(**phrase_data))
            except Exception:
                log.exception('Exception caught while trying to say quit phrase')

        self.twitter_manager.quit()
        self.socket_manager.quit()

        if self.whisper_manager:
            self.whisper_manager.quit()

        sys.exit(0)

    def apply_filter(self, resp, filter):
        available_filters = {
                'strftime': _filter_strftime,
                'lower': lambda var, args: var.lower(),
                'upper': lambda var, args: var.upper(),
                'time_since_minutes': lambda var, args: 'no time' if var == 0 else time_since(var * 60, 0, format='long'),
                'time_since': lambda var, args: 'no time' if var == 0 else time_since(var, 0, format='long'),
                'time_since_dt': _filter_time_since_dt,
                'urlencode': lambda var, args: urllib.parse.urlencode(var),
                'join': _filter_join,
                'number_format': _filter_number_format,
                }
        if filter.name in available_filters:
            return available_filters[filter.name](resp, filter.arguments)
        return resp

    def find_unique_urls(self, message):
        from pajbot.modules.linkchecker import find_unique_urls
        return find_unique_urls(self.url_regex, message)
Esempio n. 13
0
class FollowAgeModule(BaseModule):

    ID = __name__.split('.')[-1]
    NAME = 'Follow age'
    DESCRIPTION = 'Makes two commands available: !followage and !followsince'
    CATEGORY = 'Feature'
    SETTINGS = [
        ModuleSetting(key='action_followage',
                      label='MessageAction for !followage',
                      type='options',
                      required=True,
                      default='say',
                      options=['say', 'whisper', 'reply']),
        ModuleSetting(key='action_followsince',
                      label='MessageAction for !followsince',
                      type='options',
                      required=True,
                      default='say',
                      options=['say', 'whisper', 'reply']),
    ]

    def __init__(self):
        super().__init__()
        self.action_queue = ActionQueue()
        self.action_queue.start()

    def load_commands(self, **options):
        # TODO: Have delay modifiable in settings

        self.commands['followage'] = pajbot.models.command.Command.raw_command(
            self.follow_age,
            delay_all=4,
            delay_user=8,
            description='Check your or someone elses follow age for a channel',
            can_execute_with_whisper=True,
            examples=[
                pajbot.models.command.CommandExample(
                    None,
                    'Check your own follow age',
                    chat='user:!followage\n'
                    'bot:pajlada, you have been following Karl_Kons for 4 months and 24 days',
                    description=
                    'Check how long you have been following the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someone elses follow age',
                    chat='user:!followage NightNacht\n'
                    'bot:pajlada, NightNacht has been following Karl_Kons for 5 months and 4 days',
                    description=
                    'Check how long any user has been following the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someones follow age for a certain streamer',
                    chat='user:!followage NightNacht forsenlol\n'
                    'bot:pajlada, NightNacht has been following forsenlol for 1 year and 4 months',
                    description=
                    'Check how long NightNacht has been following forsenlol').
                parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check your own follow age for a certain streamer',
                    chat='user:!followage pajlada forsenlol\n'
                    'bot:pajlada, you have been following forsenlol for 1 year and 3 months',
                    description=
                    'Check how long you have been following forsenlol').parse(
                    ),
            ],
        )

        self.commands['followsince'] = pajbot.models.command.Command.raw_command(
            self.follow_since,
            delay_all=4,
            delay_user=8,
            description=
            'Check from when you or someone else first followed a channel',
            can_execute_with_whisper=True,
            examples=[
                pajbot.models.command.CommandExample(
                    None,
                    'Check your own follow since',
                    chat='user:!followsince\n'
                    'bot:pajlada, you have been following Karl_Kons since 04 March 2015, 07:02:01 UTC',
                    description=
                    'Check when you first followed the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someone elses follow since',
                    chat='user:!followsince NightNacht\n'
                    'bot:pajlada, NightNacht has been following Karl_Kons since 03 July 2014, 04:12:42 UTC',
                    description=
                    'Check when NightNacht first followed the current streamer (Karl_Kons in this case)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check someone elses follow since for another streamer',
                    chat='user:!followsince NightNacht forsenlol\n'
                    'bot:pajlada, NightNacht has been following forsenlol since 13 June 2013, 13:10:51 UTC',
                    description=
                    'Check when NightNacht first followed the given streamer (forsenlol)'
                ).parse(),
                pajbot.models.command.CommandExample(
                    None,
                    'Check your follow since for another streamer',
                    chat='user:!followsince pajlada forsenlol\n'
                    'bot:pajlada, you have been following forsenlol since 16 December 1990, 03:06:51 UTC',
                    description=
                    'Check when you first followed the given streamer (forsenlol)'
                ).parse(),
            ],
        )

    def check_follow_age(self, bot, source, username, streamer, event):
        streamer = bot.streamer if streamer is None else streamer.lower()
        age = bot.twitchapi.get_follow_relationship(username, streamer)
        is_self = source.username == username
        message = ''

        if age:
            # Following
            human_age = time_since(
                datetime.datetime.now().timestamp() - age.timestamp(), 0)
            suffix = 'been following {} for {}'.format(streamer, human_age)
            if is_self:
                message = 'You have ' + suffix
            else:
                message = username + ' has ' + suffix
        else:
            # Not following
            suffix = 'not following {}'.format(streamer)
            if is_self:
                message = 'You are ' + suffix
            else:
                message = username + ' is ' + suffix

        bot.send_message_to_user(source,
                                 message,
                                 event,
                                 method=self.settings['action_followage'])

    def check_follow_since(self, bot, source, username, streamer, event):
        streamer = bot.streamer if streamer is None else streamer.lower()
        follow_since = bot.twitchapi.get_follow_relationship(
            username, streamer)
        is_self = source.username == username
        message = ''

        if follow_since:
            # Following
            human_age = follow_since.strftime('%d %B %Y, %X')
            suffix = 'been following {} since {} UTC'.format(
                streamer, human_age)
            if is_self:
                message = 'You have ' + suffix
            else:
                message = username + ' has ' + suffix
        else:
            # Not following
            suffix = 'not following {}'.format(streamer)
            if is_self:
                message = 'You are ' + suffix
            else:
                message = username + ' is ' + suffix

        bot.send_message_to_user(source,
                                 message,
                                 event,
                                 method=self.settings['action_followsince'])

    def follow_age(self, **options):
        source = options['source']
        message = options['message']
        bot = options['bot']
        event = options['event']

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_age,
                              args=[bot, source, username, streamer, event])

    def follow_since(self, **options):
        bot = options['bot']
        source = options['source']
        message = options['message']
        event = options['event']

        username, streamer = self.parse_message(bot, source, message)

        self.action_queue.add(self.check_follow_since,
                              args=[bot, source, username, streamer, event])

    def parse_message(self, bot, source, message):
        username = source.username
        streamer = None
        if message is not None and len(message) > 0:
            message_split = message.split(' ')
            if len(message_split[0]) and message_split[0].replace(
                    '_', '').isalnum():
                username = message_split[0].lower()
            if len(message_split) > 1 and message_split[1].replace(
                    '_', '').isalnum():
                streamer = message_split[1]

        return username, streamer
Esempio n. 14
0
class Bot:
    """
    Main class for the twitch bot
    """

    version = '2.7.3'
    date_fmt = '%H:%M'
    admin = None
    url_regex_str = r'\(?(?:(http|https):\/\/)?(?:((?:[^\W\s]|\.|-|[:]{1})+)@{1})?((?:www.)?(?:[^\W\s]|\.|-)+[\.][^\W\s]{2,4}|localhost(?=\/)|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?::(\d*))?([\/]?[^\s\?]*[\/]{1})*(?:\/?([^\s\n\?\[\]\{\}\#]*(?:(?=\.)){1}|[^\s\n\?\[\]\{\}\.\#]*)?([\.]{1}[^\s\?\#]*)?)?(?:\?{1}([^\s\n\#\[\]]*))?([\#][^\s\n]*)?\)?'

    def parse_args():
        parser = argparse.ArgumentParser()
        parser.add_argument('--config',
                            '-c',
                            default='config.ini',
                            help='Specify which config file to use '
                            '(default: config.ini)')
        parser.add_argument('--silent',
                            action='count',
                            help='Decides whether the bot should be '
                            'silent or not')
        # TODO: Add a log level argument.

        return parser.parse_args()

    def load_default_phrases(self):
        default_phrases = {
            'welcome':
            False,
            'quit':
            False,
            'nl':
            '{username} has typed {num_lines} messages in this channel!',
            'nl_0':
            '{username} has not typed any messages in this channel BibleThump',
            'nl_pos':
            '{username} is rank {nl_pos} line-farmer in this channel!',
            'new_sub':
            'Sub hype! {username} just subscribed PogChamp',
            'resub':
            'Resub hype! {username} just subscribed, {num_months} months in a row PogChamp <3 PogChamp',
            'point_pos':
            '{username_w_verb} rank {point_pos} point-hoarder in this channel with {points} points.',
        }
        if 'phrases' in self.config:
            self.phrases = {}

            for phrase_key, phrase_value in self.config['phrases'].items():
                if len(phrase_value.strip()) <= 0:
                    self.phrases[phrase_key] = False
                else:
                    self.phrases[phrase_key] = phrase_value

            for phrase_key, phrase_value in default_phrases.items():
                if phrase_key not in self.phrases:
                    self.phrases[phrase_key] = phrase_value
        else:
            self.phrases = default_phrases

    def load_config(self, config):
        self.config = config

        self.nickname = config['main'].get('nickname', 'pajbot')
        self.password = config['main'].get('password', 'abcdef')

        self.timezone = config['main'].get('timezone', 'UTC')

        self.trusted_mods = config.getboolean('main', 'trusted_mods')

        TimeManager.init_timezone(self.timezone)

        if 'streamer' in config['main']:
            self.streamer = config['main']['streamer']
            self.channel = '#' + self.streamer
        elif 'target' in config['main']:
            self.channel = config['main']['target']
            self.streamer = self.channel[1:]

        self.wolfram = None
        try:
            if 'wolfram' in config['main']:
                import wolframalpha
                self.wolfram = wolframalpha.Client(config['main']['wolfram'])
        except:
            pass

        self.silent = False
        self.dev = False

        if 'flags' in config:
            self.silent = True if 'silent' in config['flags'] and config[
                'flags']['silent'] == '1' else self.silent
            self.dev = True if 'dev' in config['flags'] and config['flags'][
                'dev'] == '1' else self.dev

        DBManager.init(self.config['main']['db'])

        redis_options = {}
        if 'redis' in config:
            log.info(config._sections['redis'])
            redis_options = config._sections['redis']

        RedisManager.init(**redis_options)

    def __init__(self, config, args=None):
        self.load_config(config)
        self.last_ping = datetime.datetime.now()
        self.last_pong = datetime.datetime.now()

        self.load_default_phrases()

        self.db_session = DBManager.create_session()

        try:
            subprocess.check_call(
                ['alembic', 'upgrade', 'head'] +
                ['--tag="{0}"'.format(' '.join(sys.argv[1:]))])
        except subprocess.CalledProcessError:
            log.exception('aaaa')
            log.error(
                'Unable to call `alembic upgrade head`, this means the database could be out of date. Quitting.'
            )
            sys.exit(1)
        except PermissionError:
            log.error(
                'No permission to run `alembic upgrade head`. This means your user probably doesn\'t have execution rights on the `alembic` binary.'
            )
            log.error(
                'The error can also occur if it can\'t find `alembic` in your PATH, and instead tries to execute the alembic folder.'
            )
            sys.exit(1)
        except FileNotFoundError:
            log.error(
                'Could not found an installation of alembic. Please install alembic to continue.'
            )
            sys.exit(1)
        except:
            log.exception('Unhandled exception when calling db update')
            sys.exit(1)

        # Actions in this queue are run in a separate thread.
        # This means actions should NOT access any database-related stuff.
        self.action_queue = ActionQueue()
        self.action_queue.start()

        self.reactor = irc.client.Reactor(self.on_connect)
        self.start_time = datetime.datetime.now()
        ActionParser.bot = self

        HandlerManager.init_handlers()

        self.socket_manager = SocketManager(self)
        self.stream_manager = StreamManager(self)

        StreamHelper.init_bot(self, self.stream_manager)
        ScheduleManager.init()

        self.users = UserManager()
        self.decks = DeckManager()
        self.module_manager = ModuleManager(self.socket_manager,
                                            bot=self).load()
        self.commands = CommandManager(socket_manager=self.socket_manager,
                                       module_manager=self.module_manager,
                                       bot=self).load()
        self.filters = FilterManager().reload()
        self.banphrase_manager = BanphraseManager(self).load()
        self.timer_manager = TimerManager(self).load()
        self.kvi = KVIManager()
        self.emotes = EmoteManager(self).reload()
        self.twitter_manager = TwitterManager(self)
        self.duel_manager = DuelManager(self)

        HandlerManager.trigger('on_managers_loaded')

        # Reloadable managers
        self.reloadable = {
            'filters': self.filters,
            'emotes': self.emotes,
        }

        # Commitable managers
        self.commitable = {
            'commands': self.commands,
            'filters': self.filters,
            'emotes': self.emotes,
            'users': self.users,
            'banphrases': self.banphrase_manager,
        }

        self.execute_every(10 * 60, self.commit_all)

        try:
            self.admin = self.config['main']['admin']
        except KeyError:
            log.warning(
                'No admin user specified. See the [main] section in config.example.ini for its usage.'
            )
        if self.admin:
            self.users[self.admin].level = 2000

        self.parse_version()

        relay_host = self.config['main'].get('relay_host', None)
        relay_password = self.config['main'].get('relay_password', None)
        if relay_host is None or relay_password is None:
            self.irc = MultiIRCManager(self)
        else:
            self.irc = SingleIRCManager(self)

        self.reactor.add_global_handler('all_events', self.irc._dispatcher,
                                        -10)

        twitch_client_id = None
        twitch_oauth = None
        if 'twitchapi' in self.config:
            twitch_client_id = self.config['twitchapi'].get('client_id', None)
            twitch_oauth = self.config['twitchapi'].get('oauth', None)

        self.twitchapi = TwitchAPI(twitch_client_id, twitch_oauth)

        self.ascii_timeout_duration = 120
        self.msg_length_timeout_duration = 120

        self.data = {}
        self.data_cb = {}
        self.url_regex = re.compile(self.url_regex_str, re.IGNORECASE)

        self.data['broadcaster'] = self.streamer
        self.data['version'] = self.version
        self.data_cb['status_length'] = self.c_status_length
        self.data_cb['stream_status'] = self.c_stream_status
        self.data_cb['bot_uptime'] = self.c_uptime
        self.data_cb['current_time'] = self.c_current_time

        self.silent = True if args.silent else self.silent

        if self.silent:
            log.info('Silent mode enabled')

        self.reconnection_interval = 5
        """
        For actions that need to access the main thread,
        we can use the mainthread_queue.
        """
        self.mainthread_queue = ActionQueue()
        self.execute_every(1, self.mainthread_queue.parse_action)

        self.websocket_manager = WebSocketManager(self)

        try:
            if self.config['twitchapi']['update_subscribers'] == '1':
                self.execute_every(30 * 60, self.action_queue.add,
                                   (self.update_subscribers_stage1, ))
        except:
            pass

        # XXX: TEMPORARY UGLY CODE
        HandlerManager.add_handler('on_user_gain_tokens',
                                   self.on_user_gain_tokens)

    def on_connect(self, sock):
        return self.irc.on_connect(sock)

    def on_user_gain_tokens(self, user, tokens_gained):
        self.whisper(
            user.username,
            'You finished todays quest! You have been awarded with {} tokens.'.
            format(tokens_gained))

    def update_subscribers_stage1(self):
        limit = 100
        offset = 0
        subscribers = []
        log.info('Starting stage1 subscribers update')

        try:
            retry_same = 0
            while True:
                log.debug('Beginning sub request {0} {1}'.format(
                    limit, offset))
                subs, retry_same, error = self.twitchapi.get_subscribers(
                    self.streamer, limit, offset,
                    0 if retry_same is False else retry_same)
                log.debug('got em!')

                if error is True:
                    log.error('Too many attempts, aborting')
                    return False

                if retry_same is False:
                    offset += limit
                    if len(subs) == 0:
                        # We don't need to retry, and the last query finished propery
                        # Break out of the loop and start fiddling with the subs!
                        log.debug('Done!')
                        break
                    else:
                        log.debug('Fetched {0} subs'.format(len(subs)))
                        subscribers.extend(subs)

                if retry_same is not False:
                    # In case the next attempt is a retry, wait for 3 seconds
                    log.debug('waiting for 3 seconds...')
                    time.sleep(3)
                    log.debug('waited enough!')
            log.debug('Finished with the while True loop!')
        except:
            log.exception(
                'Caught an exception while trying to get subscribers')
            return

        log.info('Ended stage1 subscribers update')
        if len(subscribers) > 0:
            log.info(
                'Got some subscribers, so we are pushing them to stage 2!')
            self.mainthread_queue.add(self.update_subscribers_stage2,
                                      args=[subscribers])
            log.info('Pushed them now.')

    def update_subscribers_stage2(self, subscribers):
        log.debug('begiunning stage 2 of update subs')
        self.kvi['active_subs'].set(len(subscribers) - 1)

        log.debug('Bulk loading subs...')
        loaded_subscribers = self.users.bulk_load(subscribers)
        log.debug('ok!')

        log.debug('settings all loaded users as non-subs')
        self.users.reset_subs()
        """
        for username, user in self.users.items():
            if user.subscriber:
                user.subscriber = False
                """

        log.debug('ok!, setting loaded subs as subs')
        for user in loaded_subscribers:
            user.subscriber = True

        log.debug('end of stage 2 of update subs')

    def start(self):
        """Start the IRC client."""
        self.reactor.process_forever()

    def get_kvi_value(self, key, extra={}):
        return self.kvi[key].get()

    def get_last_tweet(self, key, extra={}):
        return self.twitter_manager.get_last_tweet(key)

    def get_emote_tm(self, key, extra={}):
        emote = self.emotes.find(key)
        if emote:
            return emote.tm
        return None

    def get_emote_count(self, key, extra={}):
        emote = self.emotes.find(key)
        if emote:
            return '{0:,d}'.format(emote.count)
        return None

    def get_emote_tm_record(self, key, extra={}):
        emote = self.emotes.find(key)
        if emote:
            return '{0:,d}'.format(emote.tm_record)
        return None

    def get_source_value(self, key, extra={}):
        try:
            return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_user_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_usersource_value(self, key, extra={}):
        try:
            user = self.users.find(extra['argument'])
            if user:
                return getattr(user, key)
            else:
                return getattr(extra['source'], key)
        except:
            log.exception('Caught exception in get_source_value')

        return None

    def get_time_value(self, key, extra={}):
        try:
            tz = timezone(key)
            return datetime.datetime.now(tz).strftime(self.date_fmt)
        except:
            log.exception('Unhandled exception in get_time_value')

        return None

    def get_current_song_value(self, key, extra={}):
        if self.stream_manager.online:
            current_song = PleblistManager.get_current_song(
                self.stream_manager.current_stream.id)
            inner_keys = key.split('.')
            val = current_song
            for inner_key in inner_keys:
                val = getattr(val, inner_key, None)
                if val is None:
                    return None
            if val is not None:
                return val
        return None

    def get_strictargs_value(self, key, extra={}):
        ret = self.get_args_value(key, extra)

        if len(ret) == 0:
            return None
        return ret

    def get_args_value(self, key, extra={}):
        range = None
        try:
            msg_parts = extra['message'].split(' ')
        except (KeyError, AttributeError):
            msg_parts = ['']

        try:
            if '-' in key:
                range_str = key.split('-')
                if len(range_str) == 2:
                    range = (int(range_str[0]), int(range_str[1]))

            if range is None:
                range = (int(key), len(msg_parts))
        except (TypeError, ValueError):
            range = (0, len(msg_parts))

        try:
            return ' '.join(msg_parts[range[0]:range[1]])
        except AttributeError:
            return ''
        except:
            log.exception('UNHANDLED ERROR IN get_args_value')
            return ''

    def get_notify_value(self, key, extra={}):
        payload = {
            'message': extra['message'] or '',
            'trigger': extra['trigger'],
            'user': extra['source'].username_raw,
        }
        self.websocket_manager.emit('notify', payload)

        return ''

    def get_value(self, key, extra={}):
        if key in extra:
            return extra[key]
        elif key in self.data:
            return self.data[key]
        elif key in self.data_cb:
            return self.data_cb[key]()

        log.warning('Unknown key passed to get_value: {0}'.format(key))
        return None

    def privmsg(self, message, channel=None, increase_message=True):
        if channel is None:
            channel = self.channel

        return self.irc.privmsg(message,
                                channel,
                                increase_message=increase_message)

    def c_uptime(self):
        return time_since(datetime.datetime.now().timestamp(),
                          self.start_time.timestamp())

    def c_current_time(self):
        return datetime.datetime.now()

    @property
    def is_online(self):
        return self.stream_manager.online

    def c_stream_status(self):
        return 'online' if self.stream_manager.online else 'offline'

    def c_status_length(self):
        if self.stream_manager.online:
            return time_since(
                time.time(),
                self.stream_manager.current_stream.stream_start.timestamp())
        else:
            if self.stream_manager.last_stream is not None:
                return time_since(
                    time.time(),
                    self.stream_manager.last_stream.stream_end.timestamp())
            else:
                return 'No recorded stream FeelsBadMan '

    def _ban(self, username):
        self.privmsg('.ban {0}'.format(username), increase_message=False)

    def execute_at(self, at, function, arguments=()):
        self.reactor.execute_at(at, function, arguments)

    def execute_delayed(self, delay, function, arguments=()):
        self.reactor.execute_delayed(delay, function, arguments)

    def execute_every(self, period, function, arguments=()):
        self.reactor.execute_every(period, function, arguments)

    def ban(self, username):
        log.debug('Banning {}'.format(username))
        self._timeout(username, 30)
        self.execute_delayed(1, self._ban, (username, ))

    def ban_user(self, user):
        if not user.ban_immune:
            self._timeout(user.username, 30)
            self.execute_delayed(1, self._ban, (user.username, ))

    def unban(self, username):
        self.privmsg('.unban {0}'.format(username), increase_message=False)

    def _timeout(self, username, duration):
        self.privmsg('.timeout {0} {1}'.format(username, duration),
                     increase_message=False)

    def timeout(self, username, duration):
        log.debug('Timing out {} for {} seconds'.format(username, duration))
        self._timeout(username, duration)
        self.execute_delayed(1, self._timeout, (username, duration))

    def timeout_warn(self, user, duration):
        duration, punishment = user.timeout(
            duration, warning_module=self.module_manager['warning'])
        if not user.ban_immune:
            self.timeout(user.username, duration)
            return (duration, punishment)
        return (0, punishment)

    def timeout_user(self, user, duration):
        if not user.ban_immune:
            self._timeout(user.username, duration)
            self.execute_delayed(1, self._timeout, (user.username, duration))

    def whisper(self, username, *messages, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string as a whisper to username
        """

        if len(messages) < 0:
            return False

        message = separator.join(messages)

        return self.irc.whisper(username, message)

    def say(self, *messages, channel=None, separator='. '):
        """
        Takes a sequence of strings and concatenates them with separator.
        Then sends that string to the given channel.
        """

        if len(messages) < 0:
            return False

        if not self.silent:
            message = separator.join(messages).strip()

            if len(message) >= 1:
                if (message[0] == '.'
                        or message[0] == '/') and not message[1:3] == 'me':
                    log.warning(
                        'Message we attempted to send started with . or /, skipping.'
                    )
                    return

                log.info('Sending message: {0}'.format(message))

                self.privmsg(message[:510], channel)

    def me(self, message, channel=None):
        if not self.silent:
            message = message.strip()

            if len(message) >= 1:
                if message[0] == '.' or message[0] == '/':
                    log.warning(
                        'Message we attempted to send started with . or /, skipping.'
                    )
                    return

                log.info('Sending message: {0}'.format(message))

                self.privmsg('.me ' + message[:500], channel)

    def parse_version(self):
        self.version = self.version

        if self.dev:
            try:
                current_branch = subprocess.check_output(
                    ['git', 'rev-parse', '--abbrev-ref',
                     'HEAD']).decode('utf8').strip()
                latest_commit = subprocess.check_output(
                    ['git', 'rev-parse', 'HEAD']).decode('utf8').strip()[:8]
                commit_number = subprocess.check_output(
                    ['git', 'rev-list', 'HEAD',
                     '--count']).decode('utf8').strip()
                self.version = '{0} DEV ({1}, {2}, commit {3})'.format(
                    self.version, current_branch, latest_commit, commit_number)
            except:
                pass

    def on_welcome(self, chatconn, event):
        return self.irc.on_welcome(chatconn, event)

    def connect(self):
        return self.irc.start()

    def on_disconnect(self, chatconn, event):
        self.irc.on_disconnect(chatconn, event)

    def parse_message(self, msg_raw, source, event, tags={}, whisper=False):
        msg_lower = msg_raw.lower()

        if source is None:
            log.error('No valid user passed to parse_message')
            return False

        if source.banned:
            self.ban(source.username)
            return False

        # If a user types when timed out, we assume he's been unbanned for a good reason and remove his flag.
        if source.timed_out is True:
            source.timed_out = False

        message_emotes = []
        for tag in tags:
            if tag['key'] == 'subscriber' and event.target == self.channel:
                if source.subscriber and tag['value'] == '0':
                    source.subscriber = False
                elif not source.subscriber and tag['value'] == '1':
                    source.subscriber = True
            elif tag['key'] == 'emotes' and tag['value']:
                emote_data = tag['value'].split('/')
                for emote in emote_data:
                    try:
                        emote_id, emote_occurrence = emote.split(':')
                        emote_indices = emote_occurrence.split(',')
                        emote_count = len(emote_indices)
                        emote = self.emotes[int(emote_id)]
                        first_index, last_index = emote_indices[0].split('-')
                        first_index = int(first_index)
                        last_index = int(last_index)
                        emote_code = msg_raw[first_index:last_index + 1]
                        if emote_code[0] == ':':
                            emote_code = emote_code.upper()
                        message_emotes.append({
                            'code': emote_code,
                            'twitch_id': emote_id,
                            'start': first_index,
                            'end': last_index,
                        })

                        tag_as = None
                        if emote_code.startswith('trump'):
                            tag_as = 'trump_sub'
                        elif emote_code.startswith('eloise'):
                            tag_as = 'eloise_sub'
                        elif emote_code.startswith('forsen'):
                            tag_as = 'forsen_sub'
                        elif emote_code.startswith('nostam'):
                            tag_as = 'nostam_sub'
                        elif emote_code.startswith('reynad'):
                            tag_as = 'reynad_sub'
                        elif emote_code.startswith('athene'):
                            tag_as = 'athene_sub'
                        elif emote_id in [
                                12760, 35600, 68498, 54065, 59411, 59412,
                                59413, 62683, 70183, 70181, 68499, 70429,
                                70432, 71432, 71433
                        ]:
                            tag_as = 'massan_sub'

                        if tag_as is not None:
                            if source.tag_as(tag_as) is True:
                                self.execute_delayed(60 * 60 * 24,
                                                     source.remove_tag,
                                                     (tag_as, ))

                        if emote.id is None and emote.code is None:
                            # The emote we just detected is new, set its code.
                            emote.code = emote_code
                            if emote.code not in self.emotes:
                                self.emotes[emote.code] = emote

                        emote.add(emote_count, self.reactor)
                    except:
                        log.exception(
                            'Exception caught while splitting emote data')
                        log.error('Emote data: {}'.format(emote_data))
                        log.error('msg_raw: {}'.format(msg_raw))
            elif tag['key'] == 'display-name' and tag['value']:
                try:
                    source.update_username(tag['value'])
                except:
                    log.exception(
                        'Exception caught while updating a users username')
            elif tag['key'] == 'user-type':
                source.moderator = tag[
                    'value'] == 'mod' or source.username == self.streamer

        for emote in self.emotes.custom_data:
            num = 0
            for match in emote.regex.finditer(msg_raw):
                num += 1
                message_emotes.append({
                    'code': emote.code,
                    'bttv_hash': emote.emote_hash,
                    'start': match.span()[0],
                    'end': match.span()[1] - 1,  # don't ask me
                })
            if num > 0:
                emote.add(num, self.reactor)

        urls = self.find_unique_urls(msg_raw)

        log.debug('{2}{0}: {1}'.format(source.username, msg_raw,
                                       '<w>' if whisper else ''))

        res = HandlerManager.trigger('on_message',
                                     source,
                                     msg_raw,
                                     message_emotes,
                                     whisper,
                                     urls,
                                     event,
                                     stop_on_false=True)
        if res is False:
            return False

        source.last_seen = datetime.datetime.now()
        source.last_active = datetime.datetime.now()

        if source.ignored:
            return False

        if msg_lower[:1] == '!':
            msg_lower_parts = msg_lower.split(' ')
            trigger = msg_lower_parts[0][1:]
            msg_raw_parts = msg_raw.split(' ')
            remaining_message = ' '.join(
                msg_raw_parts[1:]) if len(msg_raw_parts) > 1 else None
            if trigger in self.commands:
                command = self.commands[trigger]
                extra_args = {
                    'emotes': message_emotes,
                    'trigger': trigger,
                }
                command.run(self,
                            source,
                            remaining_message,
                            event=event,
                            args=extra_args,
                            whisper=whisper)

    def on_whisper(self, chatconn, event):
        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        source = self.users[event.source.user.lower()]
        self.parse_message(event.arguments[0],
                           source,
                           event,
                           whisper=True,
                           tags=event.tags)

    def on_ping(self, chatconn, event):
        # self.say('Received a ping. Last ping received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_ping.timestamp())))
        log.info('Received a ping. Last ping received {} ago'.format(
            time_since(datetime.datetime.now().timestamp(),
                       self.last_ping.timestamp())))
        self.last_ping = datetime.datetime.now()

    def on_pong(self, chatconn, event):
        # self.say('Received a pong. Last pong received {} ago'.format(time_since(datetime.datetime.now().timestamp(), self.last_pong.timestamp())))
        log.info('Received a pong. Last pong received {} ago'.format(
            time_since(datetime.datetime.now().timestamp(),
                       self.last_pong.timestamp())))
        self.last_pong = datetime.datetime.now()

    def on_pubnotice(self, chatconn, event):
        return
        type = 'whisper' if chatconn in self.whisper_manager else 'normal'
        log.debug('NOTICE {}@{}: {}'.format(type, event.target,
                                            event.arguments))

    def on_action(self, chatconn, event):
        self.on_pubmsg(chatconn, event)

    def on_pubmsg(self, chatconn, event):
        if event.source.user == self.nickname:
            return False

        # We use .lower() in case twitch ever starts sending non-lowercased usernames
        source = self.users[event.source.user.lower()]

        res = HandlerManager.trigger('on_pubmsg',
                                     source,
                                     event.arguments[0],
                                     stop_on_false=True)
        if res is False:
            return False

        self.parse_message(event.arguments[0], source, event, tags=event.tags)

    @time_method
    def reload_all(self):
        log.info('Reloading all...')
        for key, manager in self.reloadable.items():
            log.debug('Reloading {0}'.format(key))
            manager.reload()
            log.debug('Done with {0}'.format(key))
        log.info('ok!')

    @time_method
    def commit_all(self):
        log.info('Commiting all...')
        for key, manager in self.commitable.items():
            log.info('Commiting {0}'.format(key))
            manager.commit()
            log.info('Done with {0}'.format(key))
        log.info('ok!')

        HandlerManager.trigger('on_commit', stop_on_false=False)

    def quit(self, message, event, **options):
        quit_chub = self.config['main'].get('control_hub', None)
        quit_delay = 1

        if quit_chub is not None and event.target == ('#{}'.format(quit_chub)):
            quit_delay_random = 300
            try:
                if message is not None and int(message.split()[0]) >= 1:
                    quit_delay_random = int(message.split()[0])
            except (IndexError, ValueError, TypeError):
                pass
            quit_delay = random.randint(0, quit_delay_random)
            log.info('{} is restarting in {} seconds.'.format(
                self.nickname, quit_delay))

        self.execute_delayed(quit_delay, self.quit_bot)

    def quit_bot(self, **options):
        self.commit_all()
        if self.phrases['quit']:
            phrase_data = {
                'nickname': self.nickname,
                'version': self.version,
            }

            try:
                self.say(self.phrases['quit'].format(**phrase_data))
            except Exception:
                log.exception(
                    'Exception caught while trying to say quit phrase')

        self.twitter_manager.quit()
        self.socket_manager.quit()
        self.irc.quit()

        sys.exit(0)

    def apply_filter(self, resp, filter):
        available_filters = {
            'strftime':
            _filter_strftime,
            'lower':
            lambda var, args: var.lower(),
            'upper':
            lambda var, args: var.upper(),
            'time_since_minutes':
            lambda var, args: 'no time'
            if var == 0 else time_since(var * 60, 0, format='long'),
            'time_since':
            lambda var, args: 'no time'
            if var == 0 else time_since(var, 0, format='long'),
            'time_since_dt':
            _filter_time_since_dt,
            'urlencode':
            lambda var, args: urllib.parse.urlencode(var),
            'join':
            _filter_join,
            'number_format':
            _filter_number_format,
        }
        if filter.name in available_filters:
            return available_filters[filter.name](resp, filter.arguments)
        return resp

    def find_unique_urls(self, message):
        from pajbot.modules.linkchecker import find_unique_urls
        return find_unique_urls(self.url_regex, message)