def test_strip_irc_formatting(self): # Some messages from http://modern.ircdocs.horse/formatting.html#examples self.assertEqual(utils.strip_irc_formatting( "I love \x033IRC! \x03It is the \x037best protocol ever!"), "I love IRC! It is the best protocol ever!") self.assertEqual(utils.strip_irc_formatting( "This is a \x1d\x0313,9cool \x03message"), "This is a cool message") self.assertEqual(utils.strip_irc_formatting( "Don't spam 5\x0313,8,6\x03,7,8, and especially not \x029\x02\x1d!"), "Don't spam 5,6,7,8, and especially not 9!") # Should not remove the , self.assertEqual(utils.strip_irc_formatting( "\x0305,"), ",") self.assertEqual(utils.strip_irc_formatting( "\x038,,,,."), ",,,,.") # Numbers are preserved self.assertEqual(utils.strip_irc_formatting( "\x031234 "), "34 ") self.assertEqual(utils.strip_irc_formatting( "\x03\x1f12"), "12") self.assertEqual(utils.strip_irc_formatting( "\x0305t\x030,1h\x0307,02e\x0308,06 \x0309,13q\x0303,15u\x0311,14i\x0310,05c\x0312,04k\x0302,07 \x0306,08b\x0313,09r\x0305,10o\x0304,12w\x0307,02n\x0308,06 \x0309,13f\x0303,15o\x0311,14x\x0310,05 \x0312,04j\x0302,07u\x0306,08m\x0313,09p\x0305,10s\x0304,12 \x0307,02o\x0308,06v\x0309,13e\x0303,15r\x0311,14 \x0310,05t\x0312,04h\x0302,07e\x0306,08 \x0313,09l\x0305,10a\x0304,12z\x0307,02y\x0308,06 \x0309,13d\x0303,15o\x0311,14g\x0f"), "the quick brown fox jumps over the lazy dog")
def handle_textfilter(irc, source, command, args): """Antispam text filter handler.""" target = args['target'] text = args['text'] txf_settings = irc.get_service_option('antispam', 'textfilter', TEXTFILTER_DEFAULTS) if not txf_settings.get('enabled', False): return my_uid = sbot.uids.get(irc.name) # XXX workaround for single-bot protocols like Clientbot if irc.pseudoclient and not irc.has_cap('can-spawn-clients'): my_uid = irc.pseudoclient.uid if (not irc.connected.is_set()) or (not my_uid): # Break if the network isn't ready. log.debug("(%s) antispam.textfilters: skipping processing; network isn't ready", irc.name) return elif irc.is_internal_client(source): # Ignore messages from our own clients. log.debug("(%s) antispam.textfilters: skipping processing message from internal client %s", irc.name, source) return elif source not in irc.users: log.debug("(%s) antispam.textfilters: ignoring message from non-user %s", irc.name, source) return if irc.is_channel(target): channel_or_none = target if target not in irc.channels or my_uid not in irc.channels[target].users: # We're not monitoring this channel. log.debug("(%s) antispam.textfilters: skipping processing message from channel %r we're not in", irc.name, target) return else: channel_or_none = None watch_pms = txf_settings.get('watch_pms', TEXTFILTER_DEFAULTS['watch_pms']) if watch_pms == 'services': if not irc.get_service_bot(target): log.debug("(%s) antispam.textfilters: skipping processing; %r is not a service bot (watch_pms='services')", irc.name, target) return elif watch_pms == 'all': log.debug("(%s) antispam.textfilters: checking all PMs (watch_pms='all')", irc.name) pass else: # Not a channel. log.debug("(%s) antispam.textfilters: skipping processing; %r is not a channel and watch_pms is disabled", irc.name, target) return # Merge together global and local textfilter lists. txf_globs = set(conf.conf.get('antispam', {}).get('textfilter_globs', [])) | \ set(irc.serverdata.get('antispam_textfilter_globs', [])) punishment = txf_settings.get('punishment', TEXTFILTER_DEFAULTS['punishment']).lower() reason = txf_settings.get('reason', TEXTFILTER_DEFAULTS['reason']) if irc.get_service_option('antispam', 'strip_formatting', True): text = utils.strip_irc_formatting(text) if txf_settings.get('munge_unicode', TEXTFILTER_DEFAULTS['munge_unicode']): text = str.translate(text, UNICODE_CHARMAP) punished = False for filterglob in txf_globs: if utils.match_text(filterglob, text): log.info("(%s) antispam: punishing %s => %s for text filter %r", irc.name, irc.get_friendly_name(source), irc.get_friendly_name(target), filterglob) punished = _punish(irc, source, channel_or_none, punishment, reason) break return not punished # Filter this message from relay, etc. if it triggered protection
def handle_masshighlight(irc, source, command, args): """Handles mass highlight attacks.""" channel = args['target'] text = args['text'] mhl_settings = irc.get_service_option('antispam', 'masshighlight', MASSHIGHLIGHT_DEFAULTS) if not mhl_settings.get('enabled', False): return my_uid = sbot.uids.get(irc.name) # XXX workaround for single-bot protocols like Clientbot if irc.pseudoclient and not irc.has_cap('can-spawn-clients'): my_uid = irc.pseudoclient.uid if (not irc.connected.is_set()) or (not my_uid): # Break if the network isn't ready. log.debug("(%s) antispam.masshighlight: skipping processing; network isn't ready", irc.name) return elif not irc.is_channel(channel): # Not a channel - mass highlight blocking only makes sense within channels log.debug("(%s) antispam.masshighlight: skipping processing; %r is not a channel", irc.name, channel) return elif irc.is_internal_client(source): # Ignore messages from our own clients. log.debug("(%s) antispam.masshighlight: skipping processing message from internal client %s", irc.name, source) return elif source not in irc.users: log.debug("(%s) antispam.masshighlight: ignoring message from non-user %s", irc.name, source) return elif channel not in irc.channels or my_uid not in irc.channels[channel].users: # We're not monitoring this channel. log.debug("(%s) antispam.masshighlight: skipping processing message from channel %r we're not in", irc.name, channel) return elif len(text) < mhl_settings.get('min_length', MASSHIGHLIGHT_DEFAULTS['min_length']): log.debug("(%s) antispam.masshighlight: skipping processing message %r; it's too short", irc.name, text) return if irc.get_service_option('antispam', 'strip_formatting', True): text = utils.strip_irc_formatting(text) # Strip :, from potential nicks words = [word.rstrip(':,') for word in text.split()] userlist = [irc.users[uid].nick for uid in irc.channels[channel].users.copy()] min_nicks = mhl_settings.get('min_nicks', MASSHIGHLIGHT_DEFAULTS['min_nicks']) # Don't allow repeating the same nick to trigger punishment nicks_caught = set() punished = False for word in words: if word in userlist: nicks_caught.add(word) if len(nicks_caught) >= min_nicks: # Get the punishment and reason. punishment = mhl_settings.get('punishment', MASSHIGHLIGHT_DEFAULTS['punishment']).lower() reason = mhl_settings.get('reason', MASSHIGHLIGHT_DEFAULTS['reason']) log.info("(%s) antispam: punishing %s => %s for mass highlight spam", irc.name, irc.get_friendly_name(source), channel) punished = _punish(irc, source, channel, punishment, reason) break log.debug('(%s) antispam.masshighlight: got %s/%s nicks on message to %r', irc.name, len(nicks_caught), min_nicks, channel) return not punished # Filter this message from relay, etc. if it triggered protection
def _message_builder(self): """ Discord message queue handler. Also supports virtual users via webhooks. """ def _send(sender, channel, pylink_target, message_parts): """ Wrapper to send a joined message. """ text = '\n'.join(message_parts) # Handle the case when the sender is not the PyLink client (sender != None) # For channels, use either virtual webhook users or CLIENTBOT_MESSAGE forwarding (relay_clientbot). if sender: user_fields = self._get_webhook_fields(sender) if channel.guild: # This message belongs to a channel netobj = self._children[channel.guild.id] # Note: skip webhook sending for messages that contain only spaces, as that fails with # 50006 "Cannot send an empty message" errors if netobj.serverdata.get('use_webhooks') and text.strip(): user_format = netobj.serverdata.get('webhook_user_format', "$nick @ $netname") tmpl = string.Template(user_format) webhook_fake_username = tmpl.safe_substitute(self._get_webhook_fields(sender)) try: webhook = self._get_webhook(channel) webhook.execute(content=text[:self.MAX_MESSAGE_SIZE], username=webhook_fake_username, avatar_url=user_fields['avatar']) except APIException as e: if e.code == 10015 and channel.id in self.webhooks: log.info("(%s) Invalidating webhook %s for channel %s due to Unknown Webhook error (10015)", self.name, self.webhooks[channel.id], channel) del self.webhooks[channel.id] elif e.code == 50013: # Prevent spamming errors: disable webhooks we don't have the right permissions log.warning("(%s) Disabling webhooks on guild %s/%s due to insufficient permissions (50013). Rehash to re-enable.", self.name, channel.guild.id, channel.guild.name) self.serverdata.update( {'guilds': {channel.guild.id: {'use_webhooks': False} } }) else: log.error("(%s) Caught API exception when sending webhook message to channel %s: %s/%s", self.name, channel, e.response.status_code, e.code) log.debug("(%s) APIException full traceback:", self.name, exc_info=True) except: log.exception("(%s) Failed to send webhook message to channel %s", self.name, channel) else: return for line in message_parts: netobj.call_hooks([sender.uid, 'CLIENTBOT_MESSAGE', {'target': pylink_target, 'text': line}]) return else: # This is a forwarded PM - prefix the message with its sender info. pm_format = self.serverdata.get('pm_format', "Message from $nick @ $netname: $text") user_fields['text'] = text text = string.Template(pm_format).safe_substitute(user_fields) try: channel.send_message(text[:self.MAX_MESSAGE_SIZE]) except Exception as e: log.exception("(%s) Could not send message to channel %s (pylink_target=%s)", self.name, channel, pylink_target) joined_messages = collections.defaultdict(collections.deque) while not self._aborted.is_set(): try: # message is an instance of QueuedMessage (defined in this file) message = self.message_queue.get(timeout=BATCH_DELAY) message.text = utils.strip_irc_formatting(message.text) if not self.serverdata.get('allow_mention_everyone', False): message.text = message.text.replace('@here', '@ here') message.text = message.text.replace('@everyone', '@ everyone') # First, buffer messages by channel joined_messages[message.channel].append(message) except queue.Empty: # Then process them together when we run out of things in the queue for channel, messages in joined_messages.items(): next_message = [] length = 0 current_sender = None # We group messages here to avoid being throttled as often. In short, we want to send a message when: # 1) The virtual sender (for webhook purposes) changes # 2) We reach the message limit for one batch (2000 chars) # 3) We run out of messages at the end while messages: message = messages.popleft() next_message.append(message.text) length += len(message.text) if message.sender != current_sender or length >= self.MAX_MESSAGE_SIZE: current_sender = message.sender _send(current_sender, channel, message.pylink_target, next_message) next_message.clear() length = 0 # The last batch if next_message: _send(current_sender, channel, message.pylink_target, next_message) joined_messages.clear() except Exception: log.exception("Exception in message queueing thread:")