def test_strip_irc_formatting(self):
# Some messages from http://modern.ircdocs.horse/formatting.html#examples
self.assertEqual(utils.strip_irc_formatting(
"I love \x033IRC! \x03It is the \x037best protocol ever!"),
"I love IRC! It is the best protocol ever!")
self.assertEqual(utils.strip_irc_formatting(
"This is a \x1d\x0313,9cool \x03message"),
"This is a cool message")
self.assertEqual(utils.strip_irc_formatting(
"Don't spam 5\x0313,8,6\x03,7,8, and especially not \x029\x02\x1d!"),
"Don't spam 5,6,7,8, and especially not 9!")
# Should not remove the ,
self.assertEqual(utils.strip_irc_formatting(
"\x0305,"),
",")
self.assertEqual(utils.strip_irc_formatting(
"\x038,,,,."),
",,,,.")
# Numbers are preserved
self.assertEqual(utils.strip_irc_formatting(
"\x031234 "),
"34 ")
self.assertEqual(utils.strip_irc_formatting(
"\x03\x1f12"),
"12")
self.assertEqual(utils.strip_irc_formatting(
"\x0305t\x030,1h\x0307,02e\x0308,06 \x0309,13q\x0303,15u\x0311,14i\x0310,05c\x0312,04k\x0302,07 \x0306,08b\x0313,09r\x0305,10o\x0304,12w\x0307,02n\x0308,06 \x0309,13f\x0303,15o\x0311,14x\x0310,05 \x0312,04j\x0302,07u\x0306,08m\x0313,09p\x0305,10s\x0304,12 \x0307,02o\x0308,06v\x0309,13e\x0303,15r\x0311,14 \x0310,05t\x0312,04h\x0302,07e\x0306,08 \x0313,09l\x0305,10a\x0304,12z\x0307,02y\x0308,06 \x0309,13d\x0303,15o\x0311,14g\x0f"),
"the quick brown fox jumps over the lazy dog")
def handle_textfilter(irc, source, command, args):
"""Antispam text filter handler."""
target = args['target']
text = args['text']
txf_settings = irc.get_service_option('antispam', 'textfilter',
TEXTFILTER_DEFAULTS)
if not txf_settings.get('enabled', False):
return
my_uid = sbot.uids.get(irc.name)
# XXX workaround for single-bot protocols like Clientbot
if irc.pseudoclient and not irc.has_cap('can-spawn-clients'):
my_uid = irc.pseudoclient.uid
if (not irc.connected.is_set()) or (not my_uid):
# Break if the network isn't ready.
log.debug("(%s) antispam.textfilters: skipping processing; network isn't ready", irc.name)
return
elif irc.is_internal_client(source):
# Ignore messages from our own clients.
log.debug("(%s) antispam.textfilters: skipping processing message from internal client %s", irc.name, source)
return
elif source not in irc.users:
log.debug("(%s) antispam.textfilters: ignoring message from non-user %s", irc.name, source)
return
if irc.is_channel(target):
channel_or_none = target
if target not in irc.channels or my_uid not in irc.channels[target].users:
# We're not monitoring this channel.
log.debug("(%s) antispam.textfilters: skipping processing message from channel %r we're not in", irc.name, target)
return
else:
channel_or_none = None
watch_pms = txf_settings.get('watch_pms', TEXTFILTER_DEFAULTS['watch_pms'])
if watch_pms == 'services':
if not irc.get_service_bot(target):
log.debug("(%s) antispam.textfilters: skipping processing; %r is not a service bot (watch_pms='services')", irc.name, target)
return
elif watch_pms == 'all':
log.debug("(%s) antispam.textfilters: checking all PMs (watch_pms='all')", irc.name)
pass
else:
# Not a channel.
log.debug("(%s) antispam.textfilters: skipping processing; %r is not a channel and watch_pms is disabled", irc.name, target)
return
# Merge together global and local textfilter lists.
txf_globs = set(conf.conf.get('antispam', {}).get('textfilter_globs', [])) | \
set(irc.serverdata.get('antispam_textfilter_globs', []))
punishment = txf_settings.get('punishment', TEXTFILTER_DEFAULTS['punishment']).lower()
reason = txf_settings.get('reason', TEXTFILTER_DEFAULTS['reason'])
if irc.get_service_option('antispam', 'strip_formatting', True):
text = utils.strip_irc_formatting(text)
if txf_settings.get('munge_unicode', TEXTFILTER_DEFAULTS['munge_unicode']):
text = str.translate(text, UNICODE_CHARMAP)
punished = False
for filterglob in txf_globs:
if utils.match_text(filterglob, text):
log.info("(%s) antispam: punishing %s => %s for text filter %r",
irc.name,
irc.get_friendly_name(source),
irc.get_friendly_name(target),
filterglob)
punished = _punish(irc, source, channel_or_none, punishment, reason)
break
return not punished # Filter this message from relay, etc. if it triggered protection
def handle_masshighlight(irc, source, command, args):
"""Handles mass highlight attacks."""
channel = args['target']
text = args['text']
mhl_settings = irc.get_service_option('antispam', 'masshighlight',
MASSHIGHLIGHT_DEFAULTS)
if not mhl_settings.get('enabled', False):
return
my_uid = sbot.uids.get(irc.name)
# XXX workaround for single-bot protocols like Clientbot
if irc.pseudoclient and not irc.has_cap('can-spawn-clients'):
my_uid = irc.pseudoclient.uid
if (not irc.connected.is_set()) or (not my_uid):
# Break if the network isn't ready.
log.debug("(%s) antispam.masshighlight: skipping processing; network isn't ready", irc.name)
return
elif not irc.is_channel(channel):
# Not a channel - mass highlight blocking only makes sense within channels
log.debug("(%s) antispam.masshighlight: skipping processing; %r is not a channel", irc.name, channel)
return
elif irc.is_internal_client(source):
# Ignore messages from our own clients.
log.debug("(%s) antispam.masshighlight: skipping processing message from internal client %s", irc.name, source)
return
elif source not in irc.users:
log.debug("(%s) antispam.masshighlight: ignoring message from non-user %s", irc.name, source)
return
elif channel not in irc.channels or my_uid not in irc.channels[channel].users:
# We're not monitoring this channel.
log.debug("(%s) antispam.masshighlight: skipping processing message from channel %r we're not in", irc.name, channel)
return
elif len(text) < mhl_settings.get('min_length', MASSHIGHLIGHT_DEFAULTS['min_length']):
log.debug("(%s) antispam.masshighlight: skipping processing message %r; it's too short", irc.name, text)
return
if irc.get_service_option('antispam', 'strip_formatting', True):
text = utils.strip_irc_formatting(text)
# Strip :, from potential nicks
words = [word.rstrip(':,') for word in text.split()]
userlist = [irc.users[uid].nick for uid in irc.channels[channel].users.copy()]
min_nicks = mhl_settings.get('min_nicks', MASSHIGHLIGHT_DEFAULTS['min_nicks'])
# Don't allow repeating the same nick to trigger punishment
nicks_caught = set()
punished = False
for word in words:
if word in userlist:
nicks_caught.add(word)
if len(nicks_caught) >= min_nicks:
# Get the punishment and reason.
punishment = mhl_settings.get('punishment', MASSHIGHLIGHT_DEFAULTS['punishment']).lower()
reason = mhl_settings.get('reason', MASSHIGHLIGHT_DEFAULTS['reason'])
log.info("(%s) antispam: punishing %s => %s for mass highlight spam",
irc.name,
irc.get_friendly_name(source),
channel)
punished = _punish(irc, source, channel, punishment, reason)
break
log.debug('(%s) antispam.masshighlight: got %s/%s nicks on message to %r', irc.name,
len(nicks_caught), min_nicks, channel)
return not punished # Filter this message from relay, etc. if it triggered protection
def _message_builder(self):
"""
Discord message queue handler. Also supports virtual users via webhooks.
"""
def _send(sender, channel, pylink_target, message_parts):
"""
Wrapper to send a joined message.
"""
text = '\n'.join(message_parts)
# Handle the case when the sender is not the PyLink client (sender != None)
# For channels, use either virtual webhook users or CLIENTBOT_MESSAGE forwarding (relay_clientbot).
if sender:
user_fields = self._get_webhook_fields(sender)
if channel.guild: # This message belongs to a channel
netobj = self._children[channel.guild.id]
# Note: skip webhook sending for messages that contain only spaces, as that fails with
# 50006 "Cannot send an empty message" errors
if netobj.serverdata.get('use_webhooks') and text.strip():
user_format = netobj.serverdata.get('webhook_user_format', "$nick @ $netname")
tmpl = string.Template(user_format)
webhook_fake_username = tmpl.safe_substitute(self._get_webhook_fields(sender))
try:
webhook = self._get_webhook(channel)
webhook.execute(content=text[:self.MAX_MESSAGE_SIZE], username=webhook_fake_username, avatar_url=user_fields['avatar'])
except APIException as e:
if e.code == 10015 and channel.id in self.webhooks:
log.info("(%s) Invalidating webhook %s for channel %s due to Unknown Webhook error (10015)",
self.name, self.webhooks[channel.id], channel)
del self.webhooks[channel.id]
elif e.code == 50013:
# Prevent spamming errors: disable webhooks we don't have the right permissions
log.warning("(%s) Disabling webhooks on guild %s/%s due to insufficient permissions (50013). Rehash to re-enable.",
self.name, channel.guild.id, channel.guild.name)
self.serverdata.update(
{'guilds':
{channel.guild.id:
{'use_webhooks': False}
}
})
else:
log.error("(%s) Caught API exception when sending webhook message to channel %s: %s/%s", self.name, channel, e.response.status_code, e.code)
log.debug("(%s) APIException full traceback:", self.name, exc_info=True)
except:
log.exception("(%s) Failed to send webhook message to channel %s", self.name, channel)
else:
return
for line in message_parts:
netobj.call_hooks([sender.uid, 'CLIENTBOT_MESSAGE', {'target': pylink_target, 'text': line}])
return
else:
# This is a forwarded PM - prefix the message with its sender info.
pm_format = self.serverdata.get('pm_format', "Message from $nick @ $netname: $text")
user_fields['text'] = text
text = string.Template(pm_format).safe_substitute(user_fields)
try:
channel.send_message(text[:self.MAX_MESSAGE_SIZE])
except Exception as e:
log.exception("(%s) Could not send message to channel %s (pylink_target=%s)", self.name, channel, pylink_target)
joined_messages = collections.defaultdict(collections.deque)
while not self._aborted.is_set():
try:
# message is an instance of QueuedMessage (defined in this file)
message = self.message_queue.get(timeout=BATCH_DELAY)
message.text = utils.strip_irc_formatting(message.text)
if not self.serverdata.get('allow_mention_everyone', False):
message.text = message.text.replace('@here', '@ here')
message.text = message.text.replace('@everyone', '@ everyone')
# First, buffer messages by channel
joined_messages[message.channel].append(message)
except queue.Empty: # Then process them together when we run out of things in the queue
for channel, messages in joined_messages.items():
next_message = []
length = 0
current_sender = None
# We group messages here to avoid being throttled as often. In short, we want to send a message when:
# 1) The virtual sender (for webhook purposes) changes
# 2) We reach the message limit for one batch (2000 chars)
# 3) We run out of messages at the end
while messages:
message = messages.popleft()
next_message.append(message.text)
length += len(message.text)
if message.sender != current_sender or length >= self.MAX_MESSAGE_SIZE:
current_sender = message.sender
_send(current_sender, channel, message.pylink_target, next_message)
next_message.clear()
length = 0
# The last batch
if next_message:
_send(current_sender, channel, message.pylink_target, next_message)
joined_messages.clear()
except Exception:
log.exception("Exception in message queueing thread:")
