def url_handler(bot: SopelWrapper, trigger: Trigger):
"""Checks for malicious URLs."""
mode = bot.db.get_channel_value(
trigger.sender,
"safety",
bot.settings.safety.default_mode,
)
if mode == "off":
return
local_only = "local" in mode or bot.settings.safety.vt_api_key is None
strict = "strict" in mode
for url in tools.web.search_urls(trigger):
safe_url = safeify_url(url)
positives = 0 # Number of engines saying it's malicious
total = 0 # Number of total engines
try:
hostname = urlparse(url).hostname.lower()
except ValueError:
pass # Invalid address
else:
if any(regex.search(hostname) for regex in known_good):
continue # explicitly trusted
if hostname in bot.memory[SAFETY_CACHE_LOCAL_KEY]:
LOGGER.debug("[local] domain in blocklist: %r", hostname)
positives += 1
total += 1
result = virustotal_lookup(bot, url, local_only=local_only)
if result:
positives += result["positives"]
total += result["total"]
if positives >= 1:
# Possibly malicious URL detected!
LOGGER.info(
"Possibly malicious link (%s/%s) posted in %s by %s: %r",
positives,
total,
trigger.sender,
trigger.nick,
safe_url,
)
bot.say(
"{} {} of {} engine{} flagged a link {} posted as malicious".
format(
bold(color("WARNING:", colors.RED)),
positives,
total,
"" if total == 1 else "s",
bold(trigger.nick),
))
if strict:
bot.kick(trigger.nick, trigger.sender,
"Posted a malicious link")
def setup(bot: SopelWrapper) -> None:
""" Ensures that our set up configuration items are present """
# Ensure configuration
bot.config.define_section('owm', OWMSection)
# Load our OWM API into bot memory
if 'owm' not in bot.memory:
api_key = bot.config.owm.api_key
owm_api = get_api(api_key)
bot.memory['owm'] = tools.SopelMemory()
bot.memory['owm']['api'] = owm_api
def check_callbacks(bot: SopelWrapper,
url: str,
use_excludes: bool = True) -> bool:
"""Check if ``url`` is excluded or matches any URL callback patterns.
:param bot: Sopel instance
:param url: URL to check
:param use_excludes: Use or ignore the configured exclusion lists
:return: True if ``url`` is excluded or matches any URL callback pattern
This function looks at the ``bot.memory`` for ``url_exclude`` patterns and
it returns ``True`` if any matches the given ``url``. Otherwise, it looks
at the ``bot``'s URL callback patterns, and it returns ``True`` if any
matches, ``False`` otherwise.
.. seealso::
The :func:`~sopel.modules.url.setup` function that defines the
``url_exclude`` in ``bot.memory``.
.. versionchanged:: 7.0
This function **does not** trigger URL callbacks anymore when ``url``
matches a pattern.
"""
# Check if it matches the exclusion list first
excluded = use_excludes and any(
regex.search(url) for regex in bot.memory["url_exclude"])
return (excluded or any(bot.search_url_callbacks(url))
or bot.rules.check_url_callback(bot, url))
def get_weather(bot: SopelWrapper, trigger: Trigger) -> None:
"""
Gets the weather at a given location and returns the first result
"""
location_lookup_args = trigger.group(2)
if location_lookup_args is None:
location_lookup_args = bot.db.get_nick_value(trigger.nick, 'place_id')
if not location_lookup_args:
bot.reply("I don't know where you live. "
"Give me a location, like {pfx}{command} London, "
"or tell me where you live by saying {pfx}setlocation "
"London, for example.".format(
command=trigger.group(1),
pfx=bot.config.core.help_prefix))
return plugin.NOLIMIT
api = get_owm_api(bot)
location = parse_location_args(location_lookup_args)
message = get_weather_message(api, location)
bot.reply(message)
def toggle_safety(bot: SopelWrapper, trigger: Trigger):
"""Set safety setting for channel."""
if not trigger.admin and bot.channels[trigger.sender].privileges[
trigger.nick] < plugin.OP:
bot.reply('Only channel operators can change safety settings')
return
new_mode = None
if trigger.group(2):
new_mode = trigger.group(2).lower()
if not new_mode or (new_mode != "default"
and new_mode not in SAFETY_MODES):
bot.reply(
"Current mode: {}. Available modes: {}, or default ({})".format(
bot.db.get_channel_value(
trigger.sender,
"safety",
"default",
),
", ".join(SAFETY_MODES),
bot.settings.safety.default_mode,
))
return
if new_mode == "default":
bot.db.delete_channel_value(trigger.sender, "safety")
else:
bot.db.set_channel_value(trigger.sender, "safety", new_mode)
bot.say('Safety is now set to "%s" for this channel' % new_mode)
def title_auto(bot: SopelWrapper, trigger: Trigger):
"""
Automatically show titles for URLs. For shortened URLs/redirects, find
where the URL redirects to and show the title for that (or call a function
from another plugin to give more information).
"""
# Enabled or disabled by feature flag
if not bot.settings.url.enable_auto_title:
return
# Avoid fetching links from another command
if re.match(bot.config.core.prefix + r'\S+', trigger):
return
unchecked_urls = web.search_urls(
trigger, exclusion_char=bot.config.url.exclusion_char, clean=True)
urls = []
safety_cache = bot.memory.get("safety_cache", {})
safety_cache_local = bot.memory.get("safety_cache_local", {})
for url in unchecked_urls:
# Avoid fetching known malicious links
if url in safety_cache and safety_cache[url]["positives"] > 0:
continue
if urlparse(url).hostname.lower() in safety_cache_local:
continue
urls.append(url)
for url, title, domain, tinyurl, dispatched in process_urls(
bot, trigger, urls):
if not dispatched:
message = '%s | %s' % (title, domain)
if tinyurl:
message += ' ( %s )' % tinyurl
# Guard against responding to other instances of this bot.
if message != trigger:
bot.say(message)
bot.memory["last_seen_url"][trigger.sender] = url
def url_unban(bot: SopelWrapper, trigger: Trigger):
"""Allow a URL for auto title.
Use ``urlpallow`` to allow a pattern instead of a URL.
"""
url = trigger.group(2)
if not url:
bot.reply('This command requires a URL to allow.')
return
if trigger.group(1) in ['urlpallow', 'urlpunban']:
# validate regex pattern
try:
re.compile(url)
except re.error as err:
bot.reply('Invalid regex pattern: %s' % err)
return
else:
# escape the URL to ensure a valid pattern
url = re.escape(url)
patterns = bot.settings.url.exclude
if url not in patterns:
bot.reply('This URL was not excluded from auto title.')
return
# update settings
patterns.remove(url)
bot.settings.url.exclude = patterns # set the config option
bot.settings.save()
LOGGER.info('%s allowed the URL pattern "%s"', trigger.nick, url)
# re-compile
bot.memory['url_exclude'] = [re.compile(s) for s in patterns]
# tell the user
bot.reply('This URL is not excluded from auto title anymore.')
def get_or_create_shorturl(bot: SopelWrapper, url: str) -> Optional[str]:
"""Get or create a short URL for ``url``
:param bot: Sopel instance
:param url: URL to get or create a short URL for
:return: A short URL
It gets the short URL for ``url`` from the bot's memory if it exists.
Otherwise, it creates a short URL (see :func:`get_tinyurl`), stores it
into the bot's memory, then returns it.
"""
# Check bot memory to see if the shortened URL is already in
# memory
if url in bot.memory['shortened_urls']:
return bot.memory['shortened_urls'][url]
tinyurl = get_tinyurl(url)
bot.memory['shortened_urls'][url] = tinyurl
return tinyurl
def vt_command(bot: SopelWrapper, trigger: Trigger):
"""Look up VT results on demand."""
if not bot.settings.safety.vt_api_key:
bot.reply("Sorry, I don't have a VirusTotal API key configured.")
return
url = trigger.group(2)
safe_url = safeify_url(url)
result = virustotal_lookup(bot, url, max_cache_age=timedelta(minutes=1))
if not result:
bot.reply("Sorry, an error occurred while looking that up.")
return
analysis = result["virustotal_data"]["last_analysis_stats"]
result_types = {
"malicious": colors.RED,
"suspicious": colors.YELLOW,
"harmless": colors.GREEN,
"undetected": colors.GREY,
}
result_strs = []
for result_type, result_color in result_types.items():
if analysis[result_type] == 0:
result_strs.append("0 " + result_type)
else:
result_strs.append(
bold(
color(
str(analysis[result_type]) + " " + result_type,
result_color)))
results_str = ", ".join(result_strs)
vt_scan_time = datetime.fromtimestamp(
result["virustotal_data"]["last_analysis_date"],
timezone.utc,
)
bot.reply("Results: {} at {} for {}".format(
results_str,
tools.time.format_time(
bot.db,
bot.config,
nick=trigger.nick,
channel=trigger.sender,
time=vt_scan_time,
),
safe_url,
))
def title_command(bot: SopelWrapper, trigger: Trigger):
"""
Show the title or URL information for the given URL, or the last URL seen
in this channel.
"""
result_count = 0
if not trigger.group(2):
if trigger.sender not in bot.memory['last_seen_url']:
return
urls = [bot.memory["last_seen_url"][trigger.sender]]
else:
# needs to be a list so len() can be checked later
urls = list(web.search_urls(trigger))
for url, title, domain, tinyurl, dispatched in process_urls(
bot, trigger, urls, requested=True):
if dispatched:
result_count += 1
continue
message = "%s | %s" % (title, domain)
if tinyurl:
message += ' ( %s )' % tinyurl
bot.reply(message)
bot.memory['last_seen_url'][trigger.sender] = url
result_count += 1
expected_count = len(urls)
if result_count < expected_count:
if expected_count == 1:
bot.reply(
"Sorry, fetching that title failed. Make sure the site is working."
)
elif result_count == 0:
bot.reply("Sorry, I couldn't fetch titles for any of those.")
else:
bot.reply(
"I couldn't get all of the titles, but I fetched what I could!"
)
def test(configfactory, botfactory, ircfactory):
test_config = TEST_CONFIG.format(
name='NickName',
admin=admin,
owner=owner,
)
settings = configfactory('default.cfg', test_config)
bot = botfactory(settings)
server = ircfactory(bot)
server.channel_joined('#Sopel')
match = None
if hasattr(tested_func, "commands"):
for command in tested_func.commands:
regexp = sopel.tools.get_command_regexp(".", command)
match = regexp.match(msg)
if match:
break
assert match, "Example did not match any command."
sender = bot.nick if privmsg else "#channel"
hostmask = "%s!%s@%s" % (bot.nick, "UserName", "example.com")
# TODO enable message tags
full_message = ':{} PRIVMSG {} :{}'.format(hostmask, sender, msg)
pretrigger = sopel.trigger.PreTrigger(bot.nick, full_message)
trigger = sopel.trigger.Trigger(bot.settings, pretrigger, match)
pattern = re.compile(r'^%s: ' % re.escape(bot.nick))
# setup module
module = sys.modules[tested_func.__module__]
if hasattr(module, 'setup'):
module.setup(bot)
def isnt_ignored(value):
"""Return True if value doesn't match any re in ignore list."""
return not any(
re.match(ignored_line, value)
for ignored_line in ignore)
expected_output_count = 0
for _i in range(repeat):
expected_output_count += len(results)
wrapper = SopelWrapper(bot, trigger)
tested_func(wrapper, trigger)
output_triggers = (
sopel.trigger.PreTrigger(bot.nick, message.decode('utf-8'))
for message in wrapper.backend.message_sent
)
output_texts = (
# subtract "Sopel: " when necessary
pattern.sub('', output_trigger.args[-1])
for output_trigger in output_triggers
)
outputs = [text for text in output_texts if isnt_ignored(text)]
# output length
assert len(outputs) == expected_output_count
# output content
for expected, output in zip(results, outputs):
if use_regexp:
if not re.match(expected, output):
assert expected == output
else:
assert expected == output
def virustotal_lookup(
bot: SopelWrapper,
url: str,
local_only: bool = False,
max_cache_age: Optional[timedelta] = None,
) -> Optional[Dict]:
"""Check VirusTotal for flags on a URL as malicious.
:param url: The URL to look up
:param local_only: If set, only check cache, do not make a new request
:param max_cache_age: If set, don't use cache older than this value
:returns: A dict containing information about findings, or None if not found
"""
if url.startswith("hxxp"):
url = "htt" + url[3:]
elif not url.startswith("http"):
# VT only does http/https URLs
return None
safe_url = safeify_url(url)
# default: use any cache available
oldest_cache = datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc)
if max_cache_age is not None:
oldest_cache = datetime.now(timezone.utc) - max_cache_age
cache = bot.memory[SAFETY_CACHE_KEY]
if url in cache and cache[url]["fetched"] > oldest_cache:
LOGGER.debug("[VirusTotal] Using cached data for %r", safe_url)
return bot.memory[SAFETY_CACHE_KEY].get(url)
if local_only:
return None
LOGGER.debug("[VirusTotal] Looking up %r", safe_url)
url_id = urlsafe_b64encode(
url.encode("utf-8")).rstrip(b"=").decode("ascii")
attempts = 5
requested = False
while attempts > 0:
attempts -= 1
try:
r = requests.get(
VT_API_URL + "/" + url_id,
headers={"x-apikey": bot.settings.safety.vt_api_key},
)
if r.status_code == 200:
vt_data = r.json()
last_analysis = vt_data["data"]["attributes"][
"last_analysis_stats"]
# VT returns 200s for recent submissions before scan results are in...
if not requested or sum(last_analysis.values()) > 0:
break
elif not requested and r.status_code == 404:
# Not analyzed - submit new
LOGGER.debug("[VirusTotal] No scan for %r, requesting",
safe_url)
requests.post(
VT_API_URL,
data={"url": url},
headers={"x-apikey": bot.settings.safety.vt_api_key},
)
requested = True
sleep(2) # Scans seem to take ~5s minimum, so add 2s
except requests.exceptions.RequestException:
# Ignoring exceptions with VT so domain list will always work
LOGGER.debug("[VirusTotal] Error obtaining response for %r",
safe_url,
exc_info=True)
return None
except json.JSONDecodeError:
# Ignoring exceptions with VT so domain list will always work
LOGGER.debug(
"[VirusTotal] Malformed response (invalid JSON) for %r",
safe_url,
exc_info=True,
)
return None
sleep(3)
else: # Still no results
LOGGER.debug("[VirusTotal] Scan failed or unfinished for %r", safe_url)
return None
fetched = datetime.now(timezone.utc)
# Only count strong opinions (ignore suspicious/timeout/undetected)
result = {
"positives": last_analysis["malicious"],
"total": last_analysis["malicious"] + last_analysis["harmless"],
"fetched": fetched,
"virustotal_data": vt_data["data"]["attributes"],
}
bot.memory[SAFETY_CACHE_KEY][url] = result
if len(bot.memory[SAFETY_CACHE_KEY]) >= (2 * CACHE_LIMIT):
_clean_cache(bot)
return result
