def url_handler(bot: SopelWrapper, trigger: Trigger): """Checks for malicious URLs.""" mode = bot.db.get_channel_value( trigger.sender, "safety", bot.settings.safety.default_mode, ) if mode == "off": return local_only = "local" in mode or bot.settings.safety.vt_api_key is None strict = "strict" in mode for url in tools.web.search_urls(trigger): safe_url = safeify_url(url) positives = 0 # Number of engines saying it's malicious total = 0 # Number of total engines try: hostname = urlparse(url).hostname.lower() except ValueError: pass # Invalid address else: if any(regex.search(hostname) for regex in known_good): continue # explicitly trusted if hostname in bot.memory[SAFETY_CACHE_LOCAL_KEY]: LOGGER.debug("[local] domain in blocklist: %r", hostname) positives += 1 total += 1 result = virustotal_lookup(bot, url, local_only=local_only) if result: positives += result["positives"] total += result["total"] if positives >= 1: # Possibly malicious URL detected! LOGGER.info( "Possibly malicious link (%s/%s) posted in %s by %s: %r", positives, total, trigger.sender, trigger.nick, safe_url, ) bot.say( "{} {} of {} engine{} flagged a link {} posted as malicious". format( bold(color("WARNING:", colors.RED)), positives, total, "" if total == 1 else "s", bold(trigger.nick), )) if strict: bot.kick(trigger.nick, trigger.sender, "Posted a malicious link")
def setup(bot: SopelWrapper) -> None: """ Ensures that our set up configuration items are present """ # Ensure configuration bot.config.define_section('owm', OWMSection) # Load our OWM API into bot memory if 'owm' not in bot.memory: api_key = bot.config.owm.api_key owm_api = get_api(api_key) bot.memory['owm'] = tools.SopelMemory() bot.memory['owm']['api'] = owm_api
def check_callbacks(bot: SopelWrapper, url: str, use_excludes: bool = True) -> bool: """Check if ``url`` is excluded or matches any URL callback patterns. :param bot: Sopel instance :param url: URL to check :param use_excludes: Use or ignore the configured exclusion lists :return: True if ``url`` is excluded or matches any URL callback pattern This function looks at the ``bot.memory`` for ``url_exclude`` patterns and it returns ``True`` if any matches the given ``url``. Otherwise, it looks at the ``bot``'s URL callback patterns, and it returns ``True`` if any matches, ``False`` otherwise. .. seealso:: The :func:`~sopel.modules.url.setup` function that defines the ``url_exclude`` in ``bot.memory``. .. versionchanged:: 7.0 This function **does not** trigger URL callbacks anymore when ``url`` matches a pattern. """ # Check if it matches the exclusion list first excluded = use_excludes and any( regex.search(url) for regex in bot.memory["url_exclude"]) return (excluded or any(bot.search_url_callbacks(url)) or bot.rules.check_url_callback(bot, url))
def get_weather(bot: SopelWrapper, trigger: Trigger) -> None: """ Gets the weather at a given location and returns the first result """ location_lookup_args = trigger.group(2) if location_lookup_args is None: location_lookup_args = bot.db.get_nick_value(trigger.nick, 'place_id') if not location_lookup_args: bot.reply("I don't know where you live. " "Give me a location, like {pfx}{command} London, " "or tell me where you live by saying {pfx}setlocation " "London, for example.".format( command=trigger.group(1), pfx=bot.config.core.help_prefix)) return plugin.NOLIMIT api = get_owm_api(bot) location = parse_location_args(location_lookup_args) message = get_weather_message(api, location) bot.reply(message)
def toggle_safety(bot: SopelWrapper, trigger: Trigger): """Set safety setting for channel.""" if not trigger.admin and bot.channels[trigger.sender].privileges[ trigger.nick] < plugin.OP: bot.reply('Only channel operators can change safety settings') return new_mode = None if trigger.group(2): new_mode = trigger.group(2).lower() if not new_mode or (new_mode != "default" and new_mode not in SAFETY_MODES): bot.reply( "Current mode: {}. Available modes: {}, or default ({})".format( bot.db.get_channel_value( trigger.sender, "safety", "default", ), ", ".join(SAFETY_MODES), bot.settings.safety.default_mode, )) return if new_mode == "default": bot.db.delete_channel_value(trigger.sender, "safety") else: bot.db.set_channel_value(trigger.sender, "safety", new_mode) bot.say('Safety is now set to "%s" for this channel' % new_mode)
def title_auto(bot: SopelWrapper, trigger: Trigger): """ Automatically show titles for URLs. For shortened URLs/redirects, find where the URL redirects to and show the title for that (or call a function from another plugin to give more information). """ # Enabled or disabled by feature flag if not bot.settings.url.enable_auto_title: return # Avoid fetching links from another command if re.match(bot.config.core.prefix + r'\S+', trigger): return unchecked_urls = web.search_urls( trigger, exclusion_char=bot.config.url.exclusion_char, clean=True) urls = [] safety_cache = bot.memory.get("safety_cache", {}) safety_cache_local = bot.memory.get("safety_cache_local", {}) for url in unchecked_urls: # Avoid fetching known malicious links if url in safety_cache and safety_cache[url]["positives"] > 0: continue if urlparse(url).hostname.lower() in safety_cache_local: continue urls.append(url) for url, title, domain, tinyurl, dispatched in process_urls( bot, trigger, urls): if not dispatched: message = '%s | %s' % (title, domain) if tinyurl: message += ' ( %s )' % tinyurl # Guard against responding to other instances of this bot. if message != trigger: bot.say(message) bot.memory["last_seen_url"][trigger.sender] = url
def url_unban(bot: SopelWrapper, trigger: Trigger): """Allow a URL for auto title. Use ``urlpallow`` to allow a pattern instead of a URL. """ url = trigger.group(2) if not url: bot.reply('This command requires a URL to allow.') return if trigger.group(1) in ['urlpallow', 'urlpunban']: # validate regex pattern try: re.compile(url) except re.error as err: bot.reply('Invalid regex pattern: %s' % err) return else: # escape the URL to ensure a valid pattern url = re.escape(url) patterns = bot.settings.url.exclude if url not in patterns: bot.reply('This URL was not excluded from auto title.') return # update settings patterns.remove(url) bot.settings.url.exclude = patterns # set the config option bot.settings.save() LOGGER.info('%s allowed the URL pattern "%s"', trigger.nick, url) # re-compile bot.memory['url_exclude'] = [re.compile(s) for s in patterns] # tell the user bot.reply('This URL is not excluded from auto title anymore.')
def get_or_create_shorturl(bot: SopelWrapper, url: str) -> Optional[str]: """Get or create a short URL for ``url`` :param bot: Sopel instance :param url: URL to get or create a short URL for :return: A short URL It gets the short URL for ``url`` from the bot's memory if it exists. Otherwise, it creates a short URL (see :func:`get_tinyurl`), stores it into the bot's memory, then returns it. """ # Check bot memory to see if the shortened URL is already in # memory if url in bot.memory['shortened_urls']: return bot.memory['shortened_urls'][url] tinyurl = get_tinyurl(url) bot.memory['shortened_urls'][url] = tinyurl return tinyurl
def vt_command(bot: SopelWrapper, trigger: Trigger): """Look up VT results on demand.""" if not bot.settings.safety.vt_api_key: bot.reply("Sorry, I don't have a VirusTotal API key configured.") return url = trigger.group(2) safe_url = safeify_url(url) result = virustotal_lookup(bot, url, max_cache_age=timedelta(minutes=1)) if not result: bot.reply("Sorry, an error occurred while looking that up.") return analysis = result["virustotal_data"]["last_analysis_stats"] result_types = { "malicious": colors.RED, "suspicious": colors.YELLOW, "harmless": colors.GREEN, "undetected": colors.GREY, } result_strs = [] for result_type, result_color in result_types.items(): if analysis[result_type] == 0: result_strs.append("0 " + result_type) else: result_strs.append( bold( color( str(analysis[result_type]) + " " + result_type, result_color))) results_str = ", ".join(result_strs) vt_scan_time = datetime.fromtimestamp( result["virustotal_data"]["last_analysis_date"], timezone.utc, ) bot.reply("Results: {} at {} for {}".format( results_str, tools.time.format_time( bot.db, bot.config, nick=trigger.nick, channel=trigger.sender, time=vt_scan_time, ), safe_url, ))
def title_command(bot: SopelWrapper, trigger: Trigger): """ Show the title or URL information for the given URL, or the last URL seen in this channel. """ result_count = 0 if not trigger.group(2): if trigger.sender not in bot.memory['last_seen_url']: return urls = [bot.memory["last_seen_url"][trigger.sender]] else: # needs to be a list so len() can be checked later urls = list(web.search_urls(trigger)) for url, title, domain, tinyurl, dispatched in process_urls( bot, trigger, urls, requested=True): if dispatched: result_count += 1 continue message = "%s | %s" % (title, domain) if tinyurl: message += ' ( %s )' % tinyurl bot.reply(message) bot.memory['last_seen_url'][trigger.sender] = url result_count += 1 expected_count = len(urls) if result_count < expected_count: if expected_count == 1: bot.reply( "Sorry, fetching that title failed. Make sure the site is working." ) elif result_count == 0: bot.reply("Sorry, I couldn't fetch titles for any of those.") else: bot.reply( "I couldn't get all of the titles, but I fetched what I could!" )
def test(configfactory, botfactory, ircfactory): test_config = TEST_CONFIG.format( name='NickName', admin=admin, owner=owner, ) settings = configfactory('default.cfg', test_config) bot = botfactory(settings) server = ircfactory(bot) server.channel_joined('#Sopel') match = None if hasattr(tested_func, "commands"): for command in tested_func.commands: regexp = sopel.tools.get_command_regexp(".", command) match = regexp.match(msg) if match: break assert match, "Example did not match any command." sender = bot.nick if privmsg else "#channel" hostmask = "%s!%s@%s" % (bot.nick, "UserName", "example.com") # TODO enable message tags full_message = ':{} PRIVMSG {} :{}'.format(hostmask, sender, msg) pretrigger = sopel.trigger.PreTrigger(bot.nick, full_message) trigger = sopel.trigger.Trigger(bot.settings, pretrigger, match) pattern = re.compile(r'^%s: ' % re.escape(bot.nick)) # setup module module = sys.modules[tested_func.__module__] if hasattr(module, 'setup'): module.setup(bot) def isnt_ignored(value): """Return True if value doesn't match any re in ignore list.""" return not any( re.match(ignored_line, value) for ignored_line in ignore) expected_output_count = 0 for _i in range(repeat): expected_output_count += len(results) wrapper = SopelWrapper(bot, trigger) tested_func(wrapper, trigger) output_triggers = ( sopel.trigger.PreTrigger(bot.nick, message.decode('utf-8')) for message in wrapper.backend.message_sent ) output_texts = ( # subtract "Sopel: " when necessary pattern.sub('', output_trigger.args[-1]) for output_trigger in output_triggers ) outputs = [text for text in output_texts if isnt_ignored(text)] # output length assert len(outputs) == expected_output_count # output content for expected, output in zip(results, outputs): if use_regexp: if not re.match(expected, output): assert expected == output else: assert expected == output
def virustotal_lookup( bot: SopelWrapper, url: str, local_only: bool = False, max_cache_age: Optional[timedelta] = None, ) -> Optional[Dict]: """Check VirusTotal for flags on a URL as malicious. :param url: The URL to look up :param local_only: If set, only check cache, do not make a new request :param max_cache_age: If set, don't use cache older than this value :returns: A dict containing information about findings, or None if not found """ if url.startswith("hxxp"): url = "htt" + url[3:] elif not url.startswith("http"): # VT only does http/https URLs return None safe_url = safeify_url(url) # default: use any cache available oldest_cache = datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc) if max_cache_age is not None: oldest_cache = datetime.now(timezone.utc) - max_cache_age cache = bot.memory[SAFETY_CACHE_KEY] if url in cache and cache[url]["fetched"] > oldest_cache: LOGGER.debug("[VirusTotal] Using cached data for %r", safe_url) return bot.memory[SAFETY_CACHE_KEY].get(url) if local_only: return None LOGGER.debug("[VirusTotal] Looking up %r", safe_url) url_id = urlsafe_b64encode( url.encode("utf-8")).rstrip(b"=").decode("ascii") attempts = 5 requested = False while attempts > 0: attempts -= 1 try: r = requests.get( VT_API_URL + "/" + url_id, headers={"x-apikey": bot.settings.safety.vt_api_key}, ) if r.status_code == 200: vt_data = r.json() last_analysis = vt_data["data"]["attributes"][ "last_analysis_stats"] # VT returns 200s for recent submissions before scan results are in... if not requested or sum(last_analysis.values()) > 0: break elif not requested and r.status_code == 404: # Not analyzed - submit new LOGGER.debug("[VirusTotal] No scan for %r, requesting", safe_url) requests.post( VT_API_URL, data={"url": url}, headers={"x-apikey": bot.settings.safety.vt_api_key}, ) requested = True sleep(2) # Scans seem to take ~5s minimum, so add 2s except requests.exceptions.RequestException: # Ignoring exceptions with VT so domain list will always work LOGGER.debug("[VirusTotal] Error obtaining response for %r", safe_url, exc_info=True) return None except json.JSONDecodeError: # Ignoring exceptions with VT so domain list will always work LOGGER.debug( "[VirusTotal] Malformed response (invalid JSON) for %r", safe_url, exc_info=True, ) return None sleep(3) else: # Still no results LOGGER.debug("[VirusTotal] Scan failed or unfinished for %r", safe_url) return None fetched = datetime.now(timezone.utc) # Only count strong opinions (ignore suspicious/timeout/undetected) result = { "positives": last_analysis["malicious"], "total": last_analysis["malicious"] + last_analysis["harmless"], "fetched": fetched, "virustotal_data": vt_data["data"]["attributes"], } bot.memory[SAFETY_CACHE_KEY][url] = result if len(bot.memory[SAFETY_CACHE_KEY]) >= (2 * CACHE_LIMIT): _clean_cache(bot) return result