def process_urls(bot, trigger, urls): """ For each URL in the list, ensure that it isn't handled by another module. If not, find where it redirects to, if anywhere. If that redirected URL should be handled by another module, dispatch the callback for it. Return a list of (title, hostname) tuples for each URL which is not handled by another module. """ results = [] for url in urls: if not url.startswith(bot.config.url.exclusion_char): # Magic stuff to account for international domain names try: url = web.iri_to_uri(url) except: pass # First, check that the URL we got doesn't match matched = check_callbacks(bot, trigger, url, False) if matched: continue # Then see if it redirects anywhere new_url = follow_redirects(url) if not new_url: continue # Then see if the final URL matches anything matched = check_callbacks(bot, trigger, new_url, new_url != url) if matched: continue # Finally, actually show the URL title = find_title(url) if title: results.append((title, get_hostname(url))) return results
def process_urls(bot, trigger, urls): """ For each URL in the list, ensure that it isn't handled by another module. If not, find where it redirects to, if anywhere. If that redirected URL should be handled by another module, dispatch the callback for it. Return a list of (title, hostname) tuples for each URL which is not handled by another module. """ results = [] for url in urls: if not url.startswith(bot.config.url.exclusion_char): # Magic stuff to account for international domain names try: url = web.iri_to_uri(url) except: pass # First, check that the URL we got doesn't match matched = check_callbacks(bot, trigger, url, False) if matched: continue # Finally, actually show the URL title = find_title(url) if title: results.append((title, get_hostname(url))) return results
def process_urls(bot, trigger, urls): """ For each URL in the list, ensure that it isn't handled by another module. If not, find where it redirects to, if anywhere. If that redirected URL should be handled by another module, dispatch the callback for it. Return a list of (title, hostname) tuples for each URL which is not handled by another module. """ results = [] shorten_url_length = bot.config.url.shorten_url_length for url in urls: if not url.startswith(bot.config.url.exclusion_char): # Magic stuff to account for international domain names try: url = web.iri_to_uri(url) except Exception: # TODO: Be specific pass # First, check that the URL we got doesn't match matched = check_callbacks(bot, trigger, url, False) if matched: continue # If the URL is over bot.config.url.shorten_url_length, # shorten the URL tinyurl = None if (shorten_url_length > 0) and (len(url) > shorten_url_length): # Check bot memory to see if the shortened URL is already in # memory if not bot.memory.contains('shortened_urls'): # Initialize shortened_urls as a dict if it doesn't exist. bot.memory['shortened_urls'] = tools.SopelMemory() if bot.memory['shortened_urls'].contains(url): tinyurl = bot.memory['shortened_urls'][url] else: tinyurl = get_tinyurl(url) bot.memory['shortened_urls'][url] = tinyurl # Finally, actually show the URL title = find_title(url, verify=bot.config.core.verify_ssl) if title: results.append((title, get_hostname(url), tinyurl)) return results
def process_urls(bot, trigger, urls): results = [] for url in urls: try: url = web.iri_to_uri(url) except: pass title = None for regex, function in tools.iteritems(bot.memory['url_callbacks']): match = regex.search(url) if match is not None: title = function(bot, trigger, match) if title: results.append((title, get_hostname(url))) break if not title: title = find_title(url, verify=bot.config.core.verify_ssl) if title: results.append((title, get_hostname(url))) return results