def process_urls(bot, trigger, urls):
"""
For each URL in the list, ensure that it isn't handled by another module.
If not, find where it redirects to, if anywhere. If that redirected URL
should be handled by another module, dispatch the callback for it.
Return a list of (title, hostname) tuples for each URL which is not handled by
another module.
"""
results = []
for url in urls:
if not url.startswith(bot.config.url.exclusion_char):
# Magic stuff to account for international domain names
try:
url = web.iri_to_uri(url)
except:
pass
# First, check that the URL we got doesn't match
matched = check_callbacks(bot, trigger, url, False)
if matched:
continue
# Then see if it redirects anywhere
new_url = follow_redirects(url)
if not new_url:
continue
# Then see if the final URL matches anything
matched = check_callbacks(bot, trigger, new_url, new_url != url)
if matched:
continue
# Finally, actually show the URL
title = find_title(url)
if title:
results.append((title, get_hostname(url)))
return results
def process_urls(bot, trigger, urls):
"""
For each URL in the list, ensure that it isn't handled by another module.
If not, find where it redirects to, if anywhere. If that redirected URL
should be handled by another module, dispatch the callback for it.
Return a list of (title, hostname) tuples for each URL which is not handled by
another module.
"""
results = []
for url in urls:
if not url.startswith(bot.config.url.exclusion_char):
# Magic stuff to account for international domain names
try:
url = web.iri_to_uri(url)
except:
pass
# First, check that the URL we got doesn't match
matched = check_callbacks(bot, trigger, url, False)
if matched:
continue
# Finally, actually show the URL
title = find_title(url)
if title:
results.append((title, get_hostname(url)))
return results
def process_urls(bot, trigger, urls):
"""
For each URL in the list, ensure that it isn't handled by another module.
If not, find where it redirects to, if anywhere. If that redirected URL
should be handled by another module, dispatch the callback for it.
Return a list of (title, hostname) tuples for each URL which is not handled by
another module.
"""
results = []
shorten_url_length = bot.config.url.shorten_url_length
for url in urls:
if not url.startswith(bot.config.url.exclusion_char):
# Magic stuff to account for international domain names
try:
url = web.iri_to_uri(url)
except Exception: # TODO: Be specific
pass
# First, check that the URL we got doesn't match
matched = check_callbacks(bot, trigger, url, False)
if matched:
continue
# If the URL is over bot.config.url.shorten_url_length,
# shorten the URL
tinyurl = None
if (shorten_url_length > 0) and (len(url) > shorten_url_length):
# Check bot memory to see if the shortened URL is already in
# memory
if not bot.memory.contains('shortened_urls'):
# Initialize shortened_urls as a dict if it doesn't exist.
bot.memory['shortened_urls'] = tools.SopelMemory()
if bot.memory['shortened_urls'].contains(url):
tinyurl = bot.memory['shortened_urls'][url]
else:
tinyurl = get_tinyurl(url)
bot.memory['shortened_urls'][url] = tinyurl
# Finally, actually show the URL
title = find_title(url, verify=bot.config.core.verify_ssl)
if title:
results.append((title, get_hostname(url), tinyurl))
return results
def process_urls(bot, trigger, urls):
"""
For each URL in the list, ensure that it isn't handled by another module.
If not, find where it redirects to, if anywhere. If that redirected URL
should be handled by another module, dispatch the callback for it.
Return a list of (title, hostname) tuples for each URL which is not handled by
another module.
"""
results = []
shorten_url_length = bot.config.url.shorten_url_length
for url in urls:
if not url.startswith(bot.config.url.exclusion_char):
# Magic stuff to account for international domain names
try:
url = web.iri_to_uri(url)
except Exception: # TODO: Be specific
pass
# First, check that the URL we got doesn't match
matched = check_callbacks(bot, trigger, url, False)
if matched:
continue
# If the URL is over bot.config.url.shorten_url_length,
# shorten the URL
tinyurl = None
if (shorten_url_length > 0) and (len(url) > shorten_url_length):
# Check bot memory to see if the shortened URL is already in
# memory
if not bot.memory.contains('shortened_urls'):
# Initialize shortened_urls as a dict if it doesn't exist.
bot.memory['shortened_urls'] = tools.SopelMemory()
if bot.memory['shortened_urls'].contains(url):
tinyurl = bot.memory['shortened_urls'][url]
else:
tinyurl = get_tinyurl(url)
bot.memory['shortened_urls'][url] = tinyurl
# Finally, actually show the URL
title = find_title(url, verify=bot.config.core.verify_ssl)
if title:
results.append((title, get_hostname(url), tinyurl))
return results
def process_urls(bot, trigger, urls):
results = []
for url in urls:
try:
url = web.iri_to_uri(url)
except:
pass
title = None
for regex, function in tools.iteritems(bot.memory['url_callbacks']):
match = regex.search(url)
if match is not None:
title = function(bot, trigger, match)
if title:
results.append((title, get_hostname(url)))
break
if not title:
title = find_title(url, verify=bot.config.core.verify_ssl)
if title:
results.append((title, get_hostname(url)))
return results
