def test_search_urls_exclusion_char(): # assert url is excluded urls = list(search_urls('!http://example.com', exclusion_char='!')) assert not urls, 'Must not find URL, found %d' % len(urls) # assert the other url is not excluded urls = list(search_urls('http://b.com !http://a.com', exclusion_char='!')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://b.com' in urls # assert the order of appearance does not matter urls = list(search_urls('!http://a.com http://b.com', exclusion_char='!')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://b.com' in urls
def title_auto(bot, trigger): """ Automatically show titles for URLs. For shortened URLs/redirects, find where the URL redirects to and show the title for that (or call a function from another module to give more information). """ if re.match(bot.config.core.prefix + 'title', trigger): return # Avoid fetching known malicious links if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']: if bot.memory['safety_cache'][trigger]['positives'] > 1: return urls = web.search_urls(trigger, exclusion_char=bot.config.url.exclusion_char, clean=True) for url, title, domain, tinyurl in process_urls(bot, trigger, urls): message = '[ %s ] - %s' % (title, domain) if tinyurl: message += ' ( %s )' % tinyurl # Guard against responding to other instances of this bot. if message != trigger: bot.say(message) bot.memory['last_seen_url'][trigger.sender] = url
def title_command(bot, trigger): """ Show the title or URL information for the given URL, or the last URL seen in this channel. """ if not trigger.group(2): if trigger.sender not in bot.memory['last_seen_url']: return matched = check_callbacks( bot, bot.memory['last_seen_url'][trigger.sender]) if matched: return else: urls = [bot.memory['last_seen_url'][trigger.sender]] else: urls = web.search_urls( trigger, exclusion_char=bot.config.url.exclusion_char) for url, title, domain, tinyurl in process_urls(bot, trigger, urls): message = '[ %s ] - %s' % (title, domain) if tinyurl: message += ' ( %s )' % tinyurl bot.reply(message) bot.memory['last_seen_url'][trigger.sender] = url
def title_auto(bot, trigger): """ Automatically show titles for URLs. For shortened URLs/redirects, find where the URL redirects to and show the title for that (or call a function from another module to give more information). """ if re.match(bot.config.core.prefix + 'title', trigger): return # Avoid fetching known malicious links if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']: if bot.memory['safety_cache'][trigger]['positives'] > 1: return urls = web.search_urls( trigger, exclusion_char=bot.config.url.exclusion_char, clean=True) for url, title, domain, tinyurl in process_urls(bot, trigger, urls): message = '[ %s ] - %s' % (title, domain) if tinyurl: message += ' ( %s )' % tinyurl # Guard against responding to other instances of this bot. if message != trigger: bot.say(message) bot.memory['last_seen_url'][trigger.sender] = url
def test_search_urls_exclusion_char(): # assert url is excluded urls = list(search_urls('!http://example.com', exclusion_char='!')) assert not urls, 'Must not find URL, found %d' % len(urls) # assert the other url is not excluded urls = list( search_urls('http://b.com !http://a.com', exclusion_char='!')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://b.com' in urls # assert the order of appearance does not matter urls = list( search_urls('!http://a.com http://b.com', exclusion_char='!')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://b.com' in urls
def test_search_urls_exclusion_char_with_text(): urls = list( search_urls( 'before !http://a.com between http://b.com after', exclusion_char='!') ) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://b.com' in urls
def test_search_urls_exclusion_char_only_once(): # assert only the instance excluded is excluded # ie. that it is not a global exclude, otherwise that would return 1 url urls = list( search_urls('!http://a.com http://a.com http://b.com', exclusion_char='!')) assert len(urls) == 2, 'Must find 1 URL, found %d' % len(urls) assert 'http://a.com' in urls assert 'http://b.com' in urls
def test_search_urls_exclusion_char_only_once(): # assert only the instance excluded is excluded # ie. that it is not a global exclude, otherwise that would return 1 url urls = list( search_urls( '!http://a.com http://a.com http://b.com', exclusion_char='!') ) assert len(urls) == 2, 'Must find 1 URL, found %d' % len(urls) assert 'http://a.com' in urls assert 'http://b.com' in urls
def test_search_urls_multiple_urls_unique_keep_ordering(): urls = list( search_urls('http://a.com/ http://c.com/ http://b.com/ http://a.com/')) assert len(urls) == 3, 'Must find 3 URLs, found %d' % len(urls) assert 'http://a.com/' in urls assert 'http://b.com/' in urls assert 'http://c.com/' in urls assert urls == [ 'http://a.com/', 'http://c.com/', 'http://b.com/', ]
def test_search_urls_defined_schemes(scheme): expected = { 'http': 'http://a.com', 'https': 'https://c.com', 'ftp': 'ftp://b.com', 'steam': 'steam://portal2', }.get(scheme) urls = list( search_urls('http://a.com ftp://b.com https://c.com steam://portal2', schemes=[scheme])) assert len(urls) == 1, 'Only %s URLs must be found' % scheme assert expected in urls
def test_search_urls_multiple_urls_unique(): urls = list(search_urls('http://a.com/ http://b.com/ http://a.com/')) assert len(urls) == 2, 'Must find 2 URLs, found %d' % len(urls) assert 'http://a.com/' in urls assert 'http://b.com/' in urls
def test_search_urls_multiple_urls_with_text(): urls = list( search_urls('before http://a.com/ between http://b.com/ after')) assert len(urls) == 2, 'Must find 2 URLs, found %d' % len(urls) assert 'http://a.com/' in urls assert 'http://b.com/' in urls
def test_search_urls_default_schemes(): urls = list(search_urls('http://a.com ftp://b.com https://c.com')) assert len(urls) == 3, 'Must find all three URLs' assert 'http://a.com' in urls assert 'ftp://b.com' in urls assert 'https://c.com' in urls
def test_search_urls(): urls = list(search_urls('http://example.com')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://example.com' in urls
def test_search_urls_exclusion_char_with_text(): urls = list( search_urls('before !http://a.com between http://b.com after', exclusion_char='!')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://b.com' in urls
def test_search_urls_with_text(): urls = list(search_urls('before http://example.com after')) assert len(urls) == 1, 'Must find 1 URL, found %d' % len(urls) assert 'http://example.com' in urls