def test_awik_none(self):
self.term = "Ajgoajh"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
expected = "Can't find anything in the Apertium Wiki for \"%s\"."
self.assertEqual(out, expected % self.text)
def test_awik_none(self):
self.term = "Ajgoajh"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
expected = "Can't find anything in the Apertium Wiki for \"%s\"."
self.assertEqual(out, expected % self.text)
def test_awik(self):
self.term = "Apertium Turkic"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
self.keywords = ['Apertium', 'Turkic', 'working', 'group']
self.check_snippet(out)
def test_awik(self):
self.term = "Apertium Turkic"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
self.keywords = ['Apertium', 'Turkic', 'working', 'group']
self.check_snippet(out)
def test_awik_invalid(self):
self.term = "Apertium Turkic"
self.section = "Doner"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
message = "No '%s' section found." % self.section
self.assertEqual(out, '"%s" - %s' % (message, self.url))
def test_awik_fragment(self):
self.term = "Apertium Turkic"
self.section = "Translation pairs"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
self.keywords = ['work', 'systems', 'Turkic', 'languages']
self.check_snippet(out)
def test_awik_invalid(self):
self.term = "Apertium Turkic"
self.section = "Doner"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
message = "No '%s' section found." % self.section
self.assertEqual(out, '"%s" - %s' % (message, self.url))
def test_awik_fragment(self):
self.term = "Apertium Turkic"
self.section = "Translation pairs"
self.prepare()
apertium_wiki.awik(self.phenny, self.input)
out = self.phenny.say.call_args[0][0]
self.keywords = ['work', 'systems', 'Turkic', 'languages']
self.check_snippet(out)
def gettitle(phenny, uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
if uri.startswith('http://wiki.apertium.org/wiki/'):
item = uri[len('http://wiki.apertium.org/wiki/'):]
return awik(phenny, re.match(r'(blahblah)?(.*)', item))
if re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri):
item = re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri).group(1)
return w(phenny, re.match(r'(blahblah)?(.*)', web.unquote(item)))
if re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri):
match = re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri)
lang, page = match.group(1), match.group(2)
return wikipedia(phenny, page, lang)
parts = uri.split(".")
start = parts[0]
parts.pop(0)
uri = start + "." + web.quote('.'.join(parts))
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
'http://localhost:', 'https://localhost:',
]
for s in localhost:
if uri.startswith(s):
return #phenny.reply('Sorry, access forbidden.')
if not hasattr(phenny.config, 'blacklisted_urls'):
phenny.config.blacklisted_urls = []
if not hasattr(phenny.bot, 'blacklisted_urls'):
phenny.bot.blacklisted_urls = []
for s in phenny.config.blacklisted_urls:
phenny.bot.blacklisted_urls.append(re.compile(s))
for regex in phenny.bot.blacklisted_urls:
if regex.match(uri):
return
try:
redirects = 0
while True:
try:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
except web.HTTPError:
try:
info = requests.get(uri, headers=web.default_headers, verify=True)
status = str(info.status_code)
info = info.headers
except web.HTTPError:
return None
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else:
break
redirects += 1
if redirects >= 25:
return None
try:
mtype = info['content-type']
except:
return None
if not mtype or not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
try:
bytes = web.get(uri)
except:
return None
#bytes = u.read(262144)
#u.close()
except web.ConnectionError:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
title = "[ {0} ]".format(title)
else:
title = None
return title
def gettitle(phenny, input, uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
if uri.startswith('http://wiki.apertium.org/wiki/'):
item = uri[len('http://wiki.apertium.org/wiki/'):]
return apertium_wiki.awik(phenny, re.match(r'(blahblah)?(.*)()', item))
if re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri):
item = re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri).group(1)
return wiktionary.w(phenny,
re.match(r'(blahblah)?(.*)()', web.unquote(item)))
if re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri):
match = re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri)
lang, page = match.group(1), match.group(2)
return wikipedia.wikipedia(phenny, page, lang)
parts = uri.split(".")
start = parts[0]
parts.pop(0)
uri = start + "." + web.quote('.'.join(parts), safe='/#')
title = None
localhost = [
'http://localhost/',
'http://localhost:80/',
'http://localhost:8080/',
'http://127.0.0.1/',
'http://127.0.0.1:80/',
'http://127.0.0.1:8080/',
'https://localhost/',
'https://localhost:80/',
'https://localhost:8080/',
'https://127.0.0.1/',
'https://127.0.0.1:80/',
'https://127.0.0.1:8080/',
'http://localhost:',
'https://localhost:',
]
for s in localhost:
if uri.startswith(s):
return #phenny.reply('Sorry, access forbidden.')
if not hasattr(phenny.config, 'blacklisted_urls'):
phenny.config.blacklisted_urls = []
if not hasattr(phenny, 'blacklisted_urls'):
phenny.blacklisted_urls = []
for s in phenny.config.blacklisted_urls:
phenny.blacklisted_urls.append(re.compile(s))
for regex in phenny.blacklisted_urls:
if regex.match(uri):
return
try:
redirects = 0
while True:
try:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
except web.HTTPError:
try:
info = requests.get(uri,
headers=web.default_headers,
verify=True,
timeout=REQUEST_TIMEOUT)
status = str(info.status_code)
info = info.headers
except web.HTTPError:
return None
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else:
break
redirects += 1
if redirects >= 25:
return None
try:
mtype = info['content-type']
except:
return None
if not mtype or not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
try:
bytes = web.get(uri)
except:
return None
#bytes = u.read(262144)
#u.close()
except:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
title = "[ {0} ]".format(title)
else:
title = None
return title