def short(text):
"""
This function creates a bitly url for each url in the provided "text" string.
The return type is a list.
"""
if not bitly_loaded: return list()
if not text: return list()
bitlys = list()
try:
a = re.findall(url_finder, text)
k = len(a)
i = 0
while i < k:
b = unicode.decode(a[i][0])
if not b.startswith("http://bit.ly") or not b.startswith("http://j.mp/"):
# check to see if the url is valid
try: c = web.head(b)
except: return [[None, None]]
url = "http://api.j.mp/v3/shorten?login=%s&apiKey=%s&longUrl=%s&format=txt" % (bitly_user, bitly_api_key, urllib2.quote(b))
shorter = web.get(url)
shorter.strip()
bitlys.append([b, shorter])
i += 1
return bitlys
except:
return
def service(code, input, command, args):
t = o.services[command]
template = t.replace('${args}', urllib.quote(args.encode('utf-8'), ''))
template = template.replace('${nick}', urllib.quote(input.nick, ''))
uri = template.replace('${sender}', urllib.quote(input.sender, ''))
info = web.head(uri)
if isinstance(info, list):
info = info[0]
if not 'text/plain' in info.get('content-type', '').lower():
return code.reply(code.color('red', 'Sorry, the service didn\'t respond in plain text.'))
bytes = web.get(uri)
lines = bytes.splitlines()
if not lines:
return code.reply(code.color('red', 'Sorry, the service didn\'t respond any output.'))
try: line = lines[0].encode('utf-8')[:350]
except: line = lines[0][:250]
if input.group(1) == 'urban':
if line.find('ENOTFOUND') > -1:
line = "I'm sorry, that definition %s found." % code.bold('wasn\'t')
code.say(line)
elif line.find('Traceback (most recent call last)') > -1:
line = code.color('red', 'Failed to search for that definition. Please try again.')
code.say(line)
else:
code.say(line)
def val(jenni, input):
"""Check a webpage using the W3C Markup Validator."""
if not input.group(2):
return jenni.reply("Nothing to validate.")
uri = input.group(2)
if not uri.startswith('http://'):
uri = 'http://' + uri
path = '/check?uri=%s;output=xml' % web.urllib.quote(uri)
info = web.head('http://validator.w3.org' + path)
result = uri + ' is '
if isinstance(info, list):
return jenni.say('Got HTTP response %s' % info[1])
if info.has_key('X-W3C-Validator-Status'):
result += str(info['X-W3C-Validator-Status'])
if info['X-W3C-Validator-Status'] != 'Valid':
if info.has_key('X-W3C-Validator-Errors'):
n = int(info['X-W3C-Validator-Errors'].split(' ')[0])
if n != 1:
result += ' (%s errors)' % n
else: result += ' (%s error)' % n
else: result += 'Unvalidatable: no X-W3C-Validator-Status'
jenni.reply(result)
def service(code, input, command, args):
t = o.services[command]
template = t.replace('${args}', urllib.quote(args.encode('utf-8'), ''))
template = template.replace('${nick}', urllib.quote(input.nick, ''))
uri = template.replace('${sender}', urllib.quote(input.sender, ''))
info = web.head(uri)
if isinstance(info, list):
info = info[0]
if not 'text/plain' in info.get('content-type', '').lower():
return code.reply(
code.color('red',
'Sorry, the service didn\'t respond in plain text.'))
bytes = web.get(uri)
lines = bytes.splitlines()
if not lines:
return code.reply(
code.color('red',
'Sorry, the service didn\'t respond any output.'))
try:
line = lines[0].encode('utf-8')[:350]
except:
line = lines[0][:250]
if input.group(1) == 'urban':
if line.find('ENOTFOUND') > -1:
line = "I'm sorry, that definition %s found." % code.bold(
'wasn\'t')
code.say(line)
elif line.find('Traceback (most recent call last)') > -1:
line = code.color(
'red',
'Failed to search for that definition. Please try again.')
code.say(line)
else:
code.say(line)
def head(jenni, input):
"""Provide HTTP HEAD information."""
uri = input.group(2)
uri = (uri or '').encode('utf-8')
if ' ' in uri:
uri, header = uri.rsplit(' ', 1)
else:
uri, header = uri, None
if not uri and hasattr(jenni, 'last_seen_uri'):
try:
uri = jenni.bot.last_seen_uri[input.sender]
except KeyError:
return jenni.say('?')
if not uri.startswith('htt'):
uri = 'http://' + uri
if '/#!' in uri:
uri = uri.replace('/#!', '/?_escaped_fragment_=')
try:
info = web.head(uri)
except IOError:
return jenni.say("Can't connect to %s" % uri)
except httplib.InvalidURL:
return jenni.say("Not a valid URI, sorry.")
if not isinstance(info, list):
try:
info = dict(info)
except TypeError:
return jenni.reply(
'Try .head http://example.org/ [optional header]')
info['Status'] = '200'
else:
newInfo = dict(info[0])
newInfo['Status'] = str(info[1])
info = newInfo
if header is None:
data = []
if info.has_key('Status'):
data.append(info['Status'])
if info.has_key('content-type'):
data.append(info['content-type'].replace('; charset=', ', '))
if info.has_key('last-modified'):
modified = info['last-modified']
modified = time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')
data.append(time.strftime('%Y-%m-%d %H:%M:%S UTC', modified))
if info.has_key('content-length'):
data.append(info['content-length'] + ' bytes')
jenni.reply(', '.join(data))
else:
headerlower = header.lower()
if info.has_key(headerlower):
jenni.say(header + ': ' + info.get(headerlower))
else:
msg = 'There was no %s header in the response.' % header
jenni.say(msg)
def val(jenny, input):
"""Check a webpage using the W3C Markup Validator."""
uri = input.group(2)
if not uri.startswith("http://"):
uri = "http://" + uri
path = "/check?uri=%s;output=xml" % web.urllib.quote(uri)
info = web.head("http://validator.w3.org" + path)
result = uri + " is "
if isinstance(info, list):
return jenny.say("Got HTTP response %s" % info[1])
if info.has_key("X-W3C-Validator-Status"):
result += str(info["X-W3C-Validator-Status"])
if info["X-W3C-Validator-Status"] != "Valid":
if info.has_key("X-W3C-Validator-Errors"):
n = int(info["X-W3C-Validator-Errors"].split(" ")[0])
if n != 1:
result += " (%s errors)" % n
else:
result += " (%s error)" % n
else:
result += "Unvalidatable: no X-W3C-Validator-Status"
jenny.reply(result)
def val(torp, input):
"""Check a webpage using the W3C Markup Validator."""
uri = input.group(2)
if not uri.startswith('http://'):
uri = 'http://' + uri
path = '/check?uri=%s;output=xml' % web.urllib.quote(uri)
info = web.head('http://validator.w3.org' + path)
result = uri + ' is '
if isinstance(info, list):
return torp.say('Got HTTP response %s' % info[1])
if info.has_key('X-W3C-Validator-Status'):
result += str(info['X-W3C-Validator-Status'])
if info['X-W3C-Validator-Status'] != 'Valid':
if info.has_key('X-W3C-Validator-Errors'):
n = int(info['X-W3C-Validator-Errors'].split(' ')[0])
if n != 1:
result += ' (%s errors)' % n
else: result += ' (%s error)' % n
else: result += 'Unvalidatable: no X-W3C-Validator-Status'
torp.reply(result)
def val(phenny, input):
"""Check a webpage using the W3C Markup Validator."""
if not input.group(2):
return phenny.reply("Nothing to validate.")
uri = input.group(2)
if not uri.startswith('http://'):
uri = 'http://' + uri
logging.debug('Getting W3C validation for ' + uri)
path = '/check?uri=%s;output=xml' % web.urllib.quote(uri)
info = web.head('http://validator.w3.org' + path)
result = uri + ' is '
if isinstance(info, list):
return phenny.say('Got HTTP response %s' % info[1])
if info.has_key('X-W3C-Validator-Status'):
result += str(info['X-W3C-Validator-Status'])
if info['X-W3C-Validator-Status'] != 'Valid':
if info.has_key('X-W3C-Validator-Errors'):
n = int(info['X-W3C-Validator-Errors'].split(' ')[0])
if n != 1:
result += ' (%s errors)' % n
else: result += ' (%s error)' % n
else: result += 'Unvalidatable: no X-W3C-Validator-Status'
logging.debug('Validator: ' + result)
phenny.reply(result)
def short(text):
"""
This function creates a bitly url for each url in the provided "text" string.
The return type is a list.
"""
if not bitly_loaded: return [ ]
bitlys = [ ]
try:
a = re.findall(url_finder, text)
k = len(a)
i = 0
while i < k:
b = str(a[i][0])
if not b.startswith("http://bit.ly") or not b.startswith("http://j.mp/"):
# check to see if the url is valid
try: c = web.head(b)
except: return [[None, None]]
url = "http://api.j.mp/v3/shorten?login=%s&apiKey=%s&longUrl=%s&format=txt" % (bitly_user, bitly_api_key, urllib2.quote(b))
shorter = web.get(url)
shorter.strip()
bitlys.append([b, shorter])
i += 1
return bitlys
except:
return
def head(jenni, input):
"""Provide HTTP HEAD information."""
uri = input.group(2)
uri = (uri or "").encode("utf-8")
if " " in uri:
uri, header = uri.rsplit(" ", 1)
else:
uri, header = uri, None
if not uri and hasattr(jenni, "last_seen_uri"):
try:
uri = jenni.last_seen_uri[input.sender]
except KeyError:
return jenni.say("?")
if not uri.startswith("htt"):
uri = "http://" + uri
# uri = uri.replace('#!', '?_escaped_fragment_=')
try:
info = web.head(uri)
except IOError:
return jenni.say("Can't connect to %s" % uri)
except httplib.InvalidURL:
return jenni.say("Not a valid URI, sorry.")
if not isinstance(info, list):
try:
info = dict(info)
except TypeError:
return jenni.reply("Try .head http://example.org/ [optional header]")
info["Status"] = "200"
else:
newInfo = dict(info[0])
newInfo["Status"] = str(info[1])
info = newInfo
if header is None:
data = []
if info.has_key("Status"):
data.append(info["Status"])
if info.has_key("content-type"):
data.append(info["content-type"].replace("; charset=", ", "))
if info.has_key("last-modified"):
modified = info["last-modified"]
modified = time.strptime(modified, "%a, %d %b %Y %H:%M:%S %Z")
data.append(time.strftime("%Y-%m-%d %H:%M:%S UTC", modified))
if info.has_key("content-length"):
data.append(info["content-length"] + " bytes")
jenni.reply(", ".join(data))
else:
headerlower = header.lower()
if info.has_key(headerlower):
jenni.say(header + ": " + info.get(headerlower))
else:
msg = "There was no %s header in the response." % header
jenni.say(msg)
def id_tweet(tid):
link = "https://twitter.com/twitter/status/" + tid
headers, status = web.head(link)
if status == 301:
if not "Location" in headers:
return "Sorry, couldn't get a tweet from %s" % link
url = headers["Location"]
username = url.split("/")[3]
tweet = read_tweet(url)
return format(tweet, username)
return "Sorry, couldn't get a tweet from %s" % link
def head(phenny, input):
"""Provide HTTP HEAD information."""
uri = input.group(1)
uri = (uri or '')
if ' ' in uri:
uri, header = uri.rsplit(' ', 1)
else:
uri, header = uri, None
if not uri and hasattr(phenny, 'last_seen_uri'):
try:
uri = phenny.last_seen_uri[input.sender]
except KeyError:
return phenny.say('?')
if not uri.startswith('htt'):
uri = 'http://' + uri
# uri = uri.replace('#!', '?_escaped_fragment_=')
start = time.time()
try:
info = web.head(uri)
info['status'] = '200'
except web.HTTPError as e:
if hasattr(e, 'code'):
return phenny.say(str(e.code))
else:
return phenny.say(str(e.response.status_code))
except web.ConnectionError:
return phenny.say("Can't connect to %s" % uri)
resptime = time.time() - start
if header is None:
data = []
if 'Status' in info:
data.append(info['Status'])
if 'content-type' in info:
data.append(info['content-type'].replace('; charset=', ', '))
if 'last-modified' in info:
modified = info['last-modified']
modified = time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')
data.append(time.strftime('%Y-%m-%d %H:%M:%S UTC', modified))
if 'content-length' in info:
data.append(info['content-length'] + ' bytes')
data.append('{0:1.2f} s'.format(resptime))
phenny.reply(', '.join(data))
else:
headerlower = header.lower()
if headerlower in info:
phenny.say(header + ': ' + info.get(headerlower))
else:
msg = 'There was no %s header in the response.' % header
phenny.say(msg)
def id_tweet(tid):
link = 'https://twitter.com/twitter/status/' + tid
headers, status = web.head(link)
if status == 301:
if not "Location" in headers:
return "Sorry, couldn't get a tweet from %s" % link
url = headers["Location"]
username = url.split('/')[3]
tweet = read_tweet(url)
return format(tweet, username)
return "Sorry, couldn't get a tweet from %s" % link
def head(phenny, input):
"""Provide HTTP HEAD information."""
uri = input.group(2)
uri = (uri or '')
if ' ' in uri:
uri, header = uri.rsplit(' ', 1)
else:
uri, header = uri, None
if not uri and hasattr(phenny, 'last_seen_uri'):
try:
uri = phenny.last_seen_uri[input.sender]
except KeyError:
return phenny.say('?')
if not uri.startswith('htt'):
uri = 'http://' + uri
# uri = uri.replace('#!', '?_escaped_fragment_=')
start = time.time()
try:
info = web.head(uri)
info['status'] = '200'
except web.HTTPError as e:
if hasattr(e, 'code'):
return phenny.say(str(e.code))
else:
return phenny.say(str(e.response.status_code))
except web.ConnectionError:
return phenny.say("Can't connect to %s" % uri)
resptime = time.time() - start
if header is None:
data = []
if 'Status' in info:
data.append(info['Status'])
if 'content-type' in info:
data.append(info['content-type'].replace('; charset=', ', '))
if 'last-modified' in info:
modified = info['last-modified']
modified = time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')
data.append(time.strftime('%Y-%m-%d %H:%M:%S UTC', modified))
if 'content-length' in info:
data.append(info['content-length'] + ' bytes')
data.append('{0:1.2f} s'.format(resptime))
phenny.reply(', '.join(data))
else:
headerlower = header.lower()
if headerlower in info:
phenny.say(header + ': ' + info.get(headerlower))
else:
msg = 'There was no %s header in the response.' % header
phenny.say(msg)
def id_tweet(tid):
link = 'https://twitter.com/twitter/status/' + tid
data = web.head(link)
message, status = tuple(data)
if status == 301:
url = message.get("Location")
if not url: return "Sorry, couldn't get a tweet from %s" % link
username = url.split('/')[3]
tweet = read_tweet(url)
return format(tweet, username)
return "Sorry, couldn't get a tweet from %s" % link
def id_tweet(tid):
link = 'https://twitter.com/twitter/status/' + tid
data = web.head(link)
message, status = tuple(data)
if status == 301:
url = message.get("Location")
if not url: return "Sorry, couldn't get a tweet from %s" % link
username = url.split('/')[3]
tweet = read_tweet(url)
return format(tweet, username)
return "Sorry, couldn't get a tweet from %s" % link
def head(phenny, input):
"""Provide HTTP HEAD information."""
uri = input.group(2)
uri = (uri or '').encode('utf-8')
if ' ' in uri:
uri, header = uri.rsplit(' ', 1)
else: uri, header = uri, None
if not uri and hasattr(phenny, 'last_seen_uri'):
try: uri = phenny.last_seen_uri[input.sender]
except KeyError: return phenny.say('?')
if not uri.startswith('htt'):
uri = 'http://' + uri
# uri = uri.replace('#!', '?_escaped_fragment_=')
logging.debug('Getting header information for ' + uri)
try: info = web.head(uri)
except IOError: return phenny.say("Can't connect to %s" % uri)
except httplib.InvalidURL: return phenny.say("Not a valid URI, sorry.")
if not isinstance(info, list):
try: info = dict(info)
except TypeError:
return phenny.reply('Try .head http://example.org/ [optional header]')
info['Status'] = '200'
else:
newInfo = dict(info[0])
newInfo['Status'] = str(info[1])
info = newInfo
if header is None:
data = []
if info.has_key('Status'):
data.append(info['Status'])
if info.has_key('content-type'):
data.append(info['content-type'].replace('; charset=', ', '))
if info.has_key('last-modified'):
modified = info['last-modified']
modified = time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')
data.append(time.strftime('%Y-%m-%d %H:%M:%S UTC', modified))
if info.has_key('content-length'):
data.append(info['content-length'] + ' bytes')
phenny.reply(', '.join(data))
else:
headerlower = header.lower()
if info.has_key(headerlower):
phenny.say(header + ': ' + info.get(headerlower))
else:
msg = 'There was no %s header in the response.' % header
phenny.say(msg)
def head(phenny, input):
"""Provide HTTP HEAD information."""
uri = input.group(2)
uri = (uri or "").encode("utf-8")
if " " in uri:
uri, header = uri.rsplit(" ", 1)
else:
uri, header = uri, None
if not uri and hasattr(phenny, "last_seen_uri"):
try:
uri = phenny.last_seen_uri[input.sender]
except KeyError:
return phenny.say("?")
try:
info = web.head(uri)
except IOError:
return phenny.say("Can't connect to %s" % uri)
if not isinstance(info, list):
info = dict(info)
info["Status"] = "200"
else:
newInfo = dict(info[0])
newInfo["Status"] = str(info[1])
info = newInfo
if header is None:
data = []
if info.has_key("Status"):
data.append(info["Status"])
if info.has_key("content-type"):
data.append(info["content-type"].replace("; charset=", ", "))
if info.has_key("last-modified"):
modified = info["last-modified"]
modified = time.strptime(modified, "%a, %d %b %Y %H:%M:%S %Z")
data.append(time.strftime("%Y-%m-%d %H:%M:%S UTC", modified))
if info.has_key("content-length"):
data.append(info["content-length"] + " bytes")
phenny.reply(", ".join(data))
else:
headerlower = header.lower()
if info.has_key(headerlower):
phenny.say(header + ": " + info.get(headerlower))
else:
msg = "There was no %s header in the response." % header
phenny.say(msg)
def service(jenni, input, command, args):
t = o.services[command]
template = t.replace("${args}", urllib.quote(args.encode("utf-8"), ""))
template = template.replace("${nick}", urllib.quote(input.nick, ""))
uri = template.replace("${sender}", urllib.quote(input.sender, ""))
info = web.head(uri)
if isinstance(info, list):
info = info[0]
if not "text/plain" in info.get("content-type", "").lower():
return jenni.reply("Sorry, the service didn't respond in plain text.")
bytes = web.get(uri)
lines = bytes.splitlines()
if not lines:
return jenni.reply("Sorry, the service didn't respond any output.")
jenni.say(lines[0][:350])
def service(phenny, input, command, args):
t = o.services[command]
template = t.replace('${args}', urllib.quote(args.encode('utf-8'), ''))
template = template.replace('${nick}', urllib.quote(input.nick, ''))
uri = template.replace('${sender}', urllib.quote(input.sender, ''))
info = web.head(uri)
if isinstance(info, list):
info = info[0]
if not 'text/plain' in info.get('content-type', '').lower():
return phenny.reply("Sorry, the service didn't respond in plain text.")
bytes = web.get(uri)
lines = bytes.splitlines()
if not lines:
return phenny.reply("Sorry, the service didn't respond any output.")
phenny.say(lines[0][:350])
def service(phenny, input, command, args):
t = o.services[command]
template = t.replace('${args}', urllib.quote(args.encode('utf-8'), ''))
template = template.replace('${nick}', urllib.quote(input.nick, ''))
uri = template.replace('${sender}', urllib.quote(input.sender, ''))
info = web.head(uri)
if isinstance(info, list):
info = info[0]
if not 'text/plain' in info.get('content-type', '').lower():
return phenny.reply("Sorry, the service didn't respond in plain text.")
bytes = web.get(uri)
lines = bytes.splitlines()
if not lines:
return phenny.reply("Sorry, the service didn't respond any output.")
phenny.say(lines[0][:350])
def short(text):
bitlys = [ ]
try:
a = re.findall(url_finder, text)
k = len(a)
i = 0
while i < k:
b = str(a[i][0])
if not b.startswith("http://bit.ly") or not b.startswith("http://j.mp/"):
# check to see if the url is valid
try: c = web.head(b)
except: return [[None, None]]
url = "http://api.j.mp/v3/shorten?login=%s&apiKey=%s&longUrl=%s&format=txt" % (bitly_user, bitly_api_key, urllib2.quote(b))
shorter = web.get(url)
shorter.strip()
bitlys.append([b, shorter])
i += 1
return bitlys
except:
return
def gettitle(uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
]
for s in localhost:
if uri.startswith(s):
return phenny.reply('Sorry, access forbidden.')
youtube = re.compile('http(s)?://(www.)?youtube.(com|co.uk|ca)?/watch(.*)\?v(.*)')
if youtube.match(uri) or re.compile('http(s)?://youtu.be/(.*)').match(uri):
return get_youtube_title(uri)
fimfiction = re.compile('http(s)?://(www.)?fimfiction.net/story/')
if fimfiction.match(uri):
return get_story_title(uri)
# TODO: add e621, twentypercentcooler and derpibooru support
if re.compile('http(s)?://(www.)?bad-dragon.com/').match(uri) and not check_cookie('baddragon_age_checked'):
urllib.request.urlopen('http://bad-dragon.com/agecheck/accept')
if re.compile('http(s)?://(www.)?((e621)|(e926)).net/post/show/').match(uri): #e621 or e926 link
return ouroboros('e621',uri)
if re.compile('http(s)?://(www.)?twentypercentcooler.net/post/show/').match(uri):
return ouroboros('twentypercentcooler',uri)
if re.compile('http(s)?://(www.)?derpiboo((.ru)|(ru.org))(/images)?/').match(uri):
return derpibooru(uri)
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else: break
redirects += 1
if redirects >= 25:
return None
try: mtype = info['content-type']
except:
return None
if not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
bytes = web.get(uri)
#bytes = u.read(262144)
#u.close()
except IOError:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
else: title = None
return title
def gettitle(phenny, input, uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
if uri.startswith('http://wiki.apertium.org/wiki/'):
item = uri[len('http://wiki.apertium.org/wiki/'):]
return apertium_wiki.awik(phenny, re.match(r'(blahblah)?(.*)()', item))
if re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri):
item = re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri).group(1)
return wiktionary.w(phenny,
re.match(r'(blahblah)?(.*)()', web.unquote(item)))
if re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri):
match = re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri)
lang, page = match.group(1), match.group(2)
return wikipedia.wikipedia(phenny, page, lang)
parts = uri.split(".")
start = parts[0]
parts.pop(0)
uri = start + "." + web.quote('.'.join(parts), safe='/#')
title = None
localhost = [
'http://localhost/',
'http://localhost:80/',
'http://localhost:8080/',
'http://127.0.0.1/',
'http://127.0.0.1:80/',
'http://127.0.0.1:8080/',
'https://localhost/',
'https://localhost:80/',
'https://localhost:8080/',
'https://127.0.0.1/',
'https://127.0.0.1:80/',
'https://127.0.0.1:8080/',
'http://localhost:',
'https://localhost:',
]
for s in localhost:
if uri.startswith(s):
return #phenny.reply('Sorry, access forbidden.')
if not hasattr(phenny.config, 'blacklisted_urls'):
phenny.config.blacklisted_urls = []
if not hasattr(phenny, 'blacklisted_urls'):
phenny.blacklisted_urls = []
for s in phenny.config.blacklisted_urls:
phenny.blacklisted_urls.append(re.compile(s))
for regex in phenny.blacklisted_urls:
if regex.match(uri):
return
try:
redirects = 0
while True:
try:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
except web.HTTPError:
try:
info = requests.get(uri,
headers=web.default_headers,
verify=True,
timeout=REQUEST_TIMEOUT)
status = str(info.status_code)
info = info.headers
except web.HTTPError:
return None
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else:
break
redirects += 1
if redirects >= 25:
return None
try:
mtype = info['content-type']
except:
return None
if not mtype or not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
try:
bytes = web.get(uri)
except:
return None
#bytes = u.read(262144)
#u.close()
except:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
title = "[ {0} ]".format(title)
else:
title = None
return title
#!/usr/bin/env python
import cgi
import cgitb; cgitb.enable()
import sys
sys.path.append("..")
from data import db_gw
from data import is_address
from fmt import fmt_rule
import web
print web.head()
body = []
args = cgi.FieldStorage()
if 'fw' not in args:
raise Exception("Need a firewall to edit")
fw = args['fw'].value
db = db_gw(fw)
if 'id' not in args:
body.append(" <table class='ui-widget' cellspacing='0px' cellpadding='2px'><tr class='ui-widget-header'><th>Name</th><th>Description</th></tr>")
for g in sorted(db.get_hosts(host_ids=db.get_fw_groups()), cmp=lambda a,b: cmp(a['name'].lower(), b['name'].lower())):
g['fw'] = fw
body.append(" <tr><td><a href='host.py?id=%(hosts.id)s&fw=%(fw)s'>%(name)s</a></td><td>%(description)s</td></tr>"%g)
body.append(" </table>")
else:
gid = int(args['id'].value)
gids = []
if 'remove' in args:
rid = int(args['remove'].value)
def gettitle(uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
]
for s in localhost:
if uri.startswith(s):
return phenny.reply('Sorry, access forbidden.')
if re.compile('http(s)?://(www.)?bad-dragon.com/').match(uri) and not check_cookie('baddragon_age_checked'):
urllib.request.urlopen('http://bad-dragon.com/agecheck/accept')
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else: break
redirects += 1
if redirects >= 25:
return None
try: mtype = info['content-type']
except:
return None
try:
# Occasionally throws type errors if a CSS file is given.
if not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
except:
return None
bytes = web.get(uri)
#bytes = u.read(262144)
#u.close()
except IOError:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
else: title = None
return title
def gettitle(uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
]
for s in localhost:
if uri.startswith(s):
return
if re.compile('http(s)?://(www.)?bad-dragon.com/').match(uri) and not check_cookie('baddragon_age_checked'):
urllib.request.urlopen('http://bad-dragon.com/agecheck/accept')
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else: break
redirects += 1
if redirects >= 25:
return None
try: mtype = info['content-type']
except:
return None
try:
# Occasionally throws type errors if a CSS file is given.
if not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
except:
return None
bytes = web.get(uri)
#bytes = u.read(262144)
#u.close()
except IOError:
return
except UnicodeError:
'''
Due to the way Python implemented the urllib.request.urlopen()
function, it is not possible to correct for Unicode characters
like € in a URL. Therefore, we just catch the error and don't
provide a title for the link. Other options may be worth
exploring, and could be included here.
'''
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
else: title = None
return title
def tock(phenny, input):
"""Shows the time from the USNO's atomic clock."""
info = web.head('http://tycho.usno.navy.mil/cgi-bin/timer.pl')
phenny.say('"' + info['Date'] + '" - tycho.usno.navy.mil')
def f_title(phenny, input):
""".title <URI> - Return the title of URI."""
uri = input.group(2)
uri = (uri or "").encode("utf-8")
if not uri and hasattr(self, "last_seen_uri"):
uri = self.last_seen_uri.get(input.sender)
if not uri:
return phenny.reply("I need a URI to give the title of...")
if not ":" in uri:
uri = "http://" + uri
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = "200"
else:
status = str(info[1])
info = info[0]
if status.startswith("3"):
uri = urlparse.urljoin(uri, info["Location"])
else:
break
redirects += 1
if redirects >= 25:
phenny.reply("Too many redirects")
return
try:
mtype = info["content-type"]
except:
err = "Couldn't get the Content-Type, sorry"
return phenny.reply(err)
if not (("/html" in mtype) or ("/xhtml" in mtype)):
phenny.reply("Document isn't HTML")
return
u = urllib.urlopen(uri)
bytes = u.read(32768)
u.close()
except IOError:
phenny.say("Can't connect to %s" % uri)
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace("\t", " ")
title = title.replace("\r", " ")
title = title.replace("\n", " ")
while " " in title:
title = title.replace(" ", " ")
if len(title) > 200:
title = title[:200] + "[...]"
def e(m):
entity = m.group(0)
if entity.startswith("&#x"):
cp = int(entity[3:-1], 16)
return unichr(cp).encode("utf-8")
elif entity.startswith("&#"):
cp = int(entity[2:-1])
return unichr(cp).encode("utf-8")
else:
char = name2codepoint[entity[1:-1]]
return unichr(char).encode("utf-8")
title = r_entity.sub(e, title)
if not title:
title = '[Title is the empty document, "".]'
phenny.reply(title)
else:
phenny.reply(" No title found")
def gettitle(uri):
if re.compile('http(s)?://(.*).(jpg|jpeg|png|gif|tiff|bmp)').match(uri):
return None
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
title = None
localhost = [
'http://localhost/',
'http://localhost:80/',
'http://localhost:8080/',
'http://127.0.0.1/',
'http://127.0.0.1:80/',
'http://127.0.0.1:8080/',
'https://localhost/',
'https://localhost:80/',
'https://localhost:8080/',
'https://127.0.0.1/',
'https://127.0.0.1:80/',
'https://127.0.0.1:8080/',
]
for s in localhost:
if uri.startswith(s):
return
if re.compile('http(s)?://(www.)?bad-dragon.com/').match(
uri) and not check_cookie('baddragon_age_checked'):
urllib.request.urlopen('http://bad-dragon.com/agecheck/accept')
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else:
break
redirects += 1
if redirects >= 25:
return None
try:
mtype = info['content-type']
except:
return None
try:
# Occasionally throws type errors if a CSS file is given.
if not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
except:
return None
bytes = web.get(uri, isSecure=False)
#bytes = u.read(262144)
#u.close()
except IOError:
return
except UnicodeError:
'''
Due to the way Python implemented the urllib.request.urlopen()
function, it is not possible to correct for Unicode characters
like € in a URL. Therefore, we just catch the error and don't
provide a title for the link. Other options may be worth
exploring, and could be included here.
'''
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
else:
title = None
return title
def gettitle(phenny, uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
]
for s in localhost:
if uri.startswith(s):
return phenny.reply('Sorry, access forbidden.')
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else:
break
redirects += 1
if redirects >= 25:
return None
try:
mtype = info['content-type']
except:
return None
if not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
bytes = web.get(uri)
#bytes = u.read(262144)
#u.close()
except IOError:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
title = "[ {0} ]".format(title)
else:
title = None
return title
def tock(phenny, input):
"""Shows the time from the USNO's atomic clock."""
info = web.head('http://tycho.usno.navy.mil/cgi-bin/timer.pl')
phenny.say('"' + info['Date'] + '" - tycho.usno.navy.mil')
def gettitle(uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
]
for s in localhost:
if uri.startswith(s):
return phenny.reply('Sorry, access forbidden.')
try:
redirects = 0
while True:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else: break
redirects += 1
if redirects >= 25:
return None
try: mtype = info['content-type']
except:
return None
if not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
bytes = web.get(uri)
#bytes = u.read(262144)
#u.close()
except IOError:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
else: title = None
return title
def gettitle(phenny, uri):
if not ':' in uri:
uri = 'http://' + uri
uri = uri.replace('#!', '?_escaped_fragment_=')
if uri.startswith('http://wiki.apertium.org/wiki/'):
item = uri[len('http://wiki.apertium.org/wiki/'):]
return awik(phenny, re.match(r'(blahblah)?(.*)', item))
if re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri):
item = re.match(r'https?://en.wiktionary.org/wiki/(.*)', uri).group(1)
return w(phenny, re.match(r'(blahblah)?(.*)', web.unquote(item)))
if re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri):
match = re.match(r'https?://([a-z]{2,3}).wikipedia.org/wiki/(.*)', uri)
lang, page = match.group(1), match.group(2)
return wikipedia(phenny, page, lang)
parts = uri.split(".")
start = parts[0]
parts.pop(0)
uri = start + "." + web.quote('.'.join(parts))
title = None
localhost = [
'http://localhost/', 'http://localhost:80/',
'http://localhost:8080/', 'http://127.0.0.1/',
'http://127.0.0.1:80/', 'http://127.0.0.1:8080/',
'https://localhost/', 'https://localhost:80/',
'https://localhost:8080/', 'https://127.0.0.1/',
'https://127.0.0.1:80/', 'https://127.0.0.1:8080/',
'http://localhost:', 'https://localhost:',
]
for s in localhost:
if uri.startswith(s):
return #phenny.reply('Sorry, access forbidden.')
if not hasattr(phenny.config, 'blacklisted_urls'):
phenny.config.blacklisted_urls = []
if not hasattr(phenny.bot, 'blacklisted_urls'):
phenny.bot.blacklisted_urls = []
for s in phenny.config.blacklisted_urls:
phenny.bot.blacklisted_urls.append(re.compile(s))
for regex in phenny.bot.blacklisted_urls:
if regex.match(uri):
return
try:
redirects = 0
while True:
try:
info = web.head(uri)
if not isinstance(info, list):
status = '200'
else:
status = str(info[1])
info = info[0]
except web.HTTPError:
try:
info = requests.get(uri, headers=web.default_headers, verify=True)
status = str(info.status_code)
info = info.headers
except web.HTTPError:
return None
if status.startswith('3'):
uri = urllib.parse.urljoin(uri, info['Location'])
else:
break
redirects += 1
if redirects >= 25:
return None
try:
mtype = info['content-type']
except:
return None
if not mtype or not (('/html' in mtype) or ('/xhtml' in mtype)):
return None
try:
bytes = web.get(uri)
except:
return None
#bytes = u.read(262144)
#u.close()
except web.ConnectionError:
return
m = r_title.search(bytes)
if m:
title = m.group(1)
title = title.strip()
title = title.replace('\t', ' ')
title = title.replace('\r', ' ')
title = title.replace('\n', ' ')
while ' ' in title:
title = title.replace(' ', ' ')
if len(title) > 200:
title = title[:200] + '[...]'
def e(m):
entity = m.group(0)
if entity.startswith('&#x'):
cp = int(entity[3:-1], 16)
return chr(cp)
elif entity.startswith('&#'):
cp = int(entity[2:-1])
return chr(cp)
else:
char = name2codepoint[entity[1:-1]]
return chr(char)
title = r_entity.sub(e, title)
if title:
title = title.replace('\n', '')
title = title.replace('\r', '')
title = "[ {0} ]".format(title)
else:
title = None
return title
#!/usr/bin/env python
import cgi
import cgitb; cgitb.enable()
import sys
sys.path.append("..")
from data import db_gw
from fmt import fmt_rule
import web
print web.head(jsextra = ("js/jquery.treeview.js",), jsexec="$('#ruletree').treeview();", cssextra=("css/jquery.treeview.css",))
body = []
args = cgi.FieldStorage()
if 'fw' not in args:
raise ValueError("Missing fw parameter")
fw = args['fw'].value
db = db_gw(fw = fw)
body.append("<h1>%s</h1>"%fw)
term_targets = set(['ACCEPT', 'REJECT', 'DROP', 'LOG'])
rules = {}
columns = ['chain.id', 'chain.name', 'for_user.name', 'proto.name', 'src.id', 'src.name', 'sport.port', 'sport.endport', 'dst.id', 'dst.name','dport.port', 'dport.endport', 'target.name', 'rules.additional', 'target.id']
for row in db.get_rules(columns = columns):
r = {}
for i in range(len(columns)):
r[columns[i]] = row[i]
r['formatted_rule'] = row[-2]
if r['chain.id'] not in rules:
rules[r['chain.id']] = []
rules[r['chain.id']].append(r)
