def testInvalidEncodingNameIsError(self):
t = self.startNormalServer(3400, 200, "text/html; charset=ksfhsdlfhhfsd", u"täti".encode("ISO-8859-1"))
try:
getHtmlSafely("http://127.0.0.1:3400")
except HttpException, e:
self.failUnless(ERR_INVALID_ENCODING in e.parameter)
return
def testEncodingMismatchIsError(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", u"täti".encode('ISO-8859-1'))
try:
getHtmlSafely("http://127.0.0.1:3400")
except HttpException, e:
self.failUnless(ERR_INVALID_ENCODING in e.parameter)
return
def testMaxRedirectsIsReached(self):
t = self.startRedirectNServer(3400, 4, 4)
try:
getHtmlSafely("http://localhost:3400/1")
except HttpException, e:
self.failUnless(ERR_TOO_MANY_REDIRECTS in e.parameter)
self.assertThreadExitsNormally(t)
return
def testMaxRedirectsIsReached(self):
t = self.startRedirectNServer(3400, 4, 4)
try:
getHtmlSafely("http://localhost:3400/1")[1]
except HttpException, e:
self.failUnless(ERR_TOO_MANY_REDIRECTS in e.parameter)
self.assertThreadExitsNormally(t)
return
def testEncodingMismatchIsError(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8",
u"täti".encode('ISO-8859-1'))
try:
getHtmlSafely("http://127.0.0.1:3400")
except HttpException, e:
self.failUnless(ERR_INVALID_ENCODING in e.parameter)
return
def testRedirectToFtpIsNotAllowed(self):
def getFunct(slf, tester):
slf.send_response(302) # Found
slf.send_header("Location", "ftp://localhost/")
slf.end_headers()
t = self.startServer(3400, getFunct, 1)
try:
getHtmlSafely("http://localhost:3400/")[1]
except HttpException, e:
self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter)
self.assertThreadExitsNormally(t)
return
def testNotFound(self):
def getFunct(slf, tester):
slf.send_response(404)
slf.end_headers()
slf.wfile.write("kissa")
t = self.startServer(3400, getFunct, 1)
try:
getHtmlSafely("http://localhost:3400/")[1]
except HttpException, e:
self.failUnless(ERR_NOT_FOUND in e.parameter)
self.assertThreadExitsNormally(t)
return
def testNotFound(self):
def getFunct(slf, tester):
slf.send_response(404)
slf.end_headers()
slf.wfile.write("kissa")
t = self.startServer(3400, getFunct, 1)
try:
getHtmlSafely("http://localhost:3400/")[1]
except HttpException, e:
self.failUnless(ERR_NOT_FOUND in e.parameter)
self.assertThreadExitsNormally(t)
return
def testRedirectToFtpIsNotAllowed(self):
def getFunct(slf, tester):
slf.send_response(302) # Found
slf.send_header("Location", "ftp://localhost/")
slf.end_headers()
t = self.startServer(3400, getFunct, 1)
try:
getHtmlSafely("http://localhost:3400/")[1]
except HttpException, e:
self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter)
self.assertThreadExitsNormally(t)
return
def checkPage(url, dictionary, clientIp, requestHeaders, checkForMaybeErrors):
log("checkPage: " + url.encode("UTF-8"))
if dictionary not in _voikko:
return u""
v = _voikko[dictionary]
try:
html = getHtmlSafely(url.encode('UTF-8'), clientIp, requestHeaders)
segments = parseHtml(html)
res = u"Analyysi sivusta " + escape(url) + u"<br />"
v.setAcceptUnfinishedParagraphsInGc(True)
for segment in segments:
segmentClass = None
checkGrammar = True
if segment[0] == SEGMENT_TYPE_HEADING:
v.setAcceptTitlesInGc(True)
v.setAcceptBulletedListsInGc(False)
segmentClass = u"webvoikkoH"
elif segment[0] == SEGMENT_TYPE_LIST_ITEM:
v.setAcceptTitlesInGc(False)
v.setAcceptBulletedListsInGc(True)
segmentClass = u"webvoikkoLi"
elif segment[0] == SEGMENT_TYPE_PARAGRAPH:
v.setAcceptTitlesInGc(False)
v.setAcceptBulletedListsInGc(False)
segmentClass = u"webvoikkoP"
elif segment[0] == SEGMENT_TYPE_OTHER:
checkGrammar = False
segmentClass = u"webvoikkoO"
res = res + u"<p class='" + segmentClass + u"'>" + doSpell(segment[1], v, checkGrammar, checkForMaybeErrors) + u"</p>"
return res
except HttpException, e:
return u"Sivua %s ei voitu hakea: %s" % (escape(url), e.parameter)
def checkPage(url, dictionary, clientIp, requestHeaders):
log("checkPage: " + url.encode("UTF-8"))
if dictionary not in _voikko:
return u""
v = _voikko[dictionary]
try:
html = getHtmlSafely(url.encode('UTF-8'), clientIp, requestHeaders)
segments = parseHtml(html)
res = u"Analyysi sivusta " + escape(url) + u"<br />"
v.setAcceptUnfinishedParagraphsInGc(True)
for segment in segments:
segmentClass = None
checkGrammar = True
if segment[0] == SEGMENT_TYPE_HEADING:
v.setAcceptTitlesInGc(True)
v.setAcceptBulletedListsInGc(False)
segmentClass = u"webvoikkoH"
elif segment[0] == SEGMENT_TYPE_LIST_ITEM:
v.setAcceptTitlesInGc(False)
v.setAcceptBulletedListsInGc(True)
segmentClass = u"webvoikkoLi"
elif segment[0] == SEGMENT_TYPE_PARAGRAPH:
v.setAcceptTitlesInGc(False)
v.setAcceptBulletedListsInGc(False)
segmentClass = u"webvoikkoP"
elif segment[0] == SEGMENT_TYPE_OTHER:
checkGrammar = False
segmentClass = u"webvoikkoO"
res = res + u"<p class='" + segmentClass + u"'>" + doSpell(
segment[1], v, checkGrammar) + u"</p>"
return res
except HttpException, e:
return u"Sivua %s ei voitu hakea: %s" % (escape(url), e.parameter)
def testGetHtmlSafely(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8",
"kissa")
(contentType, content) = getHtmlSafely("http://localhost:3400")
self.assertEquals("text/html; charset=UTF-8", contentType)
self.assertEquals(u"kissa", content)
self.assertThreadExitsNormally(t)
def testPreviousProxiesAreRetained(self):
def getFunct(slf, tester):
tester.assertEquals("123.123.123.123, 93.35.124.35", slf.headers["X-Forwarded-For"])
slf.send_response(200)
slf.end_headers()
slf.wfile.write("kissa")
t = self.startServer(3400, getFunct, 1)
html = getHtmlSafely("http://localhost:3400/", "93.35.124.35", ["X-Something: sfsf\n", "X-Forwarded-For: 123.123.123.123\n"])[1]
self.assertEquals(u"kissa", html)
self.assertThreadExitsNormally(t)
def testPreviousProxiesAreRetained(self):
def getFunct(slf, tester):
tester.assertEquals("123.123.123.123, 93.35.124.35",
slf.headers["X-Forwarded-For"])
slf.send_response(200)
slf.end_headers()
slf.wfile.write("kissa")
t = self.startServer(3400, getFunct, 1)
html = getHtmlSafely(
"http://localhost:3400/", "93.35.124.35",
["X-Something: sfsf\n", "X-Forwarded-For: 123.123.123.123\n"])[1]
self.assertEquals(u"kissa", html)
self.assertThreadExitsNormally(t)
def testFollowRedirects(self):
t = self.startRedirectServer(3400)
self.assertEquals(u"hirvi", getHtmlSafely("http://localhost:3400/1"))
self.assertThreadExitsNormally(t)
def testUnknownHost(self):
try:
getHtmlSafely("http://ksdjaksd.ajhsdjas.ajshd")
except HttpException, e:
self.failUnless(pycurl.E_COULDNT_RESOLVE_HOST in e.parameter)
return
def testLocalFilesAreNotReadWithScheme(self):
try:
getHtmlSafely("file:/etc/passwd")
except HttpException, e:
return
def testForbiddenScheme(self):
try:
getHtmlSafely("ftp://localhost")
except HttpException, e:
self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter)
return
def testLatin1ContentIsParsedIfNoEncodingSpecified(self):
t = self.startNormalServer(3400, 200, "text/html", u"täti".encode("ISO-8859-1"))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400"))
def testLatin1EncodingFromMetaTagAlternateOrder(self):
response = u'<html>\n<head>\n<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type">\n</head><body>täti</body></html>'
t = self.startNormalServer(3400, 200, "text/html", response.encode("ISO-8859-1"))
self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400"))
def testSchemeIsOptional(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", "kissa")
self.assertEquals(u"kissa", getHtmlSafely("127.0.0.1:3400"))
self.assertThreadExitsNormally(t)
def testFollowRedirects(self):
t = self.startRedirectServer(3400)
self.assertEquals(u"hirvi",
getHtmlSafely("http://localhost:3400/1")[1])
self.assertThreadExitsNormally(t)
def testFtpIsNotAllowed(self):
try:
getHtmlSafely("ftp:localhost")
except HttpException, e:
return
def testLatin1ContentIsParsedIfNoEncodingSpecified(self):
t = self.startNormalServer(3400, 200, "text/html",
u"täti".encode('ISO-8859-1'))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1])
def testFtpIsNotAllowedWithProtocolAutoDetect(self):
try:
getHtmlSafely("ftp.funet.fi")
except HttpException, e:
self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter)
return
def testFtpIsNotAllowed(self):
try:
getHtmlSafely("ftp:localhost")
except HttpException, e:
return
def testLocalFilesAreNotRead(self):
try:
getHtmlSafely("/etc/passwd")
except HttpException, e:
return
def testLocalFilesAreNotReadWithScheme(self):
try:
getHtmlSafely("file:/etc/passwd")
except HttpException, e:
return
def testSchemeIsOptional(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8",
"kissa")
self.assertEquals(u"kissa", getHtmlSafely("127.0.0.1:3400")[1])
self.assertThreadExitsNormally(t)
def testGetHtmlSafely(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", "kissa")
self.assertEquals(u"kissa", getHtmlSafely("http://localhost:3400"))
self.assertThreadExitsNormally(t)
def testUtf8EncodingFromContentType(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", u"täti".encode('UTF-8'))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1])
self.assertThreadExitsNormally(t)
def testLatin1EncodingFromContentType(self):
t = self.startNormalServer(3400, 200, "text/html; charset=ISO-8859-1", u"täti".encode("ISO-8859-1"))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400"))
self.assertThreadExitsNormally(t)
def testLatin1EncodingFromMetaTag(self):
response = u'<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">\n</head><body>täti</body></html>'
t = self.startNormalServer(3400, 200, "text/html", response.encode('ISO-8859-1'))
self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromMetaTagSingleQuoteBeforeHtml(self):
response = u'\'<html>\n<head>\n<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=ISO-8859-1">\n</head><body>täti</body></html>'
t = self.startNormalServer(3400, 200, "text/html", response.encode("ISO-8859-1"))
self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400"))
def testUtf8ContentIsParsedIfNoEncodingSpecified(self):
t = self.startNormalServer(3400, 200, "text/html", u"täti".encode('UTF-8'))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromMetaTagSingleQuoteBeforeHtml(self):
response = u'\'<html>\n<head>\n<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=ISO-8859-1">\n</head><body>täti</body></html>'
t = self.startNormalServer(3400, 200, "text/html",
response.encode('ISO-8859-1'))
self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromMetaTagAlternateOrder(self):
response = u'<html>\n<head>\n<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type">\n</head><body>täti</body></html>'
t = self.startNormalServer(3400, 200, "text/html",
response.encode('ISO-8859-1'))
self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400")[1])
def testUnknownHost(self):
try:
getHtmlSafely("http://ksdjaksd.ajhsdjas.ajshd")
except HttpException, e:
self.failUnless(pycurl.E_COULDNT_RESOLVE_HOST in e.parameter)
return
def testLatin1EncodingFromContentType(self):
t = self.startNormalServer(3400, 200, "text/html; charset=ISO-8859-1",
u"täti".encode('ISO-8859-1'))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1])
self.assertThreadExitsNormally(t)
def testLocalFilesAreNotRead(self):
try:
getHtmlSafely("/etc/passwd")
except HttpException, e:
return
def testForbiddenScheme(self):
try:
getHtmlSafely("ftp://localhost")
except HttpException, e:
self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter)
return
def testFtpIsNotAllowedWithProtocolAutoDetect(self):
try:
getHtmlSafely("ftp.funet.fi")
except HttpException, e:
self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter)
return
def testUtf8EncodingFromContentType(self):
t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8",
u"täti".encode('UTF-8'))
self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400"))
self.assertThreadExitsNormally(t)
