def testInvalidEncodingNameIsError(self): t = self.startNormalServer(3400, 200, "text/html; charset=ksfhsdlfhhfsd", u"täti".encode("ISO-8859-1")) try: getHtmlSafely("http://127.0.0.1:3400") except HttpException, e: self.failUnless(ERR_INVALID_ENCODING in e.parameter) return
def testEncodingMismatchIsError(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", u"täti".encode('ISO-8859-1')) try: getHtmlSafely("http://127.0.0.1:3400") except HttpException, e: self.failUnless(ERR_INVALID_ENCODING in e.parameter) return
def testMaxRedirectsIsReached(self): t = self.startRedirectNServer(3400, 4, 4) try: getHtmlSafely("http://localhost:3400/1") except HttpException, e: self.failUnless(ERR_TOO_MANY_REDIRECTS in e.parameter) self.assertThreadExitsNormally(t) return
def testMaxRedirectsIsReached(self): t = self.startRedirectNServer(3400, 4, 4) try: getHtmlSafely("http://localhost:3400/1")[1] except HttpException, e: self.failUnless(ERR_TOO_MANY_REDIRECTS in e.parameter) self.assertThreadExitsNormally(t) return
def testRedirectToFtpIsNotAllowed(self): def getFunct(slf, tester): slf.send_response(302) # Found slf.send_header("Location", "ftp://localhost/") slf.end_headers() t = self.startServer(3400, getFunct, 1) try: getHtmlSafely("http://localhost:3400/")[1] except HttpException, e: self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter) self.assertThreadExitsNormally(t) return
def testNotFound(self): def getFunct(slf, tester): slf.send_response(404) slf.end_headers() slf.wfile.write("kissa") t = self.startServer(3400, getFunct, 1) try: getHtmlSafely("http://localhost:3400/")[1] except HttpException, e: self.failUnless(ERR_NOT_FOUND in e.parameter) self.assertThreadExitsNormally(t) return
def checkPage(url, dictionary, clientIp, requestHeaders, checkForMaybeErrors): log("checkPage: " + url.encode("UTF-8")) if dictionary not in _voikko: return u"" v = _voikko[dictionary] try: html = getHtmlSafely(url.encode('UTF-8'), clientIp, requestHeaders) segments = parseHtml(html) res = u"Analyysi sivusta " + escape(url) + u"<br />" v.setAcceptUnfinishedParagraphsInGc(True) for segment in segments: segmentClass = None checkGrammar = True if segment[0] == SEGMENT_TYPE_HEADING: v.setAcceptTitlesInGc(True) v.setAcceptBulletedListsInGc(False) segmentClass = u"webvoikkoH" elif segment[0] == SEGMENT_TYPE_LIST_ITEM: v.setAcceptTitlesInGc(False) v.setAcceptBulletedListsInGc(True) segmentClass = u"webvoikkoLi" elif segment[0] == SEGMENT_TYPE_PARAGRAPH: v.setAcceptTitlesInGc(False) v.setAcceptBulletedListsInGc(False) segmentClass = u"webvoikkoP" elif segment[0] == SEGMENT_TYPE_OTHER: checkGrammar = False segmentClass = u"webvoikkoO" res = res + u"<p class='" + segmentClass + u"'>" + doSpell(segment[1], v, checkGrammar, checkForMaybeErrors) + u"</p>" return res except HttpException, e: return u"Sivua %s ei voitu hakea: %s" % (escape(url), e.parameter)
def checkPage(url, dictionary, clientIp, requestHeaders): log("checkPage: " + url.encode("UTF-8")) if dictionary not in _voikko: return u"" v = _voikko[dictionary] try: html = getHtmlSafely(url.encode('UTF-8'), clientIp, requestHeaders) segments = parseHtml(html) res = u"Analyysi sivusta " + escape(url) + u"<br />" v.setAcceptUnfinishedParagraphsInGc(True) for segment in segments: segmentClass = None checkGrammar = True if segment[0] == SEGMENT_TYPE_HEADING: v.setAcceptTitlesInGc(True) v.setAcceptBulletedListsInGc(False) segmentClass = u"webvoikkoH" elif segment[0] == SEGMENT_TYPE_LIST_ITEM: v.setAcceptTitlesInGc(False) v.setAcceptBulletedListsInGc(True) segmentClass = u"webvoikkoLi" elif segment[0] == SEGMENT_TYPE_PARAGRAPH: v.setAcceptTitlesInGc(False) v.setAcceptBulletedListsInGc(False) segmentClass = u"webvoikkoP" elif segment[0] == SEGMENT_TYPE_OTHER: checkGrammar = False segmentClass = u"webvoikkoO" res = res + u"<p class='" + segmentClass + u"'>" + doSpell( segment[1], v, checkGrammar) + u"</p>" return res except HttpException, e: return u"Sivua %s ei voitu hakea: %s" % (escape(url), e.parameter)
def testGetHtmlSafely(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", "kissa") (contentType, content) = getHtmlSafely("http://localhost:3400") self.assertEquals("text/html; charset=UTF-8", contentType) self.assertEquals(u"kissa", content) self.assertThreadExitsNormally(t)
def testPreviousProxiesAreRetained(self): def getFunct(slf, tester): tester.assertEquals("123.123.123.123, 93.35.124.35", slf.headers["X-Forwarded-For"]) slf.send_response(200) slf.end_headers() slf.wfile.write("kissa") t = self.startServer(3400, getFunct, 1) html = getHtmlSafely("http://localhost:3400/", "93.35.124.35", ["X-Something: sfsf\n", "X-Forwarded-For: 123.123.123.123\n"])[1] self.assertEquals(u"kissa", html) self.assertThreadExitsNormally(t)
def testPreviousProxiesAreRetained(self): def getFunct(slf, tester): tester.assertEquals("123.123.123.123, 93.35.124.35", slf.headers["X-Forwarded-For"]) slf.send_response(200) slf.end_headers() slf.wfile.write("kissa") t = self.startServer(3400, getFunct, 1) html = getHtmlSafely( "http://localhost:3400/", "93.35.124.35", ["X-Something: sfsf\n", "X-Forwarded-For: 123.123.123.123\n"])[1] self.assertEquals(u"kissa", html) self.assertThreadExitsNormally(t)
def testFollowRedirects(self): t = self.startRedirectServer(3400) self.assertEquals(u"hirvi", getHtmlSafely("http://localhost:3400/1")) self.assertThreadExitsNormally(t)
def testUnknownHost(self): try: getHtmlSafely("http://ksdjaksd.ajhsdjas.ajshd") except HttpException, e: self.failUnless(pycurl.E_COULDNT_RESOLVE_HOST in e.parameter) return
def testLocalFilesAreNotReadWithScheme(self): try: getHtmlSafely("file:/etc/passwd") except HttpException, e: return
def testForbiddenScheme(self): try: getHtmlSafely("ftp://localhost") except HttpException, e: self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter) return
def testLatin1ContentIsParsedIfNoEncodingSpecified(self): t = self.startNormalServer(3400, 200, "text/html", u"täti".encode("ISO-8859-1")) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400"))
def testLatin1EncodingFromMetaTagAlternateOrder(self): response = u'<html>\n<head>\n<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type">\n</head><body>täti</body></html>' t = self.startNormalServer(3400, 200, "text/html", response.encode("ISO-8859-1")) self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400"))
def testSchemeIsOptional(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", "kissa") self.assertEquals(u"kissa", getHtmlSafely("127.0.0.1:3400")) self.assertThreadExitsNormally(t)
def testFollowRedirects(self): t = self.startRedirectServer(3400) self.assertEquals(u"hirvi", getHtmlSafely("http://localhost:3400/1")[1]) self.assertThreadExitsNormally(t)
def testFtpIsNotAllowed(self): try: getHtmlSafely("ftp:localhost") except HttpException, e: return
def testLatin1ContentIsParsedIfNoEncodingSpecified(self): t = self.startNormalServer(3400, 200, "text/html", u"täti".encode('ISO-8859-1')) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1])
def testFtpIsNotAllowedWithProtocolAutoDetect(self): try: getHtmlSafely("ftp.funet.fi") except HttpException, e: self.failUnless(ERR_FORBIDDEN_SCHEME in e.parameter) return
def testLocalFilesAreNotRead(self): try: getHtmlSafely("/etc/passwd") except HttpException, e: return
def testSchemeIsOptional(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", "kissa") self.assertEquals(u"kissa", getHtmlSafely("127.0.0.1:3400")[1]) self.assertThreadExitsNormally(t)
def testGetHtmlSafely(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", "kissa") self.assertEquals(u"kissa", getHtmlSafely("http://localhost:3400")) self.assertThreadExitsNormally(t)
def testUtf8EncodingFromContentType(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", u"täti".encode('UTF-8')) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1]) self.assertThreadExitsNormally(t)
def testLatin1EncodingFromContentType(self): t = self.startNormalServer(3400, 200, "text/html; charset=ISO-8859-1", u"täti".encode("ISO-8859-1")) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")) self.assertThreadExitsNormally(t)
def testLatin1EncodingFromMetaTag(self): response = u'<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">\n</head><body>täti</body></html>' t = self.startNormalServer(3400, 200, "text/html", response.encode('ISO-8859-1')) self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromMetaTagSingleQuoteBeforeHtml(self): response = u'\'<html>\n<head>\n<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=ISO-8859-1">\n</head><body>täti</body></html>' t = self.startNormalServer(3400, 200, "text/html", response.encode("ISO-8859-1")) self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400"))
def testUtf8ContentIsParsedIfNoEncodingSpecified(self): t = self.startNormalServer(3400, 200, "text/html", u"täti".encode('UTF-8')) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromMetaTagSingleQuoteBeforeHtml(self): response = u'\'<html>\n<head>\n<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=ISO-8859-1">\n</head><body>täti</body></html>' t = self.startNormalServer(3400, 200, "text/html", response.encode('ISO-8859-1')) self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromMetaTagAlternateOrder(self): response = u'<html>\n<head>\n<meta content="text/html; charset=ISO-8859-1" http-equiv="content-type">\n</head><body>täti</body></html>' t = self.startNormalServer(3400, 200, "text/html", response.encode('ISO-8859-1')) self.assertEquals(response, getHtmlSafely("http://127.0.0.1:3400")[1])
def testLatin1EncodingFromContentType(self): t = self.startNormalServer(3400, 200, "text/html; charset=ISO-8859-1", u"täti".encode('ISO-8859-1')) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")[1]) self.assertThreadExitsNormally(t)
def testUtf8EncodingFromContentType(self): t = self.startNormalServer(3400, 200, "text/html; charset=UTF-8", u"täti".encode('UTF-8')) self.assertEquals(u"täti", getHtmlSafely("http://127.0.0.1:3400")) self.assertThreadExitsNormally(t)