def test_safename():
cases = (
('http://example.org/fred/?a=b',
'example.org,fred,a=b,58489f63a7a83c3b7794a6a398ee8b1f'),
('http://example.org/fred?/a=b',
'example.org,fred,a=b,8c5946d56fec453071f43329ff0be46b'),
('http://www.example.org/fred?/a=b',
'www.example.org,fred,a=b,499c44b8d844a011b67ea2c015116968'),
('https://www.example.org/fred?/a=b',
'www.example.org,fred,a=b,692e843a333484ce0095b070497ab45d'),
(httplib2.urlnorm('http://WWW')[-1],
httplib2.safename(httplib2.urlnorm('http://www')[-1])),
(u'http://\u2304.org/fred/?a=b',
'xn--http,-4y1d.org,fred,a=b,579924c35db315e5a32e3d9963388193'),
)
for a, b in cases:
assert httplib2.safename(a) == b
assert httplib2.safename('http://www') != httplib2.safename('https://www')
# Test the max length limits
uri = 'http://' + ('w' * 200) + '.org'
uri2 = 'http://' + ('w' * 201) + '.org'
assert httplib2.safename(uri) != httplib2.safename(uri2)
# Max length should be 200 + 1 (',') + 32
assert len(httplib2.safename(uri2)) == 233
assert len(httplib2.safename(uri)) == 233
def test_epgguide(self):
cache_dir = os.getcwd() + "/.epguide"
# os.makedirs(cache_dir)
print cache_dir
cache = FileCache(cache_dir)
h = Http(cache=cache)
user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0"
httplib2.debuglevel = 255
resp, content = h.request(
"http://www.teleman.pl/program-tv/stacje/TVP-2",
headers={'user-agent': user_agent})
print "content:"
print content
print "response:"
print resp
print "fromcache:" + str(resp.fromcache)
print "status:" + str(resp.status)
#url = "http://www.teleman.pl/program-tv/stacje/TVP-2"
url = "http://www.teleman.pl/tv/Dr-House-7-152-885990"
resp, content = h.request(url, headers={'user-agent': user_agent})
print "fromcache:" + str(resp.fromcache)
print "status:" + str(resp.status)
safe = httplib2.safename(url)
print "safe:" + safe
cached_value = cache.get(url)
info, cached = cached_value.split('\r\n\r\n', 1)
print "===="
print content
print "===="
print cached
print "===="
self.assertEqual(content, cached)
def test_epgguide(self):
cache_dir = os.getcwd() + "/.epguide"
# os.makedirs(cache_dir)
print cache_dir
cache = FileCache(cache_dir)
h = Http(cache = cache)
user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0"
httplib2.debuglevel = 255
resp, content = h.request("http://www.teleman.pl/program-tv/stacje/TVP-2", headers={'user-agent':user_agent})
print "content:"
print content
print "response:"
print resp
print "fromcache:" + str(resp.fromcache)
print "status:" + str(resp.status)
#url = "http://www.teleman.pl/program-tv/stacje/TVP-2"
url = "http://www.teleman.pl/tv/Dr-House-7-152-885990"
resp, content = h.request(url, headers={'user-agent':user_agent})
print "fromcache:" + str(resp.fromcache)
print "status:" + str(resp.status)
safe = httplib2.safename(url)
print "safe:" + safe
cached_value = cache.get(url)
info, cached = cached_value.split('\r\n\r\n', 1)
print "===="
print content
print "===="
print cached
print "===="
self.assertEqual(content, cached)
def testGet304(self):
# Test that we use ETags properly to validate our cache
uri = urllib.parse.urljoin(base, "304/test_etag.txt")
(response, content) = self.http.request(uri, "GET")
self.assertNotEqual(response['etag'], "")
(response, content) = self.http.request(uri, "GET")
(response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'must-revalidate'})
self.assertEqual(response.status, 200)
self.assertEqual(response.fromcache, True)
cache_file_name = os.path.join(cacheDirName, httplib2.safename(httplib2.urlnorm(uri)[-1]))
f = open(cache_file_name, "r")
status_line = f.readline()
f.close()
self.assertTrue(status_line.startswith("status:"))
(response, content) = self.http.request(uri, "HEAD")
self.assertEqual(response.status, 200)
self.assertEqual(response.fromcache, True)
(response, content) = self.http.request(uri, "GET", headers = {'range': 'bytes=0-0'})
self.assertEqual(response.status, 206)
self.assertEqual(response.fromcache, False)
def test(self):
# Test that different URIs end up generating different safe names
self.assertEqual( "example.org,fred,a=b,58489f63a7a83c3b7794a6a398ee8b1f", httplib2.safename("http://example.org/fred/?a=b"))
self.assertEqual( "example.org,fred,a=b,8c5946d56fec453071f43329ff0be46b", httplib2.safename("http://example.org/fred?/a=b"))
self.assertEqual( "www.example.org,fred,a=b,499c44b8d844a011b67ea2c015116968", httplib2.safename("http://www.example.org/fred?/a=b"))
self.assertEqual( httplib2.safename(httplib2.urlnorm("http://www")[-1]), httplib2.safename(httplib2.urlnorm("http://WWW")[-1]))
self.assertEqual( "www.example.org,fred,a=b,692e843a333484ce0095b070497ab45d", httplib2.safename("https://www.example.org/fred?/a=b"))
self.assertNotEqual( httplib2.safename("http://www"), httplib2.safename("https://www"))
# Test the max length limits
uri = "http://" + ("w" * 200) + ".org"
uri2 = "http://" + ("w" * 201) + ".org"
self.assertNotEqual( httplib2.safename(uri2), httplib2.safename(uri))
# Max length should be 200 + 1 (",") + 32
self.assertEqual(233, len(httplib2.safename(uri2)))
self.assertEqual(233, len(httplib2.safename(uri)))
# Unicode
if sys.version_info >= (2,3):
self.assertEqual( "xn--http,-4y1d.org,fred,a=b,579924c35db315e5a32e3d9963388193", httplib2.safename("http://\u2304.org/fred/?a=b"))
def test_etag_used():
# Test that we use ETags properly to validate our cache
cache_path = tests.get_cache_path()
http = httplib2.Http(cache=cache_path)
response_kwargs = dict(
add_date=True,
add_etag=True,
body=b'something',
headers={
'cache-control': 'public,max-age=300',
},
)
def handler(request):
if request.headers.get('range'):
return tests.http_response_bytes(status=206, **response_kwargs)
return tests.http_response_bytes(**response_kwargs)
with tests.server_request(handler, request_count=2) as uri:
response, _ = http.request(uri,
'GET',
headers={'accept-encoding': 'identity'})
assert response['etag'] == '"437b930db84b8079c2dd804a71936b5f"'
http.request(uri, 'GET', headers={'accept-encoding': 'identity'})
response, _ = http.request(
uri,
'GET',
headers={
'accept-encoding': 'identity',
'cache-control': 'must-revalidate'
},
)
assert response.status == 200
assert response.fromcache
# TODO: API to read cache item, at least internal to tests
cache_file_name = os.path.join(
cache_path, httplib2.safename(httplib2.urlnorm(uri)[-1]))
with open(cache_file_name, 'r') as f:
status_line = f.readline()
assert status_line.startswith("status:")
response, content = http.request(
uri, 'HEAD', headers={'accept-encoding': 'identity'})
assert response.status == 200
assert response.fromcache
response, content = http.request(uri,
'GET',
headers={
'accept-encoding': 'identity',
'range': 'bytes=0-0'
})
assert response.status == 206
assert not response.fromcache
def wrapped(*args):
args = list(args) #tuple to list
args[1] = httplib2.safename(args[1])
# if len(args[1])>200:
# args = list(args)
# args[1] = md5(args[1]).hexdigest()
# elif isinstance(args[1], unicode):
# args = list(args)
# args[1] = args[1].encode('utf-8')
return func(*args)
def request(self, uri, method='GET', body=None, headers=None,
redirections=5, connection_type=None):
file = os.path.join(HTTP_DATA_DIR, httplib2.safename(uri))
print file
if os.path.exists(file):
response = message_from_file(open(file))
body = response.get_payload()
headers = httplib2.Response(response)
return (headers, body)
else:
return (httplib2.Response({"status": "404"}), "")
def request(self, uri, method='GET', body=None, headers=None,
redirections=5, connection_type=None):
file = os.path.join(HTTP_DATA_DIR, httplib2.safename(uri))
if os.path.exists(file):
response = message_from_file(open(file))
headers = httplib2.Response(response)
body = bytes(response.get_payload(), charset_from_headers(headers))
return (headers, body)
else:
return (httplib2.Response({"status": "404"}),
"Resource %r unavailable from test data store" % file)
def test_safename2():
assert httplib2.safename("http://www") != httplib2.safename("https://www")
# Test the max length limits
uri = "http://" + ("w" * 200) + ".org"
uri2 = "http://" + ("w" * 201) + ".org"
assert httplib2.safename(uri) != httplib2.safename(uri2)
# Max length should be 90 + 1 (',') + 32 = 123
assert len(httplib2.safename(uri2)) == 123
assert len(httplib2.safename(uri)) == 123
def test_safename2():
assert httplib2.safename("http://www") != httplib2.safename("https://www")
# Test the max length limits
uri = "http://" + ("w" * 200) + ".org"
uri2 = "http://" + ("w" * 201) + ".org"
assert httplib2.safename(uri) != httplib2.safename(uri2)
# Max length should be 90 + 1 (',') + 32 = 123
assert len(httplib2.safename(uri2)) == 123
assert len(httplib2.safename(uri)) == 123
def test_safename2():
assert httplib2.safename('http://www') != httplib2.safename('https://www')
# Test the max length limits
uri = 'http://' + ('w' * 200) + '.org'
uri2 = 'http://' + ('w' * 201) + '.org'
assert httplib2.safename(uri) != httplib2.safename(uri2)
# Max length should be 90 + 1 (',') + 32 = 123
assert len(httplib2.safename(uri2)) == 123
assert len(httplib2.safename(uri)) == 123
def test_safename2():
assert httplib2.safename('http://www') != httplib2.safename('https://www')
# Test the max length limits
uri = 'http://' + ('w' * 200) + '.org'
uri2 = 'http://' + ('w' * 201) + '.org'
assert httplib2.safename(uri) != httplib2.safename(uri2)
# Max length should be 90 + 1 (',') + 32 = 123
assert len(httplib2.safename(uri2)) == 123
assert len(httplib2.safename(uri)) == 123
def test_etag_used():
# Test that we use ETags properly to validate our cache
cache_path = tests.get_cache_path()
http = httplib2.Http(cache=cache_path)
response_kwargs = dict(
add_date=True,
add_etag=True,
body=b"something",
headers={"cache-control": "public,max-age=300"},
)
def handler(request):
if request.headers.get("range"):
return tests.http_response_bytes(status=206, **response_kwargs)
return tests.http_response_bytes(**response_kwargs)
with tests.server_request(handler, request_count=2) as uri:
response, _ = http.request(uri, "GET", headers={"accept-encoding": "identity"})
assert response["etag"] == '"437b930db84b8079c2dd804a71936b5f"'
http.request(uri, "GET", headers={"accept-encoding": "identity"})
response, _ = http.request(
uri,
"GET",
headers={"accept-encoding": "identity", "cache-control": "must-revalidate"},
)
assert response.status == 200
assert response.fromcache
# TODO: API to read cache item, at least internal to tests
cache_file_name = os.path.join(
cache_path, httplib2.safename(httplib2.urlnorm(uri)[-1])
)
with open(cache_file_name, "r") as f:
status_line = f.readline()
assert status_line.startswith("status:")
response, content = http.request(
uri, "HEAD", headers={"accept-encoding": "identity"}
)
assert response.status == 200
assert response.fromcache
response, content = http.request(
uri, "GET", headers={"accept-encoding": "identity", "range": "bytes=0-0"}
)
assert response.status == 206
assert not response.fromcache
def request_mock(uri, method='GET', body=None, headers=None,
redirections=5, connection_type=None):
"""Http mock side effect that returns saved entries.
Implementation tests should never span network boundaries.
"""
file = os.path.join("tests/data", httplib2.safename(uri))
if os.path.exists(file):
response = message_from_file(open(file))
headers = httplib2.Response(response)
body = bytes(response.get_payload(), charset_from_headers(headers))
return (headers, body)
else:
return (httplib2.Response({"status": "404"}),
"Resource %r unavailable from test data store" % file)
def request_mock(uri,
method='GET',
body=None,
headers=None,
redirections=5,
connection_type=None):
"""Http mock side effect that returns saved entries.
Implementation tests should never span network boundaries.
"""
file = os.path.join("tests/data", httplib2.safename(uri))
if os.path.exists(file):
response = message_from_file(open(file))
headers = httplib2.Response(response)
body = bytes(response.get_payload(), charset_from_headers(headers))
return (headers, body)
else:
return (httplib2.Response({"status": "404"}),
"Resource %r unavailable from test data store" % file)
def downloadLocal(url):
"""Download the url to a local file and return the name of the local
file.
Ideally this would be cache-aware. httlib2 is cache-aware, but it
forces me to load the entire response into memory. If we got
streaming support in httplib2 then we could use it.
"""
localDir = os.path.join(baseDir, "local")
if not os.path.exists(localDir): os.makedirs(localDir)
local = os.path.join(localDir, httplib2.safename(url))
if not os.path.exists(local):
try:
print "Getting", url, "as", local
logging.info("Getting %s as %s", url, local)
urllib.urlretrieve(url, local, progress)
except:
if (os.path.exists(local)): os.unlink(local)
raise
return local
def test_safename(data):
result = httplib2.safename(data[0])
assert result == data[1]
),
(
"http://example.org/fred?/a=b",
"example.orgfreda=b,8c5946d56fec453071f43329ff0be46b",
),
(
"http://www.example.org/fred?/a=b",
"www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968",
),
(
"https://www.example.org/fred?/a=b",
"www.example.orgfreda=b,692e843a333484ce0095b070497ab45d",
),
(
httplib2.urlnorm("http://WWW")[-1],
httplib2.safename(httplib2.urlnorm("http://www")[-1]),
),
(
u"http://\u2304.org/fred/?a=b",
".orgfreda=b,ecaf0f97756c0716de76f593bd60a35e",
),
(
"normal-resource-name.js",
"normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54",
),
(
"foo://dom/path/brath/carapath",
"dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3",
),
("with/slash", "withslash,17cc656656bb8ce2411bd41ead56d176"),
(
def test_safename(data):
result = httplib2.safename(data[0])
assert result == data[1]
pass
@pytest.mark.parametrize(
'data', (
('', ',d41d8cd98f00b204e9800998ecf8427e'),
('http://example.org/fred/?a=b',
'example.orgfreda=b,58489f63a7a83c3b7794a6a398ee8b1f'),
('http://example.org/fred?/a=b',
'example.orgfreda=b,8c5946d56fec453071f43329ff0be46b'),
('http://www.example.org/fred?/a=b',
'www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968'),
('https://www.example.org/fred?/a=b',
'www.example.orgfreda=b,692e843a333484ce0095b070497ab45d'),
(httplib2.urlnorm('http://WWW')[-1],
httplib2.safename(httplib2.urlnorm('http://www')[-1])),
(u'http://\u2304.org/fred/?a=b',
'.orgfreda=b,ecaf0f97756c0716de76f593bd60a35e'),
('normal-resource-name.js', 'normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54'),
('foo://dom/path/brath/carapath', 'dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3'),
('with/slash', 'withslash,17cc656656bb8ce2411bd41ead56d176'),
('thisistoomuch' * 42, ('thisistoomuch' * 6) + 'thisistoomuc,c4553439dd179422c6acf6a8ac093eb6'),
(u'\u043f\u0440', ',9f18c0db74a9734e9d18461e16345083'),
(u'\u043f\u0440'.encode('utf-8'), ',9f18c0db74a9734e9d18461e16345083'),
(b'column\tvalues/unstr.zip', 'columnvaluesunstr.zip,b9740dcd0553e11b526450ceb8f76683'),
), ids=str)
def test_safename(data):
result = httplib2.safename(data[0])
assert result == data[1]
except httplib2.RelativeURIError:
pass
@pytest.mark.parametrize('data', (
('', ',d41d8cd98f00b204e9800998ecf8427e'),
('http://example.org/fred/?a=b',
'example.orgfreda=b,58489f63a7a83c3b7794a6a398ee8b1f'),
('http://example.org/fred?/a=b',
'example.orgfreda=b,8c5946d56fec453071f43329ff0be46b'),
('http://www.example.org/fred?/a=b',
'www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968'),
('https://www.example.org/fred?/a=b',
'www.example.orgfreda=b,692e843a333484ce0095b070497ab45d'),
(httplib2.urlnorm('http://WWW')[-1],
httplib2.safename(httplib2.urlnorm('http://www')[-1])),
(u'http://\u2304.org/fred/?a=b',
'.orgfreda=b,ecaf0f97756c0716de76f593bd60a35e'),
('normal-resource-name.js',
'normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54'),
('foo://dom/path/brath/carapath',
'dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3'),
('with/slash', 'withslash,17cc656656bb8ce2411bd41ead56d176'),
('thisistoomuch' * 42,
('thisistoomuch' * 6) + 'thisistoomuc,c4553439dd179422c6acf6a8ac093eb6'),
(u'\u043f\u0440', ',9f18c0db74a9734e9d18461e16345083'),
(u'\u043f\u0440'.encode('utf-8'), ',9f18c0db74a9734e9d18461e16345083'),
(b'column\tvalues/unstr.zip',
'columnvaluesunstr.zip,b9740dcd0553e11b526450ceb8f76683'),
),
ids=str)
),
(
"http://example.org/fred?/a=b",
"example.orgfreda=b,8c5946d56fec453071f43329ff0be46b",
),
(
"http://www.example.org/fred?/a=b",
"www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968",
),
(
"https://www.example.org/fred?/a=b",
"www.example.orgfreda=b,692e843a333484ce0095b070497ab45d",
),
(
httplib2.urlnorm("http://WWW")[-1],
httplib2.safename(httplib2.urlnorm("http://www")[-1]),
),
(
u"http://\u2304.org/fred/?a=b",
".orgfreda=b,ecaf0f97756c0716de76f593bd60a35e",
),
(
"normal-resource-name.js",
"normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54",
),
(
"foo://dom/path/brath/carapath",
"dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3",
),
("with/slash", "withslash,17cc656656bb8ce2411bd41ead56d176"),
(