示例#1
0
def test_safename():
    cases = (
        ('http://example.org/fred/?a=b',
            'example.org,fred,a=b,58489f63a7a83c3b7794a6a398ee8b1f'),
        ('http://example.org/fred?/a=b',
            'example.org,fred,a=b,8c5946d56fec453071f43329ff0be46b'),
        ('http://www.example.org/fred?/a=b',
            'www.example.org,fred,a=b,499c44b8d844a011b67ea2c015116968'),
        ('https://www.example.org/fred?/a=b',
            'www.example.org,fred,a=b,692e843a333484ce0095b070497ab45d'),
        (httplib2.urlnorm('http://WWW')[-1],
            httplib2.safename(httplib2.urlnorm('http://www')[-1])),
        (u'http://\u2304.org/fred/?a=b',
            'xn--http,-4y1d.org,fred,a=b,579924c35db315e5a32e3d9963388193'),
    )
    for a, b in cases:
        assert httplib2.safename(a) == b

    assert httplib2.safename('http://www') != httplib2.safename('https://www')

    # Test the max length limits
    uri = 'http://' + ('w' * 200) + '.org'
    uri2 = 'http://' + ('w' * 201) + '.org'
    assert httplib2.safename(uri) != httplib2.safename(uri2)
    # Max length should be 200 + 1 (',') + 32
    assert len(httplib2.safename(uri2)) == 233
    assert len(httplib2.safename(uri)) == 233
示例#2
0
 def test_epgguide(self):
     cache_dir = os.getcwd() + "/.epguide"
     #        os.makedirs(cache_dir)
     print cache_dir
     cache = FileCache(cache_dir)
     h = Http(cache=cache)
     user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0"
     httplib2.debuglevel = 255
     resp, content = h.request(
         "http://www.teleman.pl/program-tv/stacje/TVP-2",
         headers={'user-agent': user_agent})
     print "content:"
     print content
     print "response:"
     print resp
     print "fromcache:" + str(resp.fromcache)
     print "status:" + str(resp.status)
     #url = "http://www.teleman.pl/program-tv/stacje/TVP-2"
     url = "http://www.teleman.pl/tv/Dr-House-7-152-885990"
     resp, content = h.request(url, headers={'user-agent': user_agent})
     print "fromcache:" + str(resp.fromcache)
     print "status:" + str(resp.status)
     safe = httplib2.safename(url)
     print "safe:" + safe
     cached_value = cache.get(url)
     info, cached = cached_value.split('\r\n\r\n', 1)
     print "===="
     print content
     print "===="
     print cached
     print "===="
     self.assertEqual(content, cached)
示例#3
0
    def test_epgguide(self):
        cache_dir = os.getcwd() + "/.epguide"
#        os.makedirs(cache_dir)
        print cache_dir
        cache = FileCache(cache_dir)
        h = Http(cache = cache)
        user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0"
        httplib2.debuglevel = 255
        resp, content = h.request("http://www.teleman.pl/program-tv/stacje/TVP-2", headers={'user-agent':user_agent})
        print "content:"
        print content
        print "response:"
        print resp
        print "fromcache:" + str(resp.fromcache)
        print "status:" + str(resp.status)
        #url = "http://www.teleman.pl/program-tv/stacje/TVP-2"
        url = "http://www.teleman.pl/tv/Dr-House-7-152-885990"
        resp, content = h.request(url, headers={'user-agent':user_agent})
        print "fromcache:" + str(resp.fromcache)
        print "status:" + str(resp.status)
        safe = httplib2.safename(url)
        print "safe:" + safe
        cached_value = cache.get(url)
        info, cached = cached_value.split('\r\n\r\n', 1)
        print "===="
        print content
        print "===="
        print cached
        print "===="
        self.assertEqual(content, cached)
示例#4
0
    def testGet304(self):
        # Test that we use ETags properly to validate our cache
        uri = urllib.parse.urljoin(base, "304/test_etag.txt")
        (response, content) = self.http.request(uri, "GET")
        self.assertNotEqual(response['etag'], "")

        (response, content) = self.http.request(uri, "GET")
        (response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'must-revalidate'})
        self.assertEqual(response.status, 200)
        self.assertEqual(response.fromcache, True)

        cache_file_name = os.path.join(cacheDirName, httplib2.safename(httplib2.urlnorm(uri)[-1]))
        f = open(cache_file_name, "r")
        status_line = f.readline()
        f.close()

        self.assertTrue(status_line.startswith("status:"))

        (response, content) = self.http.request(uri, "HEAD")
        self.assertEqual(response.status, 200)
        self.assertEqual(response.fromcache, True)

        (response, content) = self.http.request(uri, "GET", headers = {'range': 'bytes=0-0'})
        self.assertEqual(response.status, 206)
        self.assertEqual(response.fromcache, False)
示例#5
0
 def test(self):
     # Test that different URIs end up generating different safe names
     self.assertEqual( "example.org,fred,a=b,58489f63a7a83c3b7794a6a398ee8b1f", httplib2.safename("http://example.org/fred/?a=b"))
     self.assertEqual( "example.org,fred,a=b,8c5946d56fec453071f43329ff0be46b", httplib2.safename("http://example.org/fred?/a=b"))
     self.assertEqual( "www.example.org,fred,a=b,499c44b8d844a011b67ea2c015116968", httplib2.safename("http://www.example.org/fred?/a=b"))
     self.assertEqual( httplib2.safename(httplib2.urlnorm("http://www")[-1]), httplib2.safename(httplib2.urlnorm("http://WWW")[-1]))
     self.assertEqual( "www.example.org,fred,a=b,692e843a333484ce0095b070497ab45d", httplib2.safename("https://www.example.org/fred?/a=b"))
     self.assertNotEqual( httplib2.safename("http://www"), httplib2.safename("https://www"))
     # Test the max length limits
     uri = "http://" + ("w" * 200) + ".org"
     uri2 = "http://" + ("w" * 201) + ".org"
     self.assertNotEqual( httplib2.safename(uri2), httplib2.safename(uri))
     # Max length should be 200 + 1 (",") + 32
     self.assertEqual(233, len(httplib2.safename(uri2)))
     self.assertEqual(233, len(httplib2.safename(uri)))
     # Unicode
     if sys.version_info >= (2,3):
         self.assertEqual( "xn--http,-4y1d.org,fred,a=b,579924c35db315e5a32e3d9963388193", httplib2.safename("http://\u2304.org/fred/?a=b"))
示例#6
0
def test_etag_used():
    # Test that we use ETags properly to validate our cache
    cache_path = tests.get_cache_path()
    http = httplib2.Http(cache=cache_path)
    response_kwargs = dict(
        add_date=True,
        add_etag=True,
        body=b'something',
        headers={
            'cache-control': 'public,max-age=300',
        },
    )

    def handler(request):
        if request.headers.get('range'):
            return tests.http_response_bytes(status=206, **response_kwargs)
        return tests.http_response_bytes(**response_kwargs)

    with tests.server_request(handler, request_count=2) as uri:
        response, _ = http.request(uri,
                                   'GET',
                                   headers={'accept-encoding': 'identity'})
        assert response['etag'] == '"437b930db84b8079c2dd804a71936b5f"'

        http.request(uri, 'GET', headers={'accept-encoding': 'identity'})
        response, _ = http.request(
            uri,
            'GET',
            headers={
                'accept-encoding': 'identity',
                'cache-control': 'must-revalidate'
            },
        )
        assert response.status == 200
        assert response.fromcache

        # TODO: API to read cache item, at least internal to tests
        cache_file_name = os.path.join(
            cache_path, httplib2.safename(httplib2.urlnorm(uri)[-1]))
        with open(cache_file_name, 'r') as f:
            status_line = f.readline()
        assert status_line.startswith("status:")

        response, content = http.request(
            uri, 'HEAD', headers={'accept-encoding': 'identity'})
        assert response.status == 200
        assert response.fromcache

        response, content = http.request(uri,
                                         'GET',
                                         headers={
                                             'accept-encoding': 'identity',
                                             'range': 'bytes=0-0'
                                         })
        assert response.status == 206
        assert not response.fromcache
示例#7
0
 def wrapped(*args):
     args = list(args) #tuple to list
     args[1] = httplib2.safename(args[1])
     # if len(args[1])>200:
     #     args = list(args) 
     #     args[1] = md5(args[1]).hexdigest()
     # elif isinstance(args[1], unicode):
     #     args = list(args)
     #     args[1] = args[1].encode('utf-8')
     return func(*args)
示例#8
0
 def request(self, uri, method='GET', body=None, headers=None,
             redirections=5, connection_type=None):
     file = os.path.join(HTTP_DATA_DIR, httplib2.safename(uri))
     print file
     if os.path.exists(file):
         response = message_from_file(open(file))
         body = response.get_payload()
         headers = httplib2.Response(response)
         return (headers, body)
     else:
         return (httplib2.Response({"status": "404"}), "")
示例#9
0
 def request(self, uri, method='GET', body=None, headers=None,
             redirections=5, connection_type=None):
     file = os.path.join(HTTP_DATA_DIR, httplib2.safename(uri))
     if os.path.exists(file):
         response = message_from_file(open(file))
         headers = httplib2.Response(response)
         body = bytes(response.get_payload(), charset_from_headers(headers))
         return (headers, body)
     else:
         return (httplib2.Response({"status": "404"}),
                 "Resource %r unavailable from test data store" % file)
示例#10
0
def test_safename2():
    assert httplib2.safename("http://www") != httplib2.safename("https://www")

    # Test the max length limits
    uri = "http://" + ("w" * 200) + ".org"
    uri2 = "http://" + ("w" * 201) + ".org"
    assert httplib2.safename(uri) != httplib2.safename(uri2)
    # Max length should be 90 + 1 (',') + 32 = 123
    assert len(httplib2.safename(uri2)) == 123
    assert len(httplib2.safename(uri)) == 123
示例#11
0
def test_safename2():
    assert httplib2.safename("http://www") != httplib2.safename("https://www")

    # Test the max length limits
    uri = "http://" + ("w" * 200) + ".org"
    uri2 = "http://" + ("w" * 201) + ".org"
    assert httplib2.safename(uri) != httplib2.safename(uri2)
    # Max length should be 90 + 1 (',') + 32 = 123
    assert len(httplib2.safename(uri2)) == 123
    assert len(httplib2.safename(uri)) == 123
示例#12
0
def test_safename2():
    assert httplib2.safename('http://www') != httplib2.safename('https://www')

    # Test the max length limits
    uri = 'http://' + ('w' * 200) + '.org'
    uri2 = 'http://' + ('w' * 201) + '.org'
    assert httplib2.safename(uri) != httplib2.safename(uri2)
    # Max length should be 90 + 1 (',') + 32 = 123
    assert len(httplib2.safename(uri2)) == 123
    assert len(httplib2.safename(uri)) == 123
示例#13
0
def test_safename2():
    assert httplib2.safename('http://www') != httplib2.safename('https://www')

    # Test the max length limits
    uri = 'http://' + ('w' * 200) + '.org'
    uri2 = 'http://' + ('w' * 201) + '.org'
    assert httplib2.safename(uri) != httplib2.safename(uri2)
    # Max length should be 90 + 1 (',') + 32 = 123
    assert len(httplib2.safename(uri2)) == 123
    assert len(httplib2.safename(uri)) == 123
示例#14
0
def test_etag_used():
    # Test that we use ETags properly to validate our cache
    cache_path = tests.get_cache_path()
    http = httplib2.Http(cache=cache_path)
    response_kwargs = dict(
        add_date=True,
        add_etag=True,
        body=b"something",
        headers={"cache-control": "public,max-age=300"},
    )

    def handler(request):
        if request.headers.get("range"):
            return tests.http_response_bytes(status=206, **response_kwargs)
        return tests.http_response_bytes(**response_kwargs)

    with tests.server_request(handler, request_count=2) as uri:
        response, _ = http.request(uri, "GET", headers={"accept-encoding": "identity"})
        assert response["etag"] == '"437b930db84b8079c2dd804a71936b5f"'

        http.request(uri, "GET", headers={"accept-encoding": "identity"})
        response, _ = http.request(
            uri,
            "GET",
            headers={"accept-encoding": "identity", "cache-control": "must-revalidate"},
        )
        assert response.status == 200
        assert response.fromcache

        # TODO: API to read cache item, at least internal to tests
        cache_file_name = os.path.join(
            cache_path, httplib2.safename(httplib2.urlnorm(uri)[-1])
        )
        with open(cache_file_name, "r") as f:
            status_line = f.readline()
        assert status_line.startswith("status:")

        response, content = http.request(
            uri, "HEAD", headers={"accept-encoding": "identity"}
        )
        assert response.status == 200
        assert response.fromcache

        response, content = http.request(
            uri, "GET", headers={"accept-encoding": "identity", "range": "bytes=0-0"}
        )
        assert response.status == 206
        assert not response.fromcache
示例#15
0
def request_mock(uri, method='GET', body=None, headers=None,
              redirections=5, connection_type=None):
    """Http mock side effect that returns saved entries.

    Implementation tests should never span network boundaries.

    """

    file = os.path.join("tests/data", httplib2.safename(uri))
    if os.path.exists(file):
        response = message_from_file(open(file))
        headers = httplib2.Response(response)
        body = bytes(response.get_payload(), charset_from_headers(headers))
        return (headers, body)
    else:
        return (httplib2.Response({"status": "404"}),
                "Resource %r unavailable from test data store" % file)
示例#16
0
def request_mock(uri,
                 method='GET',
                 body=None,
                 headers=None,
                 redirections=5,
                 connection_type=None):
    """Http mock side effect that returns saved entries.

    Implementation tests should never span network boundaries.

    """

    file = os.path.join("tests/data", httplib2.safename(uri))
    if os.path.exists(file):
        response = message_from_file(open(file))
        headers = httplib2.Response(response)
        body = bytes(response.get_payload(), charset_from_headers(headers))
        return (headers, body)
    else:
        return (httplib2.Response({"status": "404"}),
                "Resource %r unavailable from test data store" % file)
示例#17
0
def downloadLocal(url):
    """Download the url to a local file and return the name of the local
    file.

    Ideally this would be cache-aware. httlib2 is cache-aware, but it
    forces me to load the entire response into memory. If we got
    streaming support in httplib2 then we could use it.
    """
    localDir = os.path.join(baseDir, "local")
    if not os.path.exists(localDir): os.makedirs(localDir)

    local = os.path.join(localDir, httplib2.safename(url))
    if not os.path.exists(local):
        try:
            print "Getting", url, "as", local
            logging.info("Getting %s as %s", url, local)
            urllib.urlretrieve(url, local, progress)
        except:
            if (os.path.exists(local)): os.unlink(local)
            raise

    return local
示例#18
0
def test_safename(data):
    result = httplib2.safename(data[0])
    assert result == data[1]
示例#19
0
 ),
 (
     "http://example.org/fred?/a=b",
     "example.orgfreda=b,8c5946d56fec453071f43329ff0be46b",
 ),
 (
     "http://www.example.org/fred?/a=b",
     "www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968",
 ),
 (
     "https://www.example.org/fred?/a=b",
     "www.example.orgfreda=b,692e843a333484ce0095b070497ab45d",
 ),
 (
     httplib2.urlnorm("http://WWW")[-1],
     httplib2.safename(httplib2.urlnorm("http://www")[-1]),
 ),
 (
     u"http://\u2304.org/fred/?a=b",
     ".orgfreda=b,ecaf0f97756c0716de76f593bd60a35e",
 ),
 (
     "normal-resource-name.js",
     "normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54",
 ),
 (
     "foo://dom/path/brath/carapath",
     "dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3",
 ),
 ("with/slash", "withslash,17cc656656bb8ce2411bd41ead56d176"),
 (
示例#20
0
def test_safename(data):
    result = httplib2.safename(data[0])
    assert result == data[1]
示例#21
0
        pass


@pytest.mark.parametrize(
    'data', (
        ('', ',d41d8cd98f00b204e9800998ecf8427e'),
        ('http://example.org/fred/?a=b',
            'example.orgfreda=b,58489f63a7a83c3b7794a6a398ee8b1f'),
        ('http://example.org/fred?/a=b',
            'example.orgfreda=b,8c5946d56fec453071f43329ff0be46b'),
        ('http://www.example.org/fred?/a=b',
            'www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968'),
        ('https://www.example.org/fred?/a=b',
            'www.example.orgfreda=b,692e843a333484ce0095b070497ab45d'),
        (httplib2.urlnorm('http://WWW')[-1],
            httplib2.safename(httplib2.urlnorm('http://www')[-1])),
        (u'http://\u2304.org/fred/?a=b',
            '.orgfreda=b,ecaf0f97756c0716de76f593bd60a35e'),
        ('normal-resource-name.js', 'normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54'),
        ('foo://dom/path/brath/carapath', 'dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3'),
        ('with/slash', 'withslash,17cc656656bb8ce2411bd41ead56d176'),
        ('thisistoomuch' * 42, ('thisistoomuch' * 6) + 'thisistoomuc,c4553439dd179422c6acf6a8ac093eb6'),
        (u'\u043f\u0440', ',9f18c0db74a9734e9d18461e16345083'),
        (u'\u043f\u0440'.encode('utf-8'), ',9f18c0db74a9734e9d18461e16345083'),
        (b'column\tvalues/unstr.zip', 'columnvaluesunstr.zip,b9740dcd0553e11b526450ceb8f76683'),
    ), ids=str)
def test_safename(data):
    result = httplib2.safename(data[0])
    assert result == data[1]

示例#22
0
    except httplib2.RelativeURIError:
        pass


@pytest.mark.parametrize('data', (
    ('', ',d41d8cd98f00b204e9800998ecf8427e'),
    ('http://example.org/fred/?a=b',
     'example.orgfreda=b,58489f63a7a83c3b7794a6a398ee8b1f'),
    ('http://example.org/fred?/a=b',
     'example.orgfreda=b,8c5946d56fec453071f43329ff0be46b'),
    ('http://www.example.org/fred?/a=b',
     'www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968'),
    ('https://www.example.org/fred?/a=b',
     'www.example.orgfreda=b,692e843a333484ce0095b070497ab45d'),
    (httplib2.urlnorm('http://WWW')[-1],
     httplib2.safename(httplib2.urlnorm('http://www')[-1])),
    (u'http://\u2304.org/fred/?a=b',
     '.orgfreda=b,ecaf0f97756c0716de76f593bd60a35e'),
    ('normal-resource-name.js',
     'normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54'),
    ('foo://dom/path/brath/carapath',
     'dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3'),
    ('with/slash', 'withslash,17cc656656bb8ce2411bd41ead56d176'),
    ('thisistoomuch' * 42,
     ('thisistoomuch' * 6) + 'thisistoomuc,c4553439dd179422c6acf6a8ac093eb6'),
    (u'\u043f\u0440', ',9f18c0db74a9734e9d18461e16345083'),
    (u'\u043f\u0440'.encode('utf-8'), ',9f18c0db74a9734e9d18461e16345083'),
    (b'column\tvalues/unstr.zip',
     'columnvaluesunstr.zip,b9740dcd0553e11b526450ceb8f76683'),
),
                         ids=str)
示例#23
0
 ),
 (
     "http://example.org/fred?/a=b",
     "example.orgfreda=b,8c5946d56fec453071f43329ff0be46b",
 ),
 (
     "http://www.example.org/fred?/a=b",
     "www.example.orgfreda=b,499c44b8d844a011b67ea2c015116968",
 ),
 (
     "https://www.example.org/fred?/a=b",
     "www.example.orgfreda=b,692e843a333484ce0095b070497ab45d",
 ),
 (
     httplib2.urlnorm("http://WWW")[-1],
     httplib2.safename(httplib2.urlnorm("http://www")[-1]),
 ),
 (
     u"http://\u2304.org/fred/?a=b",
     ".orgfreda=b,ecaf0f97756c0716de76f593bd60a35e",
 ),
 (
     "normal-resource-name.js",
     "normal-resource-name.js,8ff7c46fd6e61bf4e91a0a1606954a54",
 ),
 (
     "foo://dom/path/brath/carapath",
     "dompathbrathcarapath,83db942781ed975c7a5b7c24039f8ca3",
 ),
 ("with/slash", "withslash,17cc656656bb8ce2411bd41ead56d176"),
 (