Exemple #1
0
    def test_client_chunk_size_resuming(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

        node = {'ip': '1.2.3.4', 'port': 6000, 'device': 'sda'}

        source1 = TestSource(['abcd', '1234', 'abc', None])
        source2 = TestSource(['efgh5678'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(self.app,
                                   req,
                                   'Object',
                                   None,
                                   None,
                                   None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with patch.object(handler, '_get_source_and_node', lambda:
                          (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ['abcd1234', 'efgh5678'])
        self.assertEqual(handler.backend_headers['Range'], 'bytes=8-')
Exemple #2
0
    def test_client_chunk_size(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    return self.chunks.pop(0)
                else:
                    return ''

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        source = TestSource(
            ('abcd', '1234', 'abc', 'd1', '234abcd1234abcd1', '2'))
        req = Request.blank('/v1/a/c/o')
        node = {}
        handler = GetOrHeadHandler(self.app,
                                   req,
                                   None,
                                   None,
                                   None,
                                   None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source)
        client_chunks = list(app_iter)
        self.assertEqual(client_chunks,
                         ['abcd1234', 'abcd1234', 'abcd1234', 'abcd12'])
Exemple #3
0
    def test_client_chunk_size_resuming(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ""

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks if c is not None))

            def getheaders(self):
                return [("content-length", self.getheader("content-length"))]

        node = {"ip": "1.2.3.4", "port": 6000, "device": "sda"}

        source1 = TestSource(["abcd", "1234", "abc", None])
        source2 = TestSource(["efgh5678"])
        req = Request.blank("/v1/a/c/o")
        handler = GetOrHeadHandler(self.app, req, "Object", None, None, None, {}, client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with patch.object(handler, "_get_source_and_node", lambda: (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ["abcd1234", "efgh5678"])
Exemple #4
0
    def test_client_chunk_size(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)

            def read(self, _read_size):
                if self.chunks:
                    return self.chunks.pop(0)
                else:
                    return ''

        source = TestSource(
            ('abcd', '1234', 'abc', 'd1', '234abcd1234abcd1', '2'))
        req = Request.blank('/v1/a/c/o')
        node = {}
        handler = GetOrHeadHandler(self.app,
                                   req,
                                   None,
                                   None,
                                   None,
                                   None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source)
        client_chunks = list(app_iter)
        self.assertEqual(client_chunks,
                         ['abcd1234', 'abcd1234', 'abcd1234', 'abcd12'])
Exemple #5
0
    def test_client_chunk_size(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    return self.chunks.pop(0)
                else:
                    return ""

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks))

            def getheaders(self):
                return [("content-length", self.getheader("content-length"))]

        source = TestSource(("abcd", "1234", "abc", "d1", "234abcd1234abcd1", "2"))
        req = Request.blank("/v1/a/c/o")
        node = {}
        handler = GetOrHeadHandler(self.app, req, None, None, None, None, {}, client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source)
        client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ["abcd1234", "abcd1234", "abcd1234", "abcd12"])
Exemple #6
0
    def test_client_chunk_size_resuming(self):

        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

        node = {'ip': '1.2.3.4', 'port': 6000, 'device': 'sda'}

        source1 = TestSource(['abcd', '1234', 'abc', None])
        source2 = TestSource(['efgh5678'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {},
            client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with patch.object(handler, '_get_source_and_node',
                          lambda: (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ['abcd1234', 'efgh5678'])
        self.assertEqual(handler.backend_headers['Range'], 'bytes=8-')
Exemple #7
0
    def test_client_chunk_size(self):

        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    return self.chunks.pop(0)
                else:
                    return b''

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        source = TestSource((
            b'abcd', b'1234', b'abc', b'd1', b'234abcd1234abcd1', b'2'))
        req = Request.blank('/v1/a/c/o')
        node = {}
        handler = GetOrHeadHandler(self.app, req, None, None, None, None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source)
        client_chunks = list(app_iter)
        self.assertEqual(client_chunks, [
            b'abcd1234', b'abcd1234', b'abcd1234', b'abcd12'])
Exemple #8
0
    def test_client_chunk_size_resuming(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(
                        sum(len(c) for c in self.chunks if c is not None))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}

        data = ['abcd', '1234', 'efgh', '5678', 'lots', 'more', 'data']

        # NB: content length on source1 should be correct
        # but that reversed piece never makes it to the client
        source1 = TestSource(data[:2] + [data[2][::-1], None] + data[3:])
        source2 = TestSource(data[2:4] + ['nope', None])
        source3 = TestSource(data[4:])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(self.app,
                                   req,
                                   'Object',
                                   None,
                                   None,
                                   None, {},
                                   client_chunk_size=8)

        range_headers = []
        sources = [(source2, node), (source3, node)]

        def mock_get_source_and_node():
            range_headers.append(handler.backend_headers['Range'])
            return sources.pop(0)

        app_iter = handler._make_app_iter(req, node, source1)
        with mock.patch.object(handler,
                               '_get_source_and_node',
                               side_effect=mock_get_source_and_node):
            client_chunks = list(app_iter)
        self.assertEqual(range_headers, ['bytes=8-27', 'bytes=16-27'])
        self.assertEqual(client_chunks,
                         ['abcd1234', 'efgh5678', 'lotsmore', 'data'])
Exemple #9
0
    def test_client_chunk_size_resuming(self):

        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return b''

            def getheader(self, header):
                # content-length for the whole object is generated dynamically
                # by summing non-None chunks initialized as source1
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks
                                   if c is not None))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}

        source1 = TestSource([b'abcd', b'1234', None,
                              b'efgh', b'5678', b'lots', b'more', b'data'])
        # incomplete reads of client_chunk_size will be re-fetched
        source2 = TestSource([b'efgh', b'5678', b'lots', None])
        source3 = TestSource([b'lots', b'more', b'data'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {},
            client_chunk_size=8)

        range_headers = []
        sources = [(source2, node), (source3, node)]

        def mock_get_source_and_node():
            range_headers.append(handler.backend_headers['Range'])
            return sources.pop(0)

        app_iter = handler._make_app_iter(req, node, source1)
        with mock.patch.object(handler, '_get_source_and_node',
                               side_effect=mock_get_source_and_node):
            client_chunks = list(app_iter)
        self.assertEqual(range_headers, ['bytes=8-27', 'bytes=16-27'])
        self.assertEqual(client_chunks, [
            b'abcd1234', b'efgh5678', b'lotsmore', b'data'])
Exemple #10
0
    def test_client_chunk_size_resuming(self):

        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

            def getheader(self, header):
                # content-length for the whole object is generated dynamically
                # by summing non-None chunks initialized as source1
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks
                                   if c is not None))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}

        source1 = TestSource(['abcd', '1234', None,
                              'efgh', '5678', 'lots', 'more', 'data'])
        # incomplete reads of client_chunk_size will be re-fetched
        source2 = TestSource(['efgh', '5678', 'lots', None])
        source3 = TestSource(['lots', 'more', 'data'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {},
            client_chunk_size=8)

        range_headers = []
        sources = [(source2, node), (source3, node)]

        def mock_get_source_and_node():
            range_headers.append(handler.backend_headers['Range'])
            return sources.pop(0)

        app_iter = handler._make_app_iter(req, node, source1)
        with mock.patch.object(handler, '_get_source_and_node',
                               side_effect=mock_get_source_and_node):
            client_chunks = list(app_iter)
        self.assertEqual(range_headers, ['bytes=8-27', 'bytes=16-27'])
        self.assertEqual(client_chunks, [
            'abcd1234', 'efgh5678', 'lotsmore', 'data'])
    def test_disconnected_warning(self):
        self.app.logger = mock.Mock()
        req = Request.blank('/v1/a/c/o')

        class TestSource(object):
            def __init__(self):
                self.headers = {
                    'content-type': 'text/plain',
                    'content-length': len(self.read(-1))
                }
                self.status = 200

            def read(self, _read_size):
                return 'the cake is a lie'

            def getheader(self, header):
                return self.headers.get(header.lower())

            def getheaders(self):
                return self.headers

        source = TestSource()

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}
        handler = GetOrHeadHandler(self.app, req, 'Object', None, None, None,
                                   {})
        app_iter = handler._make_app_iter(req, node, source)
        app_iter.close()
        self.app.logger.warning.assert_called_once_with(
            'Client disconnected on read')

        self.app.logger = mock.Mock()
        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}
        handler = GetOrHeadHandler(self.app, req, 'Object', None, None, None,
                                   {})
        app_iter = handler._make_app_iter(req, node, source)
        next(app_iter)
        app_iter.close()
        self.app.logger.warning.assert_not_called()
Exemple #12
0
    def test_disconnected_warning(self):
        self.app.logger = mock.Mock()
        req = Request.blank('/v1/a/c/o')

        class TestSource(object):
            def __init__(self):
                self.headers = {'content-type': 'text/plain',
                                'content-length': len(self.read(-1))}
                self.status = 200

            def read(self, _read_size):
                return b'the cake is a lie'

            def getheader(self, header):
                return self.headers.get(header.lower())

            def getheaders(self):
                return self.headers

        source = TestSource()

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {})
        app_iter = handler._make_app_iter(req, node, source)
        app_iter.close()
        self.app.logger.warning.assert_called_once_with(
            'Client disconnected on read')

        self.app.logger = mock.Mock()
        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {})
        app_iter = handler._make_app_iter(req, node, source)
        next(app_iter)
        app_iter.close()
        self.app.logger.warning.assert_not_called()
Exemple #13
0
    def test_client_chunk_size_resuming_chunked(self):
        class TestChunkedSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200
                self.headers = {
                    'transfer-encoding': 'chunked',
                    'content-type': 'text/plain'
                }

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

            def getheader(self, header):
                return self.headers.get(header.lower())

            def getheaders(self):
                return self.headers

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}

        source1 = TestChunkedSource(['abcd', '1234', 'abc', None])
        source2 = TestChunkedSource(['efgh5678'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(self.app,
                                   req,
                                   'Object',
                                   None,
                                   None,
                                   None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with patch.object(handler, '_get_source_and_node', lambda:
                          (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ['abcd1234', 'efgh5678'])
Exemple #14
0
    def test_client_chunk_size_resuming(self):
        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(
                        sum(len(c) for c in self.chunks if c is not None))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}

        source1 = TestSource(['abcd', '1234', 'abc', None])
        source2 = TestSource(['efgh5678'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(self.app,
                                   req,
                                   'Object',
                                   None,
                                   None,
                                   None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with patch.object(handler, '_get_source_and_node', lambda:
                          (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ['abcd1234', 'efgh5678'])
Exemple #15
0
    def test_client_chunk_size_resuming(self):

        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return ''

            def getheader(self, header):
                if header.lower() == "content-length":
                    return str(sum(len(c) for c in self.chunks
                                   if c is not None))

            def getheaders(self):
                return [('content-length', self.getheader('content-length'))]

        node = {'ip': '1.2.3.4', 'port': 6000, 'device': 'sda'}

        source1 = TestSource(['abcd', '1234', 'abc', None])
        source2 = TestSource(['efgh5678'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {},
            client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with patch.object(handler, '_get_source_and_node',
                          lambda: (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, ['abcd1234', 'efgh5678'])
Exemple #16
0
    def test_client_chunk_size_resuming_chunked(self):

        class TestChunkedSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)
                self.status = 200
                self.headers = {'transfer-encoding': 'chunked',
                                'content-type': 'text/plain'}

            def read(self, _read_size):
                if self.chunks:
                    chunk = self.chunks.pop(0)
                    if chunk is None:
                        raise exceptions.ChunkReadTimeout()
                    else:
                        return chunk
                else:
                    return b''

            def getheader(self, header):
                return self.headers.get(header.lower())

            def getheaders(self):
                return self.headers

        node = {'ip': '1.2.3.4', 'port': 6200, 'device': 'sda'}

        source1 = TestChunkedSource([b'abcd', b'1234', b'abc', None])
        source2 = TestChunkedSource([b'efgh5678'])
        req = Request.blank('/v1/a/c/o')
        handler = GetOrHeadHandler(
            self.app, req, 'Object', None, None, None, {},
            client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source1)
        with mock.patch.object(handler, '_get_source_and_node',
                               lambda: (source2, node)):
            client_chunks = list(app_iter)
        self.assertEqual(client_chunks, [b'abcd1234', b'efgh5678'])
Exemple #17
0
    def test_client_chunk_size(self):

        class TestSource(object):
            def __init__(self, chunks):
                self.chunks = list(chunks)

            def read(self, _read_size):
                if self.chunks:
                    return self.chunks.pop(0)
                else:
                    return ''

        source = TestSource((
            'abcd', '1234', 'abc', 'd1', '234abcd1234abcd1', '2'))
        req = Request.blank('/v1/a/c/o')
        node = {}
        handler = GetOrHeadHandler(self.app, req, None, None, None, None, {},
                                   client_chunk_size=8)

        app_iter = handler._make_app_iter(req, node, source)
        client_chunks = list(app_iter)
        self.assertEqual(client_chunks, [
            'abcd1234', 'abcd1234', 'abcd1234', 'abcd12'])