示例#1
0
def fetch_stream(chunks, ranges, storage_method, headers=None, **kwargs):
    ranges = ranges or [(None, None)]
    meta_range_list = get_meta_ranges(ranges, chunks)

    for meta_range_dict in meta_range_list:
        for pos in sorted(meta_range_dict.keys()):
            meta_start, meta_end = meta_range_dict[pos]
            if meta_start is not None and meta_end is not None:
                headers['Range'] = http_header_from_ranges(
                    (meta_range_dict[pos], ))
            reader = ChunkReader(iter(chunks[pos]),
                                 READ_CHUNK_SIZE,
                                 headers=headers,
                                 **kwargs)
            try:
                it = reader.get_iter()
            except exc.NotFound as err:
                raise exc.UnrecoverableContent(
                    "Cannot download position %d: %s" % (pos, err))
            except Exception as err:
                raise exc.ServiceUnavailable(
                    "Error while downloading position %d: %s" % (pos, err))
            for part in it:
                for dat in part['iter']:
                    yield dat
示例#2
0
 def chunk_get(self, url, **kwargs):
     req_id = kwargs.get('req_id')
     if not req_id:
         req_id = utils.request_id()
     reader = ChunkReader([{'url': url}], READ_BUFFER_SIZE,
                          {'X-oio-req-id': req_id})
     # This must be done now if we want to access headers
     stream = reader.stream()
     headers = extract_headers_meta(reader.headers)
     return headers, stream
示例#3
0
    def test_reader_buf_size(self):
        reader = ChunkReader(None, 8, {})

        chunk = {}
        source = FakeSource(["1234", "abcd", "123", "4a", "bcd1234abcd1234a", "b"])

        it = reader._create_iter(chunk, source)

        data = list(it)
        self.assertEqual(data, ["1234abcd", "1234abcd", "1234abcd", "1234ab"])
示例#4
0
    def test_reader_buf_size(self):
        reader = ChunkReader(None, 8, {})

        chunk = {}
        source = FakeSource(
            ['1234', 'abcd', '123', '4a', 'bcd1234abcd1234a', 'b'])

        it = reader._create_iter(chunk, source)

        data = list(it)
        self.assertEqual(data, ['1234abcd', '1234abcd', '1234abcd', '1234ab'])
示例#5
0
 def chunk_get(self, url, **kwargs):
     req_id = kwargs.get('req_id')
     if not req_id:
         req_id = utils.request_id()
     reader = ChunkReader([{
         'url': url
     }], READ_BUFFER_SIZE, {'X-oio-req-id': req_id})
     # This must be done now if we want to access headers
     stream = reader.stream()
     headers = extract_headers_meta(reader.headers)
     return headers, stream
示例#6
0
 def chunk_get(self, url, check_headers=True, **kwargs):
     """
     :keyword check_headers: when True (the default), raise FaultyChunk
         if a mandatory response header is missing.
     :returns: a tuple with a dictionary of chunk metadata and a stream
         to the chunk's data.
     """
     url = self.resolve_url(url)
     reader = ChunkReader([{'url': url}], READ_BUFFER_SIZE, **kwargs)
     # This must be done now if we want to access headers
     stream = reader.stream()
     headers = extract_headers_meta(reader.headers, check=check_headers)
     return headers, stream
示例#7
0
    def test_reader_buf_resume(self):
        chunk = {}

        reader = ChunkReader(None, 8, {})

        # provide source0 with failure
        source0 = FakeSource(['1234', 'abcd', '123', None])

        it = reader._create_iter(chunk, source0)
        # provide source1 for recovery
        source1 = FakeSource(['5678efgh'])
        with patch.object(reader, '_get_source', lambda: (source1, chunk)):
            data = list(it)

        self.assertEqual(data, ['1234abcd', '5678efgh'])
示例#8
0
    def test_reader_buf_resume(self):
        chunk = {}

        reader = ChunkReader(None, 8, {})

        # provide source0 with failure
        source0 = FakeSource(["1234", "abcd", "123", None])

        it = reader._create_iter(chunk, source0)
        # provide source1 for recovery
        source1 = FakeSource(["5678efgh"])
        with patch.object(reader, "_get_source", lambda: (source1, chunk)):
            data = list(it)

        self.assertEqual(data, ["1234abcd", "5678efgh"])
示例#9
0
 def read_meta_chunk(self, storage_method, meta_chunk, headers={}):
     handler = ChunkReader(meta_chunk, None, headers)
     stream = handler.get_iter()
     return Response(part_iter_to_bytes_iter(stream), 200)
示例#10
0
    def test_recover(self):
        # basic without range
        reader = ChunkReader(None, None, {})
        reader.recover(10)
        self.assertEqual(reader.request_headers['Range'], 'bytes=10-')

        # full byte range
        reader = ChunkReader(None, None, {'Range': 'bytes=21-40'})
        reader.recover(10)
        self.assertEqual(reader.request_headers['Range'], 'bytes=31-40')
        # ask byte range too large
        self.assertRaises(exc.UnsatisfiableRange, reader.recover, 100)
        # ask empty byte range
        self.assertRaises(exc.EmptyByteRange, reader.recover, 10)

        # prefix byte range
        reader = ChunkReader(None, None, {'Range': 'bytes=11-'})
        reader.recover(10)
        self.assertEqual(reader.request_headers['Range'], 'bytes=21-')

        # suffix byte range
        reader = ChunkReader(None, None, {'Range': 'bytes=-50'})
        reader.recover(10)
        self.assertEqual(reader.request_headers['Range'], 'bytes=-40')

        # single byte range
        reader = ChunkReader(None, None, {'Range': 'bytes=0-0'})
        # ask empty byte range
        self.assertRaises(exc.EmptyByteRange, reader.recover, 1)
示例#11
0
    def test_recover(self):
        # basic without range
        reader = ChunkReader(None, None, {})
        reader.recover(10)
        self.assertEqual(reader.request_headers["Range"], "bytes=10-")

        # full byte range
        reader = ChunkReader(None, None, {"Range": "bytes=21-40"})
        reader.recover(10)
        self.assertEqual(reader.request_headers["Range"], "bytes=31-40")
        # ask byte range too large
        self.assertRaises(exc.UnsatisfiableRange, reader.recover, 100)
        # ask empty byte range
        self.assertRaises(exc.EmptyByteRange, reader.recover, 10)

        # prefix byte range
        reader = ChunkReader(None, None, {"Range": "bytes=11-"})
        reader.recover(10)
        self.assertEqual(reader.request_headers["Range"], "bytes=21-")

        # suffix byte range
        reader = ChunkReader(None, None, {"Range": "bytes=-50"})
        reader.recover(10)
        self.assertEqual(reader.request_headers["Range"], "bytes=-40")

        # single byte range
        reader = ChunkReader(None, None, {"Range": "bytes=0-0"})
        # ask empty byte range
        self.assertRaises(exc.EmptyByteRange, reader.recover, 1)
示例#12
0
 def read_meta_chunk(self, storage_method, meta_chunk):
     headers = {}
     handler = ChunkReader(meta_chunk, headers)
     stream = handler.get_iter()
     return Response(part_iter_to_bytes_iter(stream), 200)