예제 #1
0
 def write_ec_meta_chunk(self, source, size, storage_method, sysmeta,
                         meta_chunk):
     meta_checksum = md5()
     handler = ECChunkWriteHandler(sysmeta, meta_chunk, meta_checksum,
                                   storage_method)
     bytes_transferred, checksum, chunks = handler.stream(source, size)
     return Response("OK")
예제 #2
0
    def test_write_response_error(self):
        test_cases = [
            {
                'error': Timeout(1.0),
                'msg': 'resp: Timeout 1.0 second'
            },
            {
                'error': Exception('failure'),
                'msg': 'resp: failure'
            },
        ]
        for test in test_cases:
            checksum = self.checksum()
            source = empty_stream()
            size = CHUNK_SIZE * self.storage_method.ec_nb_data
            nb = self.storage_method.ec_nb_data + \
                self.storage_method.ec_nb_parity
            resps = [201] * (nb - 1)
            resps.append((100, test['error']))
            with set_http_connect(*resps):
                handler = ECChunkWriteHandler(self.sysmeta,
                                              self.meta_chunk_copy(), checksum,
                                              self.storage_method)
                bytes_transferred, checksum, chunks = handler.stream(
                    source, size)

            self.assertEqual(len(chunks), nb)
            for i in range(nb - 1):
                self.assertEqual(chunks[i].get('error'), None)
            self.assertEqual(chunks[nb - 1].get('error'), test['msg'])

            self.assertEqual(bytes_transferred, 0)
            self.assertEqual(checksum, EMPTY_CHECKSUM)
예제 #3
0
 def write_ec_meta_chunk(self, source, size, storage_method, sysmeta,
                         meta_chunk):
     meta_checksum = md5()
     handler = ECChunkWriteHandler(sysmeta, meta_chunk, meta_checksum,
                                   storage_method)
     bytes_transferred, checksum, chunks = handler.stream(source, size)
     return Response("OK")
예제 #4
0
    def test_write_transfer(self):
        checksum = self.checksum()
        segment_size = self.storage_method.ec_segment_size
        test_data = ('1234' * segment_size)[:-10]
        size = len(test_data)
        test_data_checksum = self.checksum(test_data).hexdigest()
        nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
        resps = [201] * nb
        source = BytesIO(test_data)

        put_reqs = defaultdict(lambda: {'parts': []})

        def cb_body(conn_id, part):
            put_reqs[conn_id]['parts'].append(part)

        # TODO test headers

        with set_http_connect(*resps, cb_body=cb_body):
            handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                          checksum, self.storage_method)
            bytes_transferred, checksum, chunks = handler.stream(source, size)

        self.assertEqual(len(test_data), bytes_transferred)
        self.assertEqual(checksum, self.checksum(test_data).hexdigest())
        fragments = []

        for conn_id, info in put_reqs.items():
            body, trailers = decode_chunked_body(''.join(info['parts']))
            fragments.append(body)
            metachunk_size = int(trailers[chunk_headers['metachunk_size']])
            metachunk_hash = trailers[chunk_headers['metachunk_hash']]
            self.assertEqual(metachunk_size, size)

            self.assertEqual(metachunk_hash, test_data_checksum)

        self.assertEqual(len(fragments), nb)
        fragment_size = self.storage_method.ec_fragment_size

        # retrieve segments
        frags = []
        for frag in fragments:
            data = [
                frag[x:x + fragment_size]
                for x in range(0, len(frag), fragment_size)
            ]
            frags.append(data)

        fragments = zip(*frags)

        final_data = ''
        for frag in fragments:
            self.assertEqual(len(frag), nb)
            frag = list(frag)
            final_data += self.storage_method.driver.decode(frag)

        self.assertEqual(len(test_data), len(final_data))
        self.assertEqual(test_data_checksum,
                         self.checksum(final_data).hexdigest())
예제 #5
0
    def test_write_transfer(self):
        checksum = self.checksum()
        segment_size = self.storage_method.ec_segment_size
        test_data = ('1234' * segment_size)[:-10]
        size = len(test_data)
        test_data_checksum = self.checksum(test_data).hexdigest()
        nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
        resps = [201] * nb
        source = StringIO(test_data)

        put_reqs = defaultdict(lambda: {'parts': []})

        def cb_body(conn_id, part):
            put_reqs[conn_id]['parts'].append(part)

        # TODO test headers

        with set_http_connect(*resps, cb_body=cb_body):
            handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                          checksum, self.storage_method)
            bytes_transferred, checksum, chunks = handler.stream(source, size)

        self.assertEqual(len(test_data), bytes_transferred)
        self.assertEqual(checksum, self.checksum(test_data).hexdigest())
        fragments = []

        for conn_id, info in put_reqs.items():
            body, trailers = decode_chunked_body(''.join(info['parts']))
            fragments.append(body)
            metachunk_size = int(trailers[chunk_headers['metachunk_size']])
            metachunk_hash = trailers[chunk_headers['metachunk_hash']]
            self.assertEqual(metachunk_size, size)

            self.assertEqual(metachunk_hash, test_data_checksum)

        self.assertEqual(len(fragments), nb)
        fragment_size = self.storage_method.ec_fragment_size

        # retrieve segments
        frags = []
        for frag in fragments:
            data = [frag[x:x + fragment_size]
                    for x in range(0, len(frag), fragment_size)]
            frags.append(data)

        fragments = zip(*frags)

        final_data = ''
        for frag in fragments:
            self.assertEqual(len(frag), nb)
            frag = list(frag)
            final_data += self.storage_method.driver.decode(frag)

        self.assertEqual(len(test_data), len(final_data))
        self.assertEqual(
            test_data_checksum, self.checksum(final_data).hexdigest())
예제 #6
0
 def test_write_simple(self):
     checksum = self.checksum()
     source = empty_stream()
     size = CHUNK_SIZE * self.storage_method.ec_nb_data
     nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
     resps = [201] * nb
     with set_http_connect(*resps):
         handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                       checksum, self.storage_method)
         bytes_transferred, checksum, chunks = handler.stream(source, size)
     self.assertEqual(len(chunks), nb)
     self.assertEqual(bytes_transferred, 0)
     self.assertEqual(checksum, EMPTY_CHECKSUM)
예제 #7
0
 def test_write_simple(self):
     checksum = self.checksum()
     source = empty_stream()
     size = CHUNK_SIZE * self.storage_method.ec_nb_data
     nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
     resps = [201] * nb
     with set_http_connect(*resps):
         handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                       checksum, self.storage_method)
         bytes_transferred, checksum, chunks = handler.stream(source, size)
     self.assertEqual(len(chunks), nb)
     self.assertEqual(bytes_transferred, 0)
     self.assertEqual(checksum, EMPTY_CHECKSUM)
예제 #8
0
    def test_write_partial_exception(self):
        checksum = self.checksum()
        source = empty_stream()
        size = CHUNK_SIZE * self.storage_method.ec_nb_data
        nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
        resps = [201] * (nb - 1)
        resps.append(Exception("failure"))
        with set_http_connect(*resps):
            handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                          checksum, self.storage_method)
            bytes_transferred, checksum, chunks = handler.stream(source, size)

        self.assertEqual(len(chunks), nb)
        for i in range(nb - 1):
            self.assertEqual(chunks[i].get('error'), None)
        self.assertEqual(chunks[nb - 1].get('error'), 'failure')

        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(checksum, EMPTY_CHECKSUM)
예제 #9
0
    def test_write_partial_exception(self):
        checksum = self.checksum()
        source = empty_stream()
        size = CHUNK_SIZE * self.storage_method.ec_nb_data
        nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
        resps = [201] * (nb - 1)
        resps.append(Exception("failure"))
        with set_http_connect(*resps):
            handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                          checksum, self.storage_method)
            bytes_transferred, checksum, chunks = handler.stream(source, size)

        self.assertEqual(len(chunks), nb)
        for i in range(nb - 1):
            self.assertEqual(chunks[i].get('error'), None)
        self.assertEqual(chunks[nb - 1].get('error'), 'failure')

        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(checksum, EMPTY_CHECKSUM)
예제 #10
0
    def test_write_quorum_success(self):
        checksum = self.checksum()
        source = empty_stream()
        size = CHUNK_SIZE * self.storage_method.ec_nb_data
        nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
        quorum_size = self.storage_method.quorum
        resps = [201] * quorum_size
        resps += [500] * (nb - quorum_size)
        with set_http_connect(*resps):
            handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                          checksum, self.storage_method)
            bytes_transferred, checksum, chunks = handler.stream(source, size)
        self.assertEqual(len(chunks), nb)

        for i in range(quorum_size):
            self.assertEqual(chunks[i].get('error'), None)
        for i in xrange(quorum_size, nb):
            self.assertEqual(chunks[i].get('error'), 'resp: HTTP 500')

        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(checksum, EMPTY_CHECKSUM)
예제 #11
0
    def test_write_quorum_success(self):
        checksum = self.checksum()
        source = empty_stream()
        size = CHUNK_SIZE * self.storage_method.ec_nb_data
        nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
        quorum_size = self.storage_method.quorum
        resps = [201] * quorum_size
        resps += [500] * (nb - quorum_size)
        with set_http_connect(*resps):
            handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
                                          checksum, self.storage_method)
            bytes_transferred, checksum, chunks = handler.stream(source, size)
        self.assertEqual(len(chunks), nb)

        for i in range(quorum_size):
            self.assertEqual(chunks[i].get('error'), None)
        for i in xrange(quorum_size, nb):
            self.assertEqual(chunks[i].get('error'), 'HTTP 500')

        self.assertEqual(bytes_transferred, 0)
        self.assertEqual(checksum, EMPTY_CHECKSUM)