def save_blob_to_store(blob_id, blob, context, max_size, store_type=None, verifier=None): """Save file to specified store type and return location info to the user :param store_type: type of the store, None means save to default store. :param blob_id: id of artifact :param blob: blob file iterator :param context: user context :param verifier:signature verified :return: tuple of values: (location_uri, size, checksums, metadata) """ data = utils.LimitingReader(utils.CooperativeReader(blob), max_size) (location, size, md5checksum, metadata) = backend.add_to_backend(CONF, blob_id, data, 0, store_type, context, verifier) checksums = { "md5": md5checksum, "sha1": data.sha1.hexdigest(), "sha256": data.sha256.hexdigest() } return location, size, checksums
def save_blob_to_store(blob_id, blob, context, max_size, store_type=None, verifier=None): """Save file to specified store type and return location info to the user. :param store_type: type of the store, None means save to default store. :param blob_id: id of blob :param blob: blob file iterator :param context: user context :param verifier:signature verified :return: tuple of values: (location_uri, size, checksums) """ data = utils.LimitingReader(utils.CooperativeReader(blob), max_size) LOG.debug('Start uploading blob %s.', blob_id) if store_type == 'database': location = database_api.add_to_backend(blob_id, data, context, verifier) else: (location, size, md5checksum, __) = backend.add_to_backend(CONF, blob_id, data, 0, store_type, context, verifier) LOG.debug('Uploading of blob %s is finished.', blob_id) checksums = { "md5": data.md5.hexdigest(), "sha1": data.sha1.hexdigest(), "sha256": data.sha256.hexdigest() } return location, data.bytes_read, checksums
def test_cooperative_reader_no_read_method(self): BYTES = 1024 stream = [b'*'] * BYTES reader = utils.CooperativeReader(stream) bytes_read = 0 byte = reader.read() while len(byte) != 0: bytes_read += 1 byte = reader.read() self.assertEqual(BYTES, bytes_read) # some data may be left in the buffer reader = utils.CooperativeReader(stream) reader.buffer = 'some data' buffer_string = reader.read() self.assertEqual('some data', buffer_string)
def test_cooperative_reader_no_read_method_buffer_size(self): # Decrease buffer size to 1000 bytes to test its overflow with mock.patch('glare.common.utils.MAX_COOP_READER_BUFFER_SIZE', 1000): BYTES = 1024 stream = [b'*'] * BYTES reader = utils.CooperativeReader(stream) # Reading 1001 bytes to the buffer leads to 413 error self.assertRaises(exc.RequestEntityTooLarge, reader.read, 1001)
def test_cooperative_reader_of_iterator_stop_iteration_err(self): """Ensure cooperative reader supports iterator backends too""" reader = utils.CooperativeReader([l * 3 for l in '']) chunks = [] while True: chunks.append(reader.read(3)) if chunks[-1] == b'': break meat = b''.join(chunks) self.assertEqual(b'', meat)
def test_cooperative_reader_iterator(self): """Ensure cooperative reader class accesses all bytes of file""" BYTES = 1024 bytes_read = 0 with tempfile.TemporaryFile('w+') as tmp_fd: tmp_fd.write('*' * BYTES) tmp_fd.seek(0) for chunk in utils.CooperativeReader(tmp_fd): bytes_read += len(chunk) self.assertEqual(BYTES, bytes_read)
def test_cooperative_reader_of_iterator(self): """Ensure cooperative reader supports iterator backends too""" data = b'abcdefgh' data_list = [data[i:i + 1] * 3 for i in range(len(data))] reader = utils.CooperativeReader(data_list) chunks = [] while True: chunks.append(reader.read(3)) if chunks[-1] == b'': break meat = b''.join(chunks) self.assertEqual(b'aaabbbcccdddeeefffggghhh', meat)
def _test_reader_chunked(self, chunk_size, read_size, max_iterations=5): generator = self._create_generator(chunk_size, max_iterations) reader = utils.CooperativeReader(generator) result = bytearray() while True: data = reader.read(read_size) if len(data) == 0: break self.assertLessEqual(len(data), read_size) result += data expected = (b'a' * chunk_size + b'b' * chunk_size + b'c' * chunk_size + b'a' * chunk_size + b'b' * chunk_size) self.assertEqual(expected, bytes(result))
def test_cooperative_reader_explicit_read(self): BYTES = 1024 bytes_read = 0 with tempfile.TemporaryFile('w+') as tmp_fd: tmp_fd.write('*' * BYTES) tmp_fd.seek(0) reader = utils.CooperativeReader(tmp_fd) byte = reader.read(1) while len(byte) != 0: bytes_read += 1 byte = reader.read(1) self.assertEqual(BYTES, bytes_read)
def save_blobs_to_store(blobs, context, max_size, store_type=None, verifier=None): """Save several files to specified store. :param store_type: type of the store, None means save to default store. :param blobs: list of tuples (blob_data_id, data) :param context: user context :param verifier:signature verified :return: dict {blob_data_id: (location_uri, size, checksums)} """ # wrap data in CooperativeReader blobs = [(blob_data_id, utils.LimitingReader(utils.CooperativeReader(data), max_size)) for (blob_data_id, data) in blobs] if store_type == 'database': locations = database_api.add_to_backend_batch(blobs, context, verifier) else: locations = [] for blob_data_id, data in blobs: (location, __, __, __) = backend.add_to_backend(CONF, blob_data_id, data, 0, store_type, context, verifier) locations.append(location) # combine location, size and checksums together res = {} for i in range(len(locations)): data = blobs[i][1] checksums = { "md5": data.md5.hexdigest(), "sha1": data.sha1.hexdigest(), "sha256": data.sha256.hexdigest() } res[blobs[i][0]] = (locations[i], data.bytes_read, checksums) return res