def write(self, data): logger.debug("Receive string with length : {0} Thread: {1}".format( len(data), threading.currentThread().getName())) self._buffer.write(data) self._buffer_len += len(data) if self._uploaded_len > CosFtpConfig().single_file_max_size: logger.error( "Uploading file: {0} exceeds the maximum file limit: {1}". format( self._key_name, str(CosFtpConfig().single_file_max_size).encode("utf-8"))) raise IOError( "Uploading file: {0} exceeds the maximum file limit: {1}". format( self._key_name, str(CosFtpConfig().single_file_max_size).encode("utf-8"))) while self._buffer_len >= self._min_part_size: if not self._has_init: self._upload_pool = UploadPool() response = self._cos_client.create_multipart_upload( Bucket=self._bucket_name, Key=self._key_name) self._multipart_uploader = MultipartUpload( self._cos_client, response) self._part_num = 1 self._isSuccess = None self._has_init = True def callback(part_num, result): with StreamUploader._lock: self._uploaded_part[str(part_num)] = result self._isSuccess = result def check_finish(): if self._isSuccess is not None and not self._isSuccess: err_msg = "Uploading file:{0} failed.".format( self._key_name) logger.error(err_msg) raise IOError(err_msg) check_finish() self._uploaded_part[str(self._part_num)] = None self._upload_pool.apply_task( self._multipart_uploader.upload_part, (self._buffer.read( self._min_part_size), self._part_num, callback)) self._part_num += 1 self._buffer_len -= self._min_part_size # 只要提交到并发上传的线程池中,就可以减掉了 self._uploaded_len += self._min_part_size logger.info("upload new part with length: {0} File: {1}".format( self._min_part_size, self._key_name))
class StreamUploader(object): MIN_PART_SIZE = CosFtpConfig().min_part_size MAX_PART_SIZE = 5 * ftp_v5.conf.common_config.GIGABYTE UPLOAD_THREAD_NUM = CosFtpConfig().upload_thread_num _lock = threading.Lock() def __init__(self, cos_client, bucket_name, object_name=None): self._cos_client = cos_client self._bucket_name = bucket_name self._key_name = object_name if CosFtpConfig().single_file_max_size > 40 * 1000 * ftp_v5.conf.common_config.GIGABYTE: logger.error("Max file size: %d is too big. Thread: %s" % ( CosFtpConfig().single_file_max_size, threading.currentThread().getName())) raise ValueError("Max file size: %d is too big" % CosFtpConfig().single_file_max_size) self._min_part_size = int(math.ceil(float(CosFtpConfig().single_file_max_size) / MultipartUpload.MaxiumPartNum)) if self._min_part_size < StreamUploader.MIN_PART_SIZE: self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB logger.info("Min part size: %d" % self._min_part_size) self._has_init = False self._has_commit = False self._buffer = FifoBuffer() self._buffer_len = 0 self._multipart_uploader = None self._part_num = 1 # 当前上传的分片号 self._uploaded_part = dict() # 已经成功上传的分片 self._uploaded_len = 0 # 已经上传字节数 self._upload_pool = None # 用于并发上传的线程池子 def write(self, data): logger.debug( "Receive string with length : {0} Thread: {1}".format(len(data), threading.currentThread().getName())) self._buffer.write(data) self._buffer_len += len(data) if self._uploaded_len > CosFtpConfig().single_file_max_size: logger.error("Uploading file: {0} exceeds the maximum file limit: {1}".format(self._key_name, str( CosFtpConfig().single_file_max_size).encode("utf-8"))) raise IOError("Uploading file: {0} exceeds the maximum file limit: {1}".format(self._key_name, str( CosFtpConfig().single_file_max_size).encode("utf-8"))) while self._buffer_len >= self._min_part_size: if not self._has_init: self._upload_pool = UploadPool() response = self._cos_client.create_multipart_upload(Bucket=self._bucket_name, Key=self._key_name) self._multipart_uploader = MultipartUpload(self._cos_client, response) self._part_num = 1 self._isSuccess = None self._has_init = True def callback(part_num, result): with StreamUploader._lock: self._uploaded_part[str(part_num)] = result self._isSuccess = result def check_finish(): if self._isSuccess is not None and not self._isSuccess: err_msg = "Uploading file:{0} failed.".format(self._key_name) logger.error(err_msg) raise IOError(err_msg) check_finish() self._uploaded_part[str(self._part_num)] = None self._upload_pool.apply_task(self._multipart_uploader.upload_part, (self._buffer.read(self._min_part_size), self._part_num, callback)) self._part_num += 1 self._buffer_len -= self._min_part_size # 只要提交到并发上传的线程池中,就可以减掉了 self._uploaded_len += self._min_part_size logger.info("upload new part with length: {0} File: {1}".format(self._min_part_size, self._key_name)) def _wait_for_finish(self): is_finish = False while not is_finish: if len(self._uploaded_part) == 0: return for part_num, result in self._uploaded_part.items(): if result is not None and not result: logger.error("Uploading file failed. Failed part_num: %d " % int(part_num)) raise IOError("Uploading part_num: %d failed. " % int(part_num)) if result is None: break else: is_finish = True if not is_finish: time.sleep(10 / 1000) # 休眠10毫秒 def close(self): logger.info( "Closing the stream upload... File: {}, Thread: {}".format(self._key_name, threading.currentThread())) if self._buffer_len != 0: if not self._has_init: # 采用简单文件上传 logger.info("Simple file upload! File: {0}".format(self._key_name)) self._cos_client.put_object(Bucket=self._bucket_name, Body=self._buffer.read(self._buffer_len), Key=self._key_name) else: logger.info("Upload the last part File:{0}".format(self._key_name)) self._multipart_uploader.upload_part(self._buffer.read(self._buffer_len), self._part_num) if self._has_init: logger.info("Wait for all the tasks to finish.") self._wait_for_finish() self._multipart_uploader.complete_upload() self._uploaded_len = 0 self._buffer.close()