def _upload(self): if self.enableCheckPoint: self._load() if self._record is None: self._prepare() unfinished_upload_parts = [] sendedBytes = const.LONG(0) for p in self._record['uploadParts']: if not p['isCompleted']: unfinished_upload_parts.append(p) else: sendedBytes += p['length'] if self.progressCallback is not None: self.notifier = progress.ProgressNotifier(self.progressCallback, self.size) self.notifier.start() try: if len(unfinished_upload_parts) > 0: if (self.size - sendedBytes) > 0: self.notifier.send(sendedBytes) thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=unfinished_upload_parts), [self._consume] * self.taskNum) thread_pools.run() if self._abort: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], extensionHeaders=self.extensionHeaders) self.obsClient.log_client.log( ERROR, 'the code from server is 4**, please check space、persimission and so on.' ) self._delete_record() if self._exception is not None: raise Exception(self._exception) for p in self._record['uploadParts']: if not p['isCompleted']: if not self.enableCheckPoint: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], extensionHeaders=self.extensionHeaders) raise Exception('some parts are failed when upload. Please try again') part_Etags = [] for part in self._record['partEtags']: part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag'])) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags), extensionHeaders=self.extensionHeaders) self._upload_handle_response(resp) return resp finally: self.notifier.end()
def upload(self, flowwith): if self.enable_checkpoint: self._load_record() else: self._prepare() if os.path.exists(self.checkpoint_file): start_total = read_cp_file(self.checkpoint_file, 0, self.cmdtype) pbar_add_size(start_total) self._upload_parts = self._get_upload_parts() part_etag_info = {} for part_info in self._record['partEtags']: part_etag_info[part_info['partNum']] = part_info['etag'] upload_info = [] for part_info in self._record['uploadParts']: upload_info.append(part_info['isCompleted']) part_etag_infos = compat.Dict(part_etag_info) upload_infos = compat.List(upload_info) status = compat.Value('i', 0) multithreading_with_sleep(self._upload_part_for_process, (part_etag_infos, upload_infos, status), self._upload_parts, self.tasknum, flowwith) if False in upload_infos: if status.value == 1: self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId']) logger.warning('the code from server is 4**, please check') self._delete_record() if len(self._exception) > 0: raise Exception(self._exception[0]) raise Exception('%s some parts are failed. Please try agagin, %s' % (self.cmdtype, self.filename)) part_etags = [] tmp_part_etags = sorted(part_etag_infos.items(), key=lambda d: d[0]) for key, value in tmp_part_etags: part_etags.append(CompletePart(partNum=to_int(key), etag=value)) resp = self.obscmdutil.complete_multipart_upload(self.bucket, self.objkey, self._record['uploadId'], CompleteMultipartUploadRequest(part_etags)) if 300 < resp.status < 500: logger.error('complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId']) if self.enable_checkpoint: self._delete_record() return resp
def _upload(self): if not self.enableCheckPoint: self._prepare() else: self._load() self.__upload_parts = self._get_upload_parts() if IS_WINDOWS: self._uploadinfos = [] self._status = True thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=self.__upload_parts), [self._consume] * self.taskNum, self._lock) thread_pools.run() if not min(self._uploadinfos): if not self._status: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check space、persimission and so on.') self._delete_record() if len(self._exception) > 0: raise Exception(self._exception[0]) raise Exception('some parts are failed when upload. Please try agagin') part_Etags = [] for part in sorted(self._record['partEtags'], key=lambda x: x['partNum']): part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag'])) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags)) else: partEtag_info = {} for part_info in self._record['partEtags']: partEtag_info[part_info['partNum']] = part_info['etag'] partEtag_infos = multiprocessing.Manager().dict(partEtag_info) upload_info = [] for part_info in self._record['uploadParts']: upload_info.append(part_info['isCompleted']) upload_infos = multiprocessing.Manager().list(upload_info) status = multiprocessing.Manager().Value('i', 0) pool = Pool(processes=self.taskNum) for part in self.__upload_parts: pool.apply_async(self._upload_part_for_process(part, partEtag_infos, upload_infos, status)) pool.close() pool.join() if not min(upload_infos): if status.value == 1: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check') self._delete_record() if len(self._exception) > 0: raise Exception(self._exception[0]) raise Exception('some parts are failed when upload. Please try agagin') part_Etags = [] partETags = sorted(partEtag_infos.items(), key=lambda d: d[0]) for key, value in partETags: part_Etags.append(CompletePart(partNum=to_int(key), etag=value)) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags)) if resp.status < 300: if self.enableCheckPoint: self._delete_record() else: if not self.enableCheckPoint: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) else: if resp.status > 300 and resp.status < 500: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) self._delete_record() return resp