def _upload(self):
if self.enableCheckPoint:
self._load()
if self._record is None:
self._prepare()
unfinished_upload_parts = []
sendedBytes = const.LONG(0)
for p in self._record['uploadParts']:
if not p['isCompleted']:
unfinished_upload_parts.append(p)
else:
sendedBytes += p['length']
if self.progressCallback is not None:
self.notifier = progress.ProgressNotifier(self.progressCallback, self.size)
self.notifier.start()
try:
if len(unfinished_upload_parts) > 0:
if (self.size - sendedBytes) > 0:
self.notifier.send(sendedBytes)
thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=unfinished_upload_parts),
[self._consume] * self.taskNum)
thread_pools.run()
if self._abort:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
extensionHeaders=self.extensionHeaders)
self.obsClient.log_client.log(
ERROR,
'the code from server is 4**, please check space、persimission and so on.'
)
self._delete_record()
if self._exception is not None:
raise Exception(self._exception)
for p in self._record['uploadParts']:
if not p['isCompleted']:
if not self.enableCheckPoint:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey,
self._record['uploadId'],
extensionHeaders=self.extensionHeaders)
raise Exception('some parts are failed when upload. Please try again')
part_Etags = []
for part in self._record['partEtags']:
part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags),
extensionHeaders=self.extensionHeaders)
self._upload_handle_response(resp)
return resp
finally:
self.notifier.end()
def upload(self, flowwith):
if self.enable_checkpoint:
self._load_record()
else:
self._prepare()
if os.path.exists(self.checkpoint_file):
start_total = read_cp_file(self.checkpoint_file, 0, self.cmdtype)
pbar_add_size(start_total)
self._upload_parts = self._get_upload_parts()
part_etag_info = {}
for part_info in self._record['partEtags']:
part_etag_info[part_info['partNum']] = part_info['etag']
upload_info = []
for part_info in self._record['uploadParts']:
upload_info.append(part_info['isCompleted'])
part_etag_infos = compat.Dict(part_etag_info)
upload_infos = compat.List(upload_info)
status = compat.Value('i', 0)
multithreading_with_sleep(self._upload_part_for_process, (part_etag_infos, upload_infos, status),
self._upload_parts, self.tasknum, flowwith)
if False in upload_infos:
if status.value == 1:
self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId'])
logger.warning('the code from server is 4**, please check')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('%s some parts are failed. Please try agagin, %s' % (self.cmdtype, self.filename))
part_etags = []
tmp_part_etags = sorted(part_etag_infos.items(), key=lambda d: d[0])
for key, value in tmp_part_etags:
part_etags.append(CompletePart(partNum=to_int(key), etag=value))
resp = self.obscmdutil.complete_multipart_upload(self.bucket, self.objkey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_etags))
if 300 < resp.status < 500:
logger.error('complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId'])
if self.enable_checkpoint:
self._delete_record()
return resp
def _upload(self):
if not self.enableCheckPoint:
self._prepare()
else:
self._load()
self.__upload_parts = self._get_upload_parts()
if IS_WINDOWS:
self._uploadinfos = []
self._status = True
thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=self.__upload_parts),
[self._consume] * self.taskNum, self._lock)
thread_pools.run()
if not min(self._uploadinfos):
if not self._status:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check space、persimission and so on.')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('some parts are failed when upload. Please try agagin')
part_Etags = []
for part in sorted(self._record['partEtags'], key=lambda x: x['partNum']):
part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags))
else:
partEtag_info = {}
for part_info in self._record['partEtags']:
partEtag_info[part_info['partNum']] = part_info['etag']
partEtag_infos = multiprocessing.Manager().dict(partEtag_info)
upload_info = []
for part_info in self._record['uploadParts']:
upload_info.append(part_info['isCompleted'])
upload_infos = multiprocessing.Manager().list(upload_info)
status = multiprocessing.Manager().Value('i', 0)
pool = Pool(processes=self.taskNum)
for part in self.__upload_parts:
pool.apply_async(self._upload_part_for_process(part, partEtag_infos, upload_infos, status))
pool.close()
pool.join()
if not min(upload_infos):
if status.value == 1:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('some parts are failed when upload. Please try agagin')
part_Etags = []
partETags = sorted(partEtag_infos.items(), key=lambda d: d[0])
for key, value in partETags:
part_Etags.append(CompletePart(partNum=to_int(key), etag=value))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags))
if resp.status < 300:
if self.enableCheckPoint:
self._delete_record()
else:
if not self.enableCheckPoint:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
else:
if resp.status > 300 and resp.status < 500:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self._delete_record()
return resp