def CompleteMultipartUpload():
Lpart1 = CompletePart(partNum=1, etag='e97f0beed4d87dd49203a21efcb557e6')
Lpart2 = CompletePart(partNum=2, etag='5f8bb5e6ce16ab638d6b7df6942f0904')
Lparts = []
Lparts.append(Lpart1)
Lparts.append(Lpart2)
LcompleteMultipartUploadRequest = CompleteMultipartUploadRequest(parts=Lparts)
resp = TestObs.completeMultipartUpload(bucketName='bucket001', objectKey='test.zip', uploadId='0001AB175A0000015C7C473EFB962682',
completeMultipartUploadRequest=LcompleteMultipartUploadRequest)
print('common msg:status:', resp.status, ',errorCode:', resp.errorCode, ',errorMessage:', resp.errorMessage)
if resp.body:
print('location:', resp.body.location, ',bucket:', resp.body.bucket, ',key:', resp.body.key, ',etag:', resp.body.etag)
def _upload_part_for_process(self, part, partEtag_infos, upload_infos, status):
if status.value == 0:
try:
resp = self.obsClient.uploadPart(self.bucketName, self.objectKey, part['partNumber'], self._record['uploadId'], self.fileName,
isFile=True, partSize=part['length'], offset=part['offset'])
except IOError:
raise IOError('can not attach file {0}'.format(self.fileName))
if resp.status < 300:
complete_part = CompletePart(toInt(part['partNumber']), resp.body.etag)
partEtag_infos[toInt(part['partNumber'])] = resp.body.etag
upload_infos[toInt(part['partNumber'])-1] = True
if self.enableCheckPoint:
with self._lock:
record = self._get_record()
record['uploadParts'][part['partNumber']-1]['isCompleted'] = True
record['partEtags'].append(complete_part)
self._write_record(record)
elif resp.status > 300 and resp.status < 500:
self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._lock.acquire()
status.value = 1
self._lock.release()
upload_infos[toInt(part['partNumber'])-1] = False
else:
upload_infos[toInt(part['partNumber'])-1] = False
def _upload_part(self, part):
if self._status:
try:
resp = self.obsClient.uploadPart(self.bucketName,
self.objectKey,
part['partNumber'],
self._record['uploadId'],
self.fileName,
isFile=True,
partSize=part['length'],
offset=part['offset'])
except IOError:
raise IOError('can not attach file {0}. Please check'.format(
self.fileName))
if resp.status < 300:
complete_part = CompletePart(toInt(part['partNumber']),
resp.body.etag)
with self._lock:
self._record['uploadParts'][part['partNumber'] -
1]['isCompleted'] = True
self._record['partEtags'].append(complete_part)
self._uploadinfos.append(True)
if self.enableCheckPoint:
self._write_record(self._record)
elif resp.status > 300 and resp.status < 500:
self.obsClient.log_client.log(
ERROR,
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
self._exception.append('errorCode:{0}.errorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self._change_status()
self._uploadinfos.append(False)
else:
self._uploadinfos.append(False)
server=server,
signature=signature,
path_style=path_style)
# Create bucket
print('Create a new bucket for demo\n')
obsClient.createBucket(bucketName)
# Step 1: initiate multipart upload
print('Step 1: initiate multipart upload \n')
resp = obsClient.initiateMultipartUpload(bucketName, objectKey)
uploadId = resp.body.uploadId
# Step 2: upload a part
print('Step 2: upload a part\n')
partNum = 1
resp = obsClient.uploadPart(bucketName,
objectKey,
partNumber=partNum,
uploadId=uploadId,
object='Hello OBS')
etag = dict(resp.header).get('etag')
# Step 3: complete multipart upload
print('Step 3: complete multipart upload\n')
from com.obs.models.complete_multipart_upload_request import CompletePart, CompleteMultipartUploadRequest
obsClient.completeMultipartUpload(
bucketName, objectKey, uploadId,
CompleteMultipartUploadRequest([CompletePart(partNum=partNum, etag=etag)]))
def _upload(self):
if not self.enableCheckPoint:
self._prepare()
else:
self._load()
self.__upload_parts = self._get_upload_parts()
if IS_WINDOWS:
self._uploadinfos = []
self._status = True
thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=self.__upload_parts),
[self._consume] * self.taskNum, self._lock)
thread_pools.run()
if not min(self._uploadinfos):
if not self._status:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check space、persimission and so on.')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('some parts are failed when upload. Please try agagin')
part_Etags = []
for part in sorted(self._record['partEtags'], key=lambda x: x['partNum']):
part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags))
else:
partEtag_info = {}
for part_info in self._record['partEtags']:
partEtag_info[part_info['partNum']] = part_info['etag']
partEtag_infos = multiprocessing.Manager().dict(partEtag_info)
upload_info = []
for part_info in self._record['uploadParts']:
upload_info.append(part_info['isCompleted'])
upload_infos = multiprocessing.Manager().list(upload_info)
status = multiprocessing.Manager().Value('i', 0)
pool = Pool(processes=self.taskNum)
for part in self.__upload_parts:
pool.apply_async(self._upload_part_for_process(part, partEtag_infos, upload_infos, status))
pool.close()
pool.join()
if not min(upload_infos):
if status.value == 1:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('some parts are failed when upload. Please try agagin')
part_Etags = []
partETags = sorted(partEtag_infos.items(), key=lambda d: d[0])
for key, value in partETags:
part_Etags.append(CompletePart(partNum=toInt(key), etag=value))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags))
if resp.status < 300:
if self.enableCheckPoint:
self._delete_record()
else:
if not self.enableCheckPoint:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
else:
if resp.status > 300 and resp.status < 500:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self._delete_record()
return resp
for p in processes:
p.join()
if len(partETags) != partCount:
raise Exception(
'Upload multiparts fail due to some parts are not finished yet')
print('Succeed to complete multiparts into an object named ' + objectKey +
'\n')
# View all parts uploaded recently
print('Listing all parts......')
resp = obsClient.listParts(bucketName, objectKey, uploadId)
for part in resp.body.parts:
print('\tPart#' + str(part.partNumber) + ', ETag=' + part.etag)
print('\n')
# Complete to upload multiparts
from com.obs.models.complete_multipart_upload_request import CompletePart, CompleteMultipartUploadRequest
partETags = sorted(partETags.items(), key=lambda d: d[0])
parts = []
for key, value in partETags:
parts.append(CompletePart(partNum=key, etag=value))
print('Completing to upload multiparts\n')
obsClient.completeMultipartUpload(bucketName, objectKey, uploadId,
CompleteMultipartUploadRequest(parts))