def _upload_part_for_process(self, part, partEtag_infos, upload_infos, status):
if status.value == 0:
try:
resp = self.obsClient.uploadPart(self.bucketName, self.objectKey, part['partNumber'], self._record['uploadId'], self.fileName,
isFile=True, partSize=part['length'], offset=part['offset'])
except IOError:
raise IOError('can not attach file {0}'.format(self.fileName))
if resp.status < 300:
complete_part = CompletePart(to_int(part['partNumber']), resp.body.etag)
partEtag_infos[to_int(part['partNumber'])] = resp.body.etag
upload_infos[to_int(part['partNumber'])-1] = True
if self.enableCheckPoint:
with self._lock:
record = self._get_record()
record['uploadParts'][part['partNumber']-1]['isCompleted'] = True
record['partEtags'].append(complete_part)
self._write_record(record)
elif resp.status > 300 and resp.status < 500:
self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._lock.acquire()
status.value = 1
self._lock.release()
upload_infos[to_int(part['partNumber'])-1] = False
else:
upload_infos[to_int(part['partNumber'])-1] = False
def _upload_part_for_process(self, func_args, item_args):
part = item_args
partEtag_infos, upload_infos, status = func_args
if status.value == 0:
resp = self.real_upload(part)
i = 0
while resp.status > 300 and i < TRY_MULTIPART_TIMES:
logger.warning('retry %s part, %d times, part number: %d' % (self.cmdtype, i, part['partNumber']))
resp = self.real_upload(part)
i += 1
if resp.status < 300:
complete_part = CompletePart(to_int(part['partNumber']), resp.body.etag)
partEtag_infos[to_int(part['partNumber'])] = resp.body.etag
upload_infos[to_int(part['partNumber']) - 1] = True
if self.enable_checkpoint:
with self._lock:
record = self._get_record()
record['uploadParts'][part['partNumber'] - 1]['isCompleted'] = True
record['partEtags'].append(complete_part)
self._write_record(record)
pbar_add_size(part['length'])
logger.info('%s part %d complete, %s, uploadid=%s' % (self.cmdtype, part['partNumber'], self.filename, self.uploadId))
elif 300 < resp.status < 500:
logger.warning('server error. ErrorCode:{0}, ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
with self._lock:
status.value = 1
upload_infos[to_int(part['partNumber']) - 1] = False
globl.append_list_lock('part_task_failed', os.getpid())
def _checkBulkTasksPara(task_num, task_queue_size, task_interval, threshold):
origine = [task_num, task_queue_size, task_interval, threshold]
default = (const.DEFAULT_TASK_NUM, const.DEFAULT_TASK_QUEUE_SIZE,
const.DEFAULT_BYTE_INTTERVAL, const.DEFAULT_MAXIMUM_SIZE)
size = len(origine)
for i in range(size):
origine[i] = util.to_int(origine[i])
if origine[i] is None or origine[i] <= 0:
origine[i] = default[i]
return tuple(origine)
def _download_part_process(self, part, download_infos, status):
if status.value == 0:
get_object_request = GetObjectRequest(versionId=self._record['versionId'])
self.header.range = str(part['offset'])+'-'+str(part['offset']+part['length'])
try:
resp = self.obsClient.getObject(bucketName=self._record['bucketName'], objectKey=self._record['objectKey'], getObjectRequest=get_object_request, headers=self.header)
if resp.status < 300:
respone = resp.body.response
chunk_size = 65536
with self._lock:
if respone is not None:
with open(_to_unicode(self._tmp_file), 'rb+') as f:
f.seek(part['offset'], 0)
position = to_int(part['offset'])
while True:
chunk = respone.read(chunk_size)
if not chunk:
break
f.write(chunk)
position += chunk_size
f.seek(position, 0)
respone.close()
f.close()
download_infos[part['partNumber'] - 1] = True
if self.enableCheckPoint:
with self._lock:
record = self._get_record()
record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file)
record['downloadParts'][part['partNumber']-1]['isCompleted'] = True
self._write_record(record)
elif resp.status > 300 and resp.status < 500:
with self._lock:
download_infos[part['partNumber'] - 1] = False
self._lock.acquire()
status.value = 1
self._lock.release()
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
else:
download_infos[part['partNumber'] - 1] = False
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
except Exception as e:
download_infos[part['partNumber'] - 1] = False
self.obsClient.log_client.log(ERROR, 'something wraong happened. Please check.')
if self.enableCheckPoint:
with self._lock:
record = self._get_record()
record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file)
self._write_record(record)
raise e
def upload(self, flowwith):
if self.enable_checkpoint:
self._load_record()
else:
self._prepare()
if os.path.exists(self.checkpoint_file):
start_total = read_cp_file(self.checkpoint_file, 0, self.cmdtype)
pbar_add_size(start_total)
self._upload_parts = self._get_upload_parts()
part_etag_info = {}
for part_info in self._record['partEtags']:
part_etag_info[part_info['partNum']] = part_info['etag']
upload_info = []
for part_info in self._record['uploadParts']:
upload_info.append(part_info['isCompleted'])
part_etag_infos = compat.Dict(part_etag_info)
upload_infos = compat.List(upload_info)
status = compat.Value('i', 0)
multithreading_with_sleep(self._upload_part_for_process, (part_etag_infos, upload_infos, status),
self._upload_parts, self.tasknum, flowwith)
if False in upload_infos:
if status.value == 1:
self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId'])
logger.warning('the code from server is 4**, please check')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('%s some parts are failed. Please try agagin, %s' % (self.cmdtype, self.filename))
part_etags = []
tmp_part_etags = sorted(part_etag_infos.items(), key=lambda d: d[0])
for key, value in tmp_part_etags:
part_etags.append(CompletePart(partNum=to_int(key), etag=value))
resp = self.obscmdutil.complete_multipart_upload(self.bucket, self.objkey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_etags))
if 300 < resp.status < 500:
logger.error('complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId'])
if self.enable_checkpoint:
self._delete_record()
return resp
def reset_partSize(self):
if self.partSize < const.DEFAULT_MINIMUM_SIZE:
self.partSize = const.DEFAULT_MINIMUM_SIZE
elif self.partSize > const.DEFAULT_MAXIMUM_SIZE:
self.partSize = const.DEFAULT_MAXIMUM_SIZE
else:
self.partSize = util.to_int(self.partSize)
if self.taskNum <= 0:
self.taskNum = 1
else:
self.taskNum = int(math.ceil(self.taskNum))
self.total_parts = int(self.size / self.partSize)
if self.total_parts >= 10000:
self.partSize = int(self.size / 10000) if self.size % 10000 == 0 else int(self.size / 10000) + 1
self.total_parts = int(self.size / self.partSize)
if self.size % self.partSize != 0:
self.total_parts += 1
def _download_part(self, part):
get_object_request = GetObjectRequest(versionId=self.versionId)
self.header.range = str(part['offset'])+'-'+str(part['offset']+part['length'])
try:
resp = self.obsClient.getObject(bucketName=self.bucketName, objectKey=self.objectKey, getObjectRequest=get_object_request, headers=self.header)
if resp.status < 300:
respone = resp.body.response
chunk_size = 65536
with self._lock:
if respone is not None:
with open(_to_unicode(self._tmp_file), 'rb+') as fs:
fs.seek(part['offset'], 0)
position = to_int(part['offset'])
while True:
chunk = respone.read(chunk_size)
if not chunk:
break
fs.write(chunk)
position += chunk_size
fs.seek(position, 0)
fs.close()
respone.close()
self._downinfos.append(True)
self._record['downloadParts'][part['partNumber']-1]['isCompleted'] = True
elif resp.status > 300 and resp.status < 500:
with self._lock:
self._downinfos.append(False)
self._change_status()
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
else:
self._downinfos.append(False)
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
except Exception as e:
self.obsClient.log_client.log(ERROR, 'something wraong happened. Please check.')
raise e
finally:
if self.enableCheckPoint:
with self._lock:
self._record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file)
self._write_record(self._record)
def _upload_part(self, part):
if not self._is_abort():
try:
resp = self.get_upload_part_resp(part)
if resp.status < 300:
self._record['uploadParts'][part['partNumber'] - 1]['isCompleted'] = True
self._record['partEtags'].append(CompletePart(util.to_int(part['partNumber']), resp.body.etag))
if self.enableCheckPoint:
with self._lock:
self._write_record(self._record)
else:
if 300 < resp.status < 500:
self._do_abort('errorCode:{0}, errorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self.obsClient.log_client.log(
ERROR,
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
except Exception as e:
self.obsClient.log_client.log(DEBUG, 'upload part %s error, %s' % (part['partNumber'], e))
self.obsClient.log_client.log(ERROR, traceback.format_exc())
def _upload_part(self, part):
if self._status:
try:
resp = self.obsClient.uploadPart(self.bucketName, self.objectKey, part['partNumber'], self._record['uploadId'], self.fileName,
isFile=True, partSize=part['length'], offset=part['offset'])
except IOError:
raise IOError('can not attach file {0}. Please check'.format(self.fileName))
if resp.status < 300:
complete_part = CompletePart(to_int(part['partNumber']), resp.body.etag)
with self._lock:
self._record['uploadParts'][part['partNumber']-1]['isCompleted'] = True
self._record['partEtags'].append(complete_part)
self._uploadinfos.append(True)
if self.enableCheckPoint:
self._write_record(self._record)
elif resp.status > 300 and resp.status < 500:
self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._change_status()
self._uploadinfos.append(False)
else:
self._uploadinfos.append(False)
def _upload_part(self, part):
if not self._is_abort():
try:
resp = self.obsClient._uploadPartWithNotifier(
self.bucketName,
self.objectKey,
part['partNumber'],
self._record['uploadId'],
self.fileName,
isFile=True,
partSize=part['length'],
offset=part['offset'],
notifier=self.notifier,
extensionHeaders=self.extensionHeaders,
sseHeader=self.headers.sseHeader)
if resp.status < 300:
self._record['uploadParts'][part['partNumber'] -
1]['isCompleted'] = True
self._record['partEtags'].append(
CompletePart(util.to_int(part['partNumber']),
resp.body.etag))
if self.enableCheckPoint:
with self._lock:
self._write_record(self._record)
else:
if 300 < resp.status < 500:
self._do_abort(
'errorCode:{0}, errorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self.obsClient.log_client.log(
ERROR,
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
except Exception as e:
self.obsClient.log_client.log(
DEBUG,
'upload part %s error, %s' % (part['partNumber'], e))
self.obsClient.log_client.log(ERROR, traceback.format_exc())
def _upload(self):
if not self.enableCheckPoint:
self._prepare()
else:
self._load()
self.__upload_parts = self._get_upload_parts()
if IS_WINDOWS:
self._uploadinfos = []
self._status = True
thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=self.__upload_parts),
[self._consume] * self.taskNum, self._lock)
thread_pools.run()
if not min(self._uploadinfos):
if not self._status:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check space、persimission and so on.')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('some parts are failed when upload. Please try agagin')
part_Etags = []
for part in sorted(self._record['partEtags'], key=lambda x: x['partNum']):
part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags))
else:
partEtag_info = {}
for part_info in self._record['partEtags']:
partEtag_info[part_info['partNum']] = part_info['etag']
partEtag_infos = multiprocessing.Manager().dict(partEtag_info)
upload_info = []
for part_info in self._record['uploadParts']:
upload_info.append(part_info['isCompleted'])
upload_infos = multiprocessing.Manager().list(upload_info)
status = multiprocessing.Manager().Value('i', 0)
pool = Pool(processes=self.taskNum)
for part in self.__upload_parts:
pool.apply_async(self._upload_part_for_process(part, partEtag_infos, upload_infos, status))
pool.close()
pool.join()
if not min(upload_infos):
if status.value == 1:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check')
self._delete_record()
if len(self._exception) > 0:
raise Exception(self._exception[0])
raise Exception('some parts are failed when upload. Please try agagin')
part_Etags = []
partETags = sorted(partEtag_infos.items(), key=lambda d: d[0])
for key, value in partETags:
part_Etags.append(CompletePart(partNum=to_int(key), etag=value))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags))
if resp.status < 300:
if self.enableCheckPoint:
self._delete_record()
else:
if not self.enableCheckPoint:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
else:
if resp.status > 300 and resp.status < 500:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'])
self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self._delete_record()
return resp