def _download_part(self, part):
get_object_request = GetObjectRequest(versionId=self.versionId, imageProcess=self.imageProcess)
get_object_header = self._copy_get_object_header(self.header)
get_object_header.range = util.to_string(part['offset']) + '-' + util.to_string(part['length'])
if not self._is_abort():
# todo 检视 response 的作用与 part_response 的重构
response = None
try:
resp = self.obsClient.getObject(bucketName=self.bucketName, objectKey=self.objectKey,
getObjectRequest=get_object_request,
headers=get_object_header, notifier=self.notifier,
extensionHeaders=self.extensionHeaders)
if resp.status < 300:
part_response = resp.body.response
self._download_part_write(part_response, part)
self._record['downloadParts'][part['partNumber'] - 1]['isCompleted'] = True
if self.enableCheckPoint:
with self._lock:
self._write_record(self._record)
else:
if 300 < resp.status < 500:
self._do_abort('errorCode:{0}, errorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
self._exception += (
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}\n'.format(
resp.errorCode, resp.errorMessage))
self.obsClient.log_client.log(
ERROR,
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
except Exception as e:
self.obsClient.log_client.log(DEBUG, 'download part %s error, %s' % (part['partNumber'], e))
self.obsClient.log_client.log(ERROR, traceback.format_exc())
finally:
if response is not None:
part_response.close()
def _download_part_process(self, func_args, item_args):
part = item_args
download_infos, status = func_args
if status.value == 0:
get_object_request = GetObjectRequest(versionId=self._record['versionId'])
self.header.range = '%d-%d' % (part['offset'], part['offset'] + part['length'] - 1)
try:
resp = self.obscmdutil.get_object(bucketName=self._record['bucketName'],
objectKey=self._record['objectKey'],
getObjectRequest=get_object_request, headers=self.header)
i = 0
while resp.status > 300 and i < TRY_MULTIPART_TIMES:
logger.warning('retry %s part, %d times, part number: %d' % (self.cmdtype, i, part['partNumber']))
resp = self.obscmdutil.get_object(bucketName=self._record['bucketName'],
objectKey=self._record['objectKey'],
getObjectRequest=get_object_request, headers=self.header)
i += 1
if resp.status < 300:
respone = resp.body.response
chunk_size = 65536
if respone is not None:
with open(_to_unicode(self._tmp_file), 'rb+') as f:
f.seek(part['offset'], 0)
while True:
chunk = respone.read(chunk_size)
if not chunk:
break
f.write(chunk)
respone.close()
download_infos[part['partNumber'] - 1] = True
logger.info('%s part %d complete, %s' % (self.cmdtype, part['partNumber'], self.filename))
if self.enable_checkpoint:
with self._lock:
record = self._get_record()
record['downloadParts'][part['partNumber'] - 1]['isCompleted'] = True
self._write_record(record)
pbar_add_size(part['length'])
else:
with self._lock:
download_infos[part['partNumber'] - 1] = False
status.value = 1
self._exception.append(
'response from server error. ErrorCode:{0}, ErrorMessage:{1}'.format(resp.errorCode, resp.errorMessage))
globl.append_list_lock('part_task_failed', os.getpid())
except Exception as e:
download_infos[part['partNumber'] - 1] = False
globl.append_list_lock('part_task_failed', os.getpid())
msg = '%s part %d error, %s, %s' % (self.cmdtype, part['partNumber'], self.filename, e)
self._exception.append(msg)
logger.warning(msg)
raise e
def _download_part(self, part):
get_object_request = GetObjectRequest(versionId=self.versionId,
imageProcess=self.imageProcess)
get_object_header = self._copy_get_object_header(self.header)
get_object_header.range = util.to_string(
part['offset']) + '-' + util.to_string(part['length'])
if not self._is_abort():
response = None
try:
resp = self.obsClient._getObjectWithNotifier(
bucketName=self.bucketName,
objectKey=self.objectKey,
getObjectRequest=get_object_request,
headers=get_object_header,
notifier=self.notifier,
extensionHeaders=self.extensionHeaders)
if resp.status < 300:
respone = resp.body.response
chunk_size = 65536
if respone is not None:
with open(_to_unicode(self._tmp_file), 'rb+') as fs:
fs.seek(part['offset'], 0)
while True:
chunk = respone.read(chunk_size)
if not chunk:
break
fs.write(chunk)
self._record['downloadParts'][part['partNumber'] -
1]['isCompleted'] = True
if self.enableCheckPoint:
with self._lock:
self._write_record(self._record)
else:
if resp.status > 300 and resp.status < 500:
self._do_abort(
'errorCode:{0}, errorMessage:{1}'.format(
resp.errorCode, resp.errorMessage))
self._exception.append(
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
self.obsClient.log_client.log(
ERROR,
'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
except Exception as e:
self.obsClient.log_client.log(
DEBUG,
'download part %s error, %s' % (part['partNumber'], e))
self.obsClient.log_client.log(ERROR, traceback.format_exc())
finally:
if response is not None:
respone.close()
def _download_part_process(self, part, download_infos, status):
if status.value == 0:
get_object_request = GetObjectRequest(versionId=self._record['versionId'])
self.header.range = str(part['offset'])+'-'+str(part['offset']+part['length'])
try:
resp = self.obsClient.getObject(bucketName=self._record['bucketName'], objectKey=self._record['objectKey'], getObjectRequest=get_object_request, headers=self.header)
if resp.status < 300:
respone = resp.body.response
chunk_size = 65536
with self._lock:
if respone is not None:
with open(_to_unicode(self._tmp_file), 'rb+') as f:
f.seek(part['offset'], 0)
position = to_int(part['offset'])
while True:
chunk = respone.read(chunk_size)
if not chunk:
break
f.write(chunk)
position += chunk_size
f.seek(position, 0)
respone.close()
f.close()
download_infos[part['partNumber'] - 1] = True
if self.enableCheckPoint:
with self._lock:
record = self._get_record()
record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file)
record['downloadParts'][part['partNumber']-1]['isCompleted'] = True
self._write_record(record)
elif resp.status > 300 and resp.status < 500:
with self._lock:
download_infos[part['partNumber'] - 1] = False
self._lock.acquire()
status.value = 1
self._lock.release()
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
else:
download_infos[part['partNumber'] - 1] = False
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
except Exception as e:
download_infos[part['partNumber'] - 1] = False
self.obsClient.log_client.log(ERROR, 'something wraong happened. Please check.')
if self.enableCheckPoint:
with self._lock:
record = self._get_record()
record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file)
self._write_record(record)
raise e
def _download_part(self, part):
get_object_request = GetObjectRequest(versionId=self.versionId)
self.header.range = str(part['offset'])+'-'+str(part['offset']+part['length'])
try:
resp = self.obsClient.getObject(bucketName=self.bucketName, objectKey=self.objectKey, getObjectRequest=get_object_request, headers=self.header)
if resp.status < 300:
respone = resp.body.response
chunk_size = 65536
with self._lock:
if respone is not None:
with open(_to_unicode(self._tmp_file), 'rb+') as fs:
fs.seek(part['offset'], 0)
position = to_int(part['offset'])
while True:
chunk = respone.read(chunk_size)
if not chunk:
break
fs.write(chunk)
position += chunk_size
fs.seek(position, 0)
fs.close()
respone.close()
self._downinfos.append(True)
self._record['downloadParts'][part['partNumber']-1]['isCompleted'] = True
elif resp.status > 300 and resp.status < 500:
with self._lock:
self._downinfos.append(False)
self._change_status()
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
else:
self._downinfos.append(False)
self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'
.format(resp.errorCode, resp.errorMessage))
except Exception as e:
self.obsClient.log_client.log(ERROR, 'something wraong happened. Please check.')
raise e
finally:
if self.enableCheckPoint:
with self._lock:
self._record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file)
self._write_record(self._record)
def _download_files(obsClient,
bucketName,
prefix,
downloadFolder=None,
taskNum=const.DEFAULT_TASK_NUM,
taskQueueSize=const.DEFAULT_TASK_QUEUE_SIZE,
headers=GetObjectHeader(),
imageProcess=None,
interval=const.DEFAULT_BYTE_INTTERVAL,
taskCallback=None,
progressCallback=None,
threshold=const.DEFAULT_MAXIMUM_SIZE,
partSize=5 * 1024 * 1024,
subTaskNum=1,
enableCheckpoint=False,
checkpointFile=None):
try:
executor = None
notifier = None
if downloadFolder is None or not os.path.isdir(downloadFolder):
raise Exception('%s is not a Folder' % downloadFolder)
if taskCallback is not None and not callable(taskCallback):
raise Exception('Invalid taskCallback')
(taskNum, taskQueueSize, interval,
threshold) = bulktasks._checkBulkTasksPara(taskNum, taskQueueSize,
interval, threshold)
taskCallback = taskCallback if taskCallback is not None else util.lazyCallback
executor = bulktasks.ThreadPool(taskNum, taskQueueSize)
state = bulktasks.ExecuteProgress()
totalTasks = const.LONG(0)
totalAmount = const.LONG(0)
notifier = progress.ProgressNotifier(
progressCallback, totalAmount, interval
) if progressCallback is not None else progress.NONE_NOTIFIER
notifier.start()
query = GetObjectRequest(imageProcess=imageProcess)
prefix = prefix if prefix is not None else ''
prefixDir = prefix[:prefix.rfind('/') + 1]
for content in _list_objects(obsClient, bucketName, prefix=prefix):
objectKey = content.key
totalTasks += 1
totalAmount += content.size
objectPath = objectKey.replace(prefixDir, '', 1)
if objectPath.startswith('/') or objectPath.find(
'//') != -1 or objectPath.find('\\') != -1:
state._failed_increment()
taskCallback(objectKey,
Exception('illegal path: %s' % objectKey))
obsClient.log_client.log(ERROR, 'illegal path: %s' % objectKey)
continue
downloadPath = os.path.join(downloadFolder, objectPath)
downloadPath = util.safe_encode(downloadPath)
if const.IS_WINDOWS:
downloadPath = util.safe_trans_to_gb2312(downloadPath)
dirName = os.path.dirname(downloadPath)
if not os.path.exists(dirName):
try:
os.makedirs(dirName, 0o755)
except Exception as e:
state._failed_increment()
taskCallback(objectKey, e)
obsClient.log_client.log(ERROR, traceback.format_exc())
continue
if objectKey.endswith(('/')):
state._successful_increment()
elif content.size < threshold:
executor.execute(_task_wrap,
obsClient,
obsClient._getObjectWithNotifier,
key=objectKey,
taskCallback=taskCallback,
state=state,
bucketName=bucketName,
objectKey=objectKey,
getObjectRequest=query,
headers=headers,
downloadPath=downloadPath,
notifier=notifier)
else:
executor.execute(_task_wrap,
obsClient,
obsClient._downloadFileWithNotifier,
key=objectKey,
taskCallback=taskCallback,
state=state,
bucketName=bucketName,
objectKey=objectKey,
downloadFile=downloadPath,
partSize=partSize,
taskNum=subTaskNum,
enableCheckpoint=enableCheckpoint,
checkpointFile=checkpointFile,
header=headers,
imageProcess=imageProcess,
notifier=notifier)
state.total_tasks = totalTasks
notifier.totalAmount = totalAmount
finally:
if executor is not None:
executor.shutdown()
if notifier is not None:
notifier.end()
return state