def _download(self):
inner_notifier = False
if self.enableCheckPoint:
self._load()
if not self._record:
self._prepare()
sendedBytes, unfinished_down_parts = self._download_prepare()
try:
if len(unfinished_down_parts) > 0:
if (self.size - sendedBytes) > 0:
if self.progressCallback is not None:
self.notifier = progress.ProgressNotifier(
self.progressCallback, self.size)
inner_notifier = True
self.notifier.start()
self.notifier.send(sendedBytes)
thread_pools = _ThreadPool(
functools.partial(self._produce,
download_parts=unfinished_down_parts),
[self._consume] * self.taskNum)
thread_pools.run()
if self._abort:
self._delete_record()
self._delete_tmp_file()
if self._exception is not None:
raise Exception(self._exception)
for p in self._record['downloadParts']:
if not p['isCompleted']:
raise Exception(
'some parts are failed when download. Please try again'
)
try:
os.rename(self._tmp_file, self.fileName)
if self.enableCheckPoint:
self._delete_record()
self.obsClient.log_client.log(INFO, 'download success.')
return self._metedata_resp
except Exception as e:
if self._do_rename():
self.obsClient.log_client.log(INFO, 'download success.')
return self._metedata_resp
if not self.enableCheckPoint:
self._delete_tmp_file()
self.obsClient.log_client.log(
INFO,
'Rename failed. The reason maybe:[the {0} exists, not a file path, not permission]. Please check.'
)
raise e
finally:
if inner_notifier:
self.notifier.end()
def _upload(self):
if self.enableCheckPoint:
self._load()
if self._record is None:
self._prepare()
unfinished_upload_parts = []
sendedBytes = const.LONG(0)
for p in self._record['uploadParts']:
if not p['isCompleted']:
unfinished_upload_parts.append(p)
else:
sendedBytes += p['length']
if self.progressCallback is not None:
self.notifier = progress.ProgressNotifier(self.progressCallback, self.size)
self.notifier.start()
try:
if len(unfinished_upload_parts) > 0:
if (self.size - sendedBytes) > 0:
self.notifier.send(sendedBytes)
thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=unfinished_upload_parts),
[self._consume] * self.taskNum)
thread_pools.run()
if self._abort:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
extensionHeaders=self.extensionHeaders)
self.obsClient.log_client.log(
ERROR,
'the code from server is 4**, please check space、persimission and so on.'
)
self._delete_record()
if self._exception is not None:
raise Exception(self._exception)
for p in self._record['uploadParts']:
if not p['isCompleted']:
if not self.enableCheckPoint:
self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey,
self._record['uploadId'],
extensionHeaders=self.extensionHeaders)
raise Exception('some parts are failed when upload. Please try again')
part_Etags = []
for part in self._record['partEtags']:
part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag']))
self.obsClient.log_client.log(INFO, 'Completing to upload multiparts')
resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'],
CompleteMultipartUploadRequest(part_Etags),
extensionHeaders=self.extensionHeaders)
self._upload_handle_response(resp)
return resp
finally:
self.notifier.end()
def _download_files(obsClient,
bucketName,
prefix,
downloadFolder=None,
taskNum=const.DEFAULT_TASK_NUM,
taskQueueSize=const.DEFAULT_TASK_QUEUE_SIZE,
headers=GetObjectHeader(),
imageProcess=None,
interval=const.DEFAULT_BYTE_INTTERVAL,
taskCallback=None,
progressCallback=None,
threshold=const.DEFAULT_MAXIMUM_SIZE,
partSize=5 * 1024 * 1024,
subTaskNum=1,
enableCheckpoint=False,
checkpointFile=None):
try:
executor = None
notifier = None
if downloadFolder is None or not os.path.isdir(downloadFolder):
raise Exception('%s is not a Folder' % downloadFolder)
if taskCallback is not None and not callable(taskCallback):
raise Exception('Invalid taskCallback')
(taskNum, taskQueueSize, interval,
threshold) = bulktasks._checkBulkTasksPara(taskNum, taskQueueSize,
interval, threshold)
taskCallback = taskCallback if taskCallback is not None else util.lazyCallback
executor = bulktasks.ThreadPool(taskNum, taskQueueSize)
state = bulktasks.ExecuteProgress()
totalTasks = const.LONG(0)
totalAmount = const.LONG(0)
notifier = progress.ProgressNotifier(
progressCallback, totalAmount, interval
) if progressCallback is not None else progress.NONE_NOTIFIER
notifier.start()
query = GetObjectRequest(imageProcess=imageProcess)
prefix = prefix if prefix is not None else ''
prefixDir = prefix[:prefix.rfind('/') + 1]
for content in _list_objects(obsClient, bucketName, prefix=prefix):
objectKey = content.key
totalTasks += 1
totalAmount += content.size
objectPath = objectKey.replace(prefixDir, '', 1)
if objectPath.startswith('/') or objectPath.find(
'//') != -1 or objectPath.find('\\') != -1:
state._failed_increment()
taskCallback(objectKey,
Exception('illegal path: %s' % objectKey))
obsClient.log_client.log(ERROR, 'illegal path: %s' % objectKey)
continue
downloadPath = os.path.join(downloadFolder, objectPath)
downloadPath = util.safe_encode(downloadPath)
if const.IS_WINDOWS:
downloadPath = util.safe_trans_to_gb2312(downloadPath)
dirName = os.path.dirname(downloadPath)
if not os.path.exists(dirName):
try:
os.makedirs(dirName, 0o755)
except Exception as e:
state._failed_increment()
taskCallback(objectKey, e)
obsClient.log_client.log(ERROR, traceback.format_exc())
continue
if objectKey.endswith(('/')):
state._successful_increment()
elif content.size < threshold:
executor.execute(_task_wrap,
obsClient,
obsClient._getObjectWithNotifier,
key=objectKey,
taskCallback=taskCallback,
state=state,
bucketName=bucketName,
objectKey=objectKey,
getObjectRequest=query,
headers=headers,
downloadPath=downloadPath,
notifier=notifier)
else:
executor.execute(_task_wrap,
obsClient,
obsClient._downloadFileWithNotifier,
key=objectKey,
taskCallback=taskCallback,
state=state,
bucketName=bucketName,
objectKey=objectKey,
downloadFile=downloadPath,
partSize=partSize,
taskNum=subTaskNum,
enableCheckpoint=enableCheckpoint,
checkpointFile=checkpointFile,
header=headers,
imageProcess=imageProcess,
notifier=notifier)
state.total_tasks = totalTasks
notifier.totalAmount = totalAmount
finally:
if executor is not None:
executor.shutdown()
if notifier is not None:
notifier.end()
return state
def _download_files_notifier(progressCallback, totalAmount, interval):
return progress.ProgressNotifier(
progressCallback, totalAmount,
interval) if progressCallback is not None else progress.NONE_NOTIFIER
