def _download(self): inner_notifier = False if self.enableCheckPoint: self._load() if not self._record: self._prepare() sendedBytes, unfinished_down_parts = self._download_prepare() try: if len(unfinished_down_parts) > 0: if (self.size - sendedBytes) > 0: if self.progressCallback is not None: self.notifier = progress.ProgressNotifier( self.progressCallback, self.size) inner_notifier = True self.notifier.start() self.notifier.send(sendedBytes) thread_pools = _ThreadPool( functools.partial(self._produce, download_parts=unfinished_down_parts), [self._consume] * self.taskNum) thread_pools.run() if self._abort: self._delete_record() self._delete_tmp_file() if self._exception is not None: raise Exception(self._exception) for p in self._record['downloadParts']: if not p['isCompleted']: raise Exception( 'some parts are failed when download. Please try again' ) try: os.rename(self._tmp_file, self.fileName) if self.enableCheckPoint: self._delete_record() self.obsClient.log_client.log(INFO, 'download success.') return self._metedata_resp except Exception as e: if self._do_rename(): self.obsClient.log_client.log(INFO, 'download success.') return self._metedata_resp if not self.enableCheckPoint: self._delete_tmp_file() self.obsClient.log_client.log( INFO, 'Rename failed. The reason maybe:[the {0} exists, not a file path, not permission]. Please check.' ) raise e finally: if inner_notifier: self.notifier.end()
def _upload(self): if self.enableCheckPoint: self._load() if self._record is None: self._prepare() unfinished_upload_parts = [] sendedBytes = const.LONG(0) for p in self._record['uploadParts']: if not p['isCompleted']: unfinished_upload_parts.append(p) else: sendedBytes += p['length'] if self.progressCallback is not None: self.notifier = progress.ProgressNotifier(self.progressCallback, self.size) self.notifier.start() try: if len(unfinished_upload_parts) > 0: if (self.size - sendedBytes) > 0: self.notifier.send(sendedBytes) thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=unfinished_upload_parts), [self._consume] * self.taskNum) thread_pools.run() if self._abort: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], extensionHeaders=self.extensionHeaders) self.obsClient.log_client.log( ERROR, 'the code from server is 4**, please check space、persimission and so on.' ) self._delete_record() if self._exception is not None: raise Exception(self._exception) for p in self._record['uploadParts']: if not p['isCompleted']: if not self.enableCheckPoint: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], extensionHeaders=self.extensionHeaders) raise Exception('some parts are failed when upload. Please try again') part_Etags = [] for part in self._record['partEtags']: part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag'])) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags), extensionHeaders=self.extensionHeaders) self._upload_handle_response(resp) return resp finally: self.notifier.end()
def _download_files(obsClient, bucketName, prefix, downloadFolder=None, taskNum=const.DEFAULT_TASK_NUM, taskQueueSize=const.DEFAULT_TASK_QUEUE_SIZE, headers=GetObjectHeader(), imageProcess=None, interval=const.DEFAULT_BYTE_INTTERVAL, taskCallback=None, progressCallback=None, threshold=const.DEFAULT_MAXIMUM_SIZE, partSize=5 * 1024 * 1024, subTaskNum=1, enableCheckpoint=False, checkpointFile=None): try: executor = None notifier = None if downloadFolder is None or not os.path.isdir(downloadFolder): raise Exception('%s is not a Folder' % downloadFolder) if taskCallback is not None and not callable(taskCallback): raise Exception('Invalid taskCallback') (taskNum, taskQueueSize, interval, threshold) = bulktasks._checkBulkTasksPara(taskNum, taskQueueSize, interval, threshold) taskCallback = taskCallback if taskCallback is not None else util.lazyCallback executor = bulktasks.ThreadPool(taskNum, taskQueueSize) state = bulktasks.ExecuteProgress() totalTasks = const.LONG(0) totalAmount = const.LONG(0) notifier = progress.ProgressNotifier( progressCallback, totalAmount, interval ) if progressCallback is not None else progress.NONE_NOTIFIER notifier.start() query = GetObjectRequest(imageProcess=imageProcess) prefix = prefix if prefix is not None else '' prefixDir = prefix[:prefix.rfind('/') + 1] for content in _list_objects(obsClient, bucketName, prefix=prefix): objectKey = content.key totalTasks += 1 totalAmount += content.size objectPath = objectKey.replace(prefixDir, '', 1) if objectPath.startswith('/') or objectPath.find( '//') != -1 or objectPath.find('\\') != -1: state._failed_increment() taskCallback(objectKey, Exception('illegal path: %s' % objectKey)) obsClient.log_client.log(ERROR, 'illegal path: %s' % objectKey) continue downloadPath = os.path.join(downloadFolder, objectPath) downloadPath = util.safe_encode(downloadPath) if const.IS_WINDOWS: downloadPath = util.safe_trans_to_gb2312(downloadPath) dirName = os.path.dirname(downloadPath) if not os.path.exists(dirName): try: os.makedirs(dirName, 0o755) except Exception as e: state._failed_increment() taskCallback(objectKey, e) obsClient.log_client.log(ERROR, traceback.format_exc()) continue if objectKey.endswith(('/')): state._successful_increment() elif content.size < threshold: executor.execute(_task_wrap, obsClient, obsClient._getObjectWithNotifier, key=objectKey, taskCallback=taskCallback, state=state, bucketName=bucketName, objectKey=objectKey, getObjectRequest=query, headers=headers, downloadPath=downloadPath, notifier=notifier) else: executor.execute(_task_wrap, obsClient, obsClient._downloadFileWithNotifier, key=objectKey, taskCallback=taskCallback, state=state, bucketName=bucketName, objectKey=objectKey, downloadFile=downloadPath, partSize=partSize, taskNum=subTaskNum, enableCheckpoint=enableCheckpoint, checkpointFile=checkpointFile, header=headers, imageProcess=imageProcess, notifier=notifier) state.total_tasks = totalTasks notifier.totalAmount = totalAmount finally: if executor is not None: executor.shutdown() if notifier is not None: notifier.end() return state
def _download_files_notifier(progressCallback, totalAmount, interval): return progress.ProgressNotifier( progressCallback, totalAmount, interval) if progressCallback is not None else progress.NONE_NOTIFIER