def __init__(self, callback=None, totalAmount=0, interval=102400): self.callback = callback if self.callback is None or not callable(self.callback): raise Exception('Invalid callback') self.totalAmount = totalAmount self.interval = interval self._transferredAmount = const.LONG(0) self._newlyTransferredAmount = const.LONG(0) self._queue = queue.Queue() self._startCheckpoint = None
def _upload(self): if self.enableCheckPoint: self._load() if self._record is None: self._prepare() unfinished_upload_parts = [] sendedBytes = const.LONG(0) for p in self._record['uploadParts']: if not p['isCompleted']: unfinished_upload_parts.append(p) else: sendedBytes += p['length'] if self.progressCallback is not None: self.notifier = progress.ProgressNotifier(self.progressCallback, self.size) self.notifier.start() try: if len(unfinished_upload_parts) > 0: if (self.size - sendedBytes) > 0: self.notifier.send(sendedBytes) thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=unfinished_upload_parts), [self._consume] * self.taskNum) thread_pools.run() if self._abort: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], extensionHeaders=self.extensionHeaders) self.obsClient.log_client.log( ERROR, 'the code from server is 4**, please check space、persimission and so on.' ) self._delete_record() if self._exception is not None: raise Exception(self._exception) for p in self._record['uploadParts']: if not p['isCompleted']: if not self.enableCheckPoint: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], extensionHeaders=self.extensionHeaders) raise Exception('some parts are failed when upload. Please try again') part_Etags = [] for part in self._record['partEtags']: part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag'])) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags), extensionHeaders=self.extensionHeaders) self._upload_handle_response(resp) return resp finally: self.notifier.end()
def _download_files(obsClient, bucketName, prefix, downloadFolder=None, taskNum=const.DEFAULT_TASK_NUM, taskQueueSize=const.DEFAULT_TASK_QUEUE_SIZE, headers=GetObjectHeader(), imageProcess=None, interval=const.DEFAULT_BYTE_INTTERVAL, taskCallback=None, progressCallback=None, threshold=const.DEFAULT_MAXIMUM_SIZE, partSize=5 * 1024 * 1024, subTaskNum=1, enableCheckpoint=False, checkpointFile=None): try: executor = None notifier = None if downloadFolder is None or not os.path.isdir(downloadFolder): raise Exception('%s is not a Folder' % downloadFolder) if taskCallback is not None and not callable(taskCallback): raise Exception('Invalid taskCallback') (taskNum, taskQueueSize, interval, threshold) = bulktasks._checkBulkTasksPara(taskNum, taskQueueSize, interval, threshold) taskCallback = taskCallback if taskCallback is not None else util.lazyCallback executor = bulktasks.ThreadPool(taskNum, taskQueueSize) state = bulktasks.ExecuteProgress() totalTasks = const.LONG(0) totalAmount = const.LONG(0) notifier = progress.ProgressNotifier( progressCallback, totalAmount, interval ) if progressCallback is not None else progress.NONE_NOTIFIER notifier.start() query = GetObjectRequest(imageProcess=imageProcess) prefix = prefix if prefix is not None else '' prefixDir = prefix[:prefix.rfind('/') + 1] for content in _list_objects(obsClient, bucketName, prefix=prefix): objectKey = content.key totalTasks += 1 totalAmount += content.size objectPath = objectKey.replace(prefixDir, '', 1) if objectPath.startswith('/') or objectPath.find( '//') != -1 or objectPath.find('\\') != -1: state._failed_increment() taskCallback(objectKey, Exception('illegal path: %s' % objectKey)) obsClient.log_client.log(ERROR, 'illegal path: %s' % objectKey) continue downloadPath = os.path.join(downloadFolder, objectPath) downloadPath = util.safe_encode(downloadPath) if const.IS_WINDOWS: downloadPath = util.safe_trans_to_gb2312(downloadPath) dirName = os.path.dirname(downloadPath) if not os.path.exists(dirName): try: os.makedirs(dirName, 0o755) except Exception as e: state._failed_increment() taskCallback(objectKey, e) obsClient.log_client.log(ERROR, traceback.format_exc()) continue if objectKey.endswith(('/')): state._successful_increment() elif content.size < threshold: executor.execute(_task_wrap, obsClient, obsClient._getObjectWithNotifier, key=objectKey, taskCallback=taskCallback, state=state, bucketName=bucketName, objectKey=objectKey, getObjectRequest=query, headers=headers, downloadPath=downloadPath, notifier=notifier) else: executor.execute(_task_wrap, obsClient, obsClient._downloadFileWithNotifier, key=objectKey, taskCallback=taskCallback, state=state, bucketName=bucketName, objectKey=objectKey, downloadFile=downloadPath, partSize=partSize, taskNum=subTaskNum, enableCheckpoint=enableCheckpoint, checkpointFile=checkpointFile, header=headers, imageProcess=imageProcess, notifier=notifier) state.total_tasks = totalTasks notifier.totalAmount = totalAmount finally: if executor is not None: executor.shutdown() if notifier is not None: notifier.end() return state
def to_long(item): try: return const.LONG(item) except Exception: return None