def _upload_part_for_process(self, part, partEtag_infos, upload_infos, status): if status.value == 0: try: resp = self.obsClient.uploadPart(self.bucketName, self.objectKey, part['partNumber'], self._record['uploadId'], self.fileName, isFile=True, partSize=part['length'], offset=part['offset']) except IOError: raise IOError('can not attach file {0}'.format(self.fileName)) if resp.status < 300: complete_part = CompletePart(to_int(part['partNumber']), resp.body.etag) partEtag_infos[to_int(part['partNumber'])] = resp.body.etag upload_infos[to_int(part['partNumber'])-1] = True if self.enableCheckPoint: with self._lock: record = self._get_record() record['uploadParts'][part['partNumber']-1]['isCompleted'] = True record['partEtags'].append(complete_part) self._write_record(record) elif resp.status > 300 and resp.status < 500: self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(resp.errorCode, resp.errorMessage)) self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage)) self._lock.acquire() status.value = 1 self._lock.release() upload_infos[to_int(part['partNumber'])-1] = False else: upload_infos[to_int(part['partNumber'])-1] = False
def _upload_part_for_process(self, func_args, item_args): part = item_args partEtag_infos, upload_infos, status = func_args if status.value == 0: resp = self.real_upload(part) i = 0 while resp.status > 300 and i < TRY_MULTIPART_TIMES: logger.warning('retry %s part, %d times, part number: %d' % (self.cmdtype, i, part['partNumber'])) resp = self.real_upload(part) i += 1 if resp.status < 300: complete_part = CompletePart(to_int(part['partNumber']), resp.body.etag) partEtag_infos[to_int(part['partNumber'])] = resp.body.etag upload_infos[to_int(part['partNumber']) - 1] = True if self.enable_checkpoint: with self._lock: record = self._get_record() record['uploadParts'][part['partNumber'] - 1]['isCompleted'] = True record['partEtags'].append(complete_part) self._write_record(record) pbar_add_size(part['length']) logger.info('%s part %d complete, %s, uploadid=%s' % (self.cmdtype, part['partNumber'], self.filename, self.uploadId)) elif 300 < resp.status < 500: logger.warning('server error. ErrorCode:{0}, ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage)) with self._lock: status.value = 1 upload_infos[to_int(part['partNumber']) - 1] = False globl.append_list_lock('part_task_failed', os.getpid())
def _checkBulkTasksPara(task_num, task_queue_size, task_interval, threshold): origine = [task_num, task_queue_size, task_interval, threshold] default = (const.DEFAULT_TASK_NUM, const.DEFAULT_TASK_QUEUE_SIZE, const.DEFAULT_BYTE_INTTERVAL, const.DEFAULT_MAXIMUM_SIZE) size = len(origine) for i in range(size): origine[i] = util.to_int(origine[i]) if origine[i] is None or origine[i] <= 0: origine[i] = default[i] return tuple(origine)
def _download_part_process(self, part, download_infos, status): if status.value == 0: get_object_request = GetObjectRequest(versionId=self._record['versionId']) self.header.range = str(part['offset'])+'-'+str(part['offset']+part['length']) try: resp = self.obsClient.getObject(bucketName=self._record['bucketName'], objectKey=self._record['objectKey'], getObjectRequest=get_object_request, headers=self.header) if resp.status < 300: respone = resp.body.response chunk_size = 65536 with self._lock: if respone is not None: with open(_to_unicode(self._tmp_file), 'rb+') as f: f.seek(part['offset'], 0) position = to_int(part['offset']) while True: chunk = respone.read(chunk_size) if not chunk: break f.write(chunk) position += chunk_size f.seek(position, 0) respone.close() f.close() download_infos[part['partNumber'] - 1] = True if self.enableCheckPoint: with self._lock: record = self._get_record() record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file) record['downloadParts'][part['partNumber']-1]['isCompleted'] = True self._write_record(record) elif resp.status > 300 and resp.status < 500: with self._lock: download_infos[part['partNumber'] - 1] = False self._lock.acquire() status.value = 1 self._lock.release() self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}' .format(resp.errorCode, resp.errorMessage)) else: download_infos[part['partNumber'] - 1] = False self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}' .format(resp.errorCode, resp.errorMessage)) except Exception as e: download_infos[part['partNumber'] - 1] = False self.obsClient.log_client.log(ERROR, 'something wraong happened. Please check.') if self.enableCheckPoint: with self._lock: record = self._get_record() record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file) self._write_record(record) raise e
def upload(self, flowwith): if self.enable_checkpoint: self._load_record() else: self._prepare() if os.path.exists(self.checkpoint_file): start_total = read_cp_file(self.checkpoint_file, 0, self.cmdtype) pbar_add_size(start_total) self._upload_parts = self._get_upload_parts() part_etag_info = {} for part_info in self._record['partEtags']: part_etag_info[part_info['partNum']] = part_info['etag'] upload_info = [] for part_info in self._record['uploadParts']: upload_info.append(part_info['isCompleted']) part_etag_infos = compat.Dict(part_etag_info) upload_infos = compat.List(upload_info) status = compat.Value('i', 0) multithreading_with_sleep(self._upload_part_for_process, (part_etag_infos, upload_infos, status), self._upload_parts, self.tasknum, flowwith) if False in upload_infos: if status.value == 1: self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId']) logger.warning('the code from server is 4**, please check') self._delete_record() if len(self._exception) > 0: raise Exception(self._exception[0]) raise Exception('%s some parts are failed. Please try agagin, %s' % (self.cmdtype, self.filename)) part_etags = [] tmp_part_etags = sorted(part_etag_infos.items(), key=lambda d: d[0]) for key, value in tmp_part_etags: part_etags.append(CompletePart(partNum=to_int(key), etag=value)) resp = self.obscmdutil.complete_multipart_upload(self.bucket, self.objkey, self._record['uploadId'], CompleteMultipartUploadRequest(part_etags)) if 300 < resp.status < 500: logger.error('complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) self.obscmdutil.abort_multipart_upload(self.bucket, self.objkey, self._record['uploadId']) if self.enable_checkpoint: self._delete_record() return resp
def reset_partSize(self): if self.partSize < const.DEFAULT_MINIMUM_SIZE: self.partSize = const.DEFAULT_MINIMUM_SIZE elif self.partSize > const.DEFAULT_MAXIMUM_SIZE: self.partSize = const.DEFAULT_MAXIMUM_SIZE else: self.partSize = util.to_int(self.partSize) if self.taskNum <= 0: self.taskNum = 1 else: self.taskNum = int(math.ceil(self.taskNum)) self.total_parts = int(self.size / self.partSize) if self.total_parts >= 10000: self.partSize = int(self.size / 10000) if self.size % 10000 == 0 else int(self.size / 10000) + 1 self.total_parts = int(self.size / self.partSize) if self.size % self.partSize != 0: self.total_parts += 1
def _download_part(self, part): get_object_request = GetObjectRequest(versionId=self.versionId) self.header.range = str(part['offset'])+'-'+str(part['offset']+part['length']) try: resp = self.obsClient.getObject(bucketName=self.bucketName, objectKey=self.objectKey, getObjectRequest=get_object_request, headers=self.header) if resp.status < 300: respone = resp.body.response chunk_size = 65536 with self._lock: if respone is not None: with open(_to_unicode(self._tmp_file), 'rb+') as fs: fs.seek(part['offset'], 0) position = to_int(part['offset']) while True: chunk = respone.read(chunk_size) if not chunk: break fs.write(chunk) position += chunk_size fs.seek(position, 0) fs.close() respone.close() self._downinfos.append(True) self._record['downloadParts'][part['partNumber']-1]['isCompleted'] = True elif resp.status > 300 and resp.status < 500: with self._lock: self._downinfos.append(False) self._change_status() self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}' .format(resp.errorCode, resp.errorMessage)) self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}' .format(resp.errorCode, resp.errorMessage)) else: self._downinfos.append(False) self._exception.append('response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}' .format(resp.errorCode, resp.errorMessage)) except Exception as e: self.obsClient.log_client.log(ERROR, 'something wraong happened. Please check.') raise e finally: if self.enableCheckPoint: with self._lock: self._record['tmpFileStatus'][1] = os.path.getmtime(self._tmp_file) self._write_record(self._record)
def _upload_part(self, part): if not self._is_abort(): try: resp = self.get_upload_part_resp(part) if resp.status < 300: self._record['uploadParts'][part['partNumber'] - 1]['isCompleted'] = True self._record['partEtags'].append(CompletePart(util.to_int(part['partNumber']), resp.body.etag)) if self.enableCheckPoint: with self._lock: self._write_record(self._record) else: if 300 < resp.status < 500: self._do_abort('errorCode:{0}, errorMessage:{1}'.format(resp.errorCode, resp.errorMessage)) self.obsClient.log_client.log( ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) except Exception as e: self.obsClient.log_client.log(DEBUG, 'upload part %s error, %s' % (part['partNumber'], e)) self.obsClient.log_client.log(ERROR, traceback.format_exc())
def _upload_part(self, part): if self._status: try: resp = self.obsClient.uploadPart(self.bucketName, self.objectKey, part['partNumber'], self._record['uploadId'], self.fileName, isFile=True, partSize=part['length'], offset=part['offset']) except IOError: raise IOError('can not attach file {0}. Please check'.format(self.fileName)) if resp.status < 300: complete_part = CompletePart(to_int(part['partNumber']), resp.body.etag) with self._lock: self._record['uploadParts'][part['partNumber']-1]['isCompleted'] = True self._record['partEtags'].append(complete_part) self._uploadinfos.append(True) if self.enableCheckPoint: self._write_record(self._record) elif resp.status > 300 and resp.status < 500: self.obsClient.log_client.log(ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}'.format(resp.errorCode, resp.errorMessage)) self._exception.append('errorCode:{0}.errorMessage:{1}'.format(resp.errorCode, resp.errorMessage)) self._change_status() self._uploadinfos.append(False) else: self._uploadinfos.append(False)
def _upload_part(self, part): if not self._is_abort(): try: resp = self.obsClient._uploadPartWithNotifier( self.bucketName, self.objectKey, part['partNumber'], self._record['uploadId'], self.fileName, isFile=True, partSize=part['length'], offset=part['offset'], notifier=self.notifier, extensionHeaders=self.extensionHeaders, sseHeader=self.headers.sseHeader) if resp.status < 300: self._record['uploadParts'][part['partNumber'] - 1]['isCompleted'] = True self._record['partEtags'].append( CompletePart(util.to_int(part['partNumber']), resp.body.etag)) if self.enableCheckPoint: with self._lock: self._write_record(self._record) else: if 300 < resp.status < 500: self._do_abort( 'errorCode:{0}, errorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) self.obsClient.log_client.log( ERROR, 'response from server is something wrong. ErrorCode:{0}, ErrorMessage:{1}' .format(resp.errorCode, resp.errorMessage)) except Exception as e: self.obsClient.log_client.log( DEBUG, 'upload part %s error, %s' % (part['partNumber'], e)) self.obsClient.log_client.log(ERROR, traceback.format_exc())
def _upload(self): if not self.enableCheckPoint: self._prepare() else: self._load() self.__upload_parts = self._get_upload_parts() if IS_WINDOWS: self._uploadinfos = [] self._status = True thread_pools = _ThreadPool(functools.partial(self._produce, upload_parts=self.__upload_parts), [self._consume] * self.taskNum, self._lock) thread_pools.run() if not min(self._uploadinfos): if not self._status: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check space、persimission and so on.') self._delete_record() if len(self._exception) > 0: raise Exception(self._exception[0]) raise Exception('some parts are failed when upload. Please try agagin') part_Etags = [] for part in sorted(self._record['partEtags'], key=lambda x: x['partNum']): part_Etags.append(CompletePart(partNum=part['partNum'], etag=part['etag'])) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags)) else: partEtag_info = {} for part_info in self._record['partEtags']: partEtag_info[part_info['partNum']] = part_info['etag'] partEtag_infos = multiprocessing.Manager().dict(partEtag_info) upload_info = [] for part_info in self._record['uploadParts']: upload_info.append(part_info['isCompleted']) upload_infos = multiprocessing.Manager().list(upload_info) status = multiprocessing.Manager().Value('i', 0) pool = Pool(processes=self.taskNum) for part in self.__upload_parts: pool.apply_async(self._upload_part_for_process(part, partEtag_infos, upload_infos, status)) pool.close() pool.join() if not min(upload_infos): if status.value == 1: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'the code from server is 4**, please check') self._delete_record() if len(self._exception) > 0: raise Exception(self._exception[0]) raise Exception('some parts are failed when upload. Please try agagin') part_Etags = [] partETags = sorted(partEtag_infos.items(), key=lambda d: d[0]) for key, value in partETags: part_Etags.append(CompletePart(partNum=to_int(key), etag=value)) self.obsClient.log_client.log(INFO, 'Completing to upload multiparts') resp = self.obsClient.completeMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId'], CompleteMultipartUploadRequest(part_Etags)) if resp.status < 300: if self.enableCheckPoint: self._delete_record() else: if not self.enableCheckPoint: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) else: if resp.status > 300 and resp.status < 500: self.obsClient.abortMultipartUpload(self.bucketName, self.objectKey, self._record['uploadId']) self.obsClient.log_client.log(ERROR, 'something is wrong when complete multipart.ErrorCode:{0}. ErrorMessage:{1}'.format( resp.errorCode, resp.errorMessage)) self._delete_record() return resp