def _init(): session = requests.Session() adapter = requests.adapters.HTTPAdapter( pool_connections=config.get_default('connection_pool'), pool_maxsize=config.get_default('connection_pool'), max_retries=config.get_default('connection_retries')) session.mount('http://', adapter) global _session _session = session
def _get(url, params, auth): try: r = requests.get( url, params=params, auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, timeout=config.get_default('connection_timeout'), headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r)
def upload(self): """上传操作""" self.blockStatus = [] if config.get_default('default_zone').up_host: host = config.get_default('default_zone').up_host else: host = config.get_default('default_zone').get_up_host_by_token( self.up_token) offset = self.recovery_from_record() for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) crc = crc32(block) ret, info = self.make_block(block, length, host) if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): if config.get_default('default_zone').up_host_backup: host = config.get_default('default_zone').up_host_backup else: host = config.get_default( 'default_zone').get_up_host_backup_by_token( self.up_token) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: return ret, info self.blockStatus.append(ret) offset += length self.record_upload_progress(offset) if (callable(self.progress_handler)): self.progress_handler( ((len(self.blockStatus) - 1) * config._BLOCK_SIZE) + length, self.size) return self.make_file(host)
def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth) if auth is not None else None timeout = config.get_default('connection_timeout') try: r = requests.post(url, json=data, auth=qn_auth, timeout=timeout, headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r)
def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None, modify_time=None, keep_last_modified=False): fields = {} if params: for k, v in params.items(): fields[k] = str(v) if crc: fields['crc32'] = crc if key is not None: fields['key'] = key fields['token'] = up_token if config.get_default('default_zone').up_host: url = config.get_default('default_zone').up_host else: url = config.get_default('default_zone').get_up_host_by_token(up_token) # name = key if key else file_name fname = file_name if not fname or not fname.strip(): fname = 'file_name' # last modify time if modify_time and keep_last_modified: fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: if config.get_default('default_zone').up_host_backup: url = config.get_default('default_zone').up_host_backup else: url = config.get_default( 'default_zone').get_up_host_backup_by_token(up_token) if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): data.seek(0) else: return r, info r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) return r, info
def _delete_with_qiniu_mac_and_headers(url, params, auth, headers): try: post_headers = _headers.copy() if headers is not None: for k, v in headers.items(): post_headers.update({k: v}) r = requests.delete(url, params=params, auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, timeout=config.get_default('connection_timeout'), headers=post_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r)
def _put(url, data, files, auth, headers=None): if _session is None: _init() try: post_headers = _headers.copy() if headers is not None: for k, v in headers.items(): post_headers.update({k: v}) r = _session.put(url, data=data, files=files, auth=auth, headers=post_headers, timeout=config.get_default('connection_timeout')) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r)
def execute(self, key, fops, force=None): """执行持久化处理: Args: key: 待处理的源文件 fops: 处理详细操作,规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop force: 强制执行持久化处理开关 Returns: 一个dict变量,返回持久化处理的persistentId,类似{"persistentId": 5476bedf7823de4068253bae}; 一个ResponseInfo对象 """ ops = ';'.join(fops) data = {'bucket': self.bucket, 'key': key, 'fops': ops} if self.pipeline: data['pipeline'] = self.pipeline if self.notify_url: data['notifyURL'] = self.notify_url if force == 1: data['force'] = 1 url = '{0}/pfop'.format(config.get_default('default_api_host')) return http._post_with_auth(url, data, self.auth)