def _init():
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(
pool_connections=config.get_default('connection_pool'),
pool_maxsize=config.get_default('connection_pool'),
max_retries=config.get_default('connection_retries'))
session.mount('http://', adapter)
global _session
_session = session
def _get(url, params, auth):
try:
r = requests.get(
url,
params=params,
auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None,
timeout=config.get_default('connection_timeout'),
headers=_headers)
except Exception as e:
return None, ResponseInfo(None, e)
return __return_wrapper(r)
def upload(self):
"""上传操作"""
self.blockStatus = []
if config.get_default('default_zone').up_host:
host = config.get_default('default_zone').up_host
else:
host = config.get_default('default_zone').get_up_host_by_token(
self.up_token)
offset = self.recovery_from_record()
for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset):
length = len(block)
crc = crc32(block)
ret, info = self.make_block(block, length, host)
if ret is None and not info.need_retry():
return ret, info
if info.connect_failed():
if config.get_default('default_zone').up_host_backup:
host = config.get_default('default_zone').up_host_backup
else:
host = config.get_default(
'default_zone').get_up_host_backup_by_token(
self.up_token)
if info.need_retry() or crc != ret['crc32']:
ret, info = self.make_block(block, length, host)
if ret is None or crc != ret['crc32']:
return ret, info
self.blockStatus.append(ret)
offset += length
self.record_upload_progress(offset)
if (callable(self.progress_handler)):
self.progress_handler(
((len(self.blockStatus) - 1) * config._BLOCK_SIZE) +
length, self.size)
return self.make_file(host)
def _post_with_qiniu_mac(url, data, auth):
qn_auth = qiniu.auth.QiniuMacRequestsAuth(
auth) if auth is not None else None
timeout = config.get_default('connection_timeout')
try:
r = requests.post(url,
json=data,
auth=qn_auth,
timeout=timeout,
headers=_headers)
except Exception as e:
return None, ResponseInfo(None, e)
return __return_wrapper(r)
def _form_put(up_token,
key,
data,
params,
mime_type,
crc,
progress_handler=None,
file_name=None,
modify_time=None,
keep_last_modified=False):
fields = {}
if params:
for k, v in params.items():
fields[k] = str(v)
if crc:
fields['crc32'] = crc
if key is not None:
fields['key'] = key
fields['token'] = up_token
if config.get_default('default_zone').up_host:
url = config.get_default('default_zone').up_host
else:
url = config.get_default('default_zone').get_up_host_by_token(up_token)
# name = key if key else file_name
fname = file_name
if not fname or not fname.strip():
fname = 'file_name'
# last modify time
if modify_time and keep_last_modified:
fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time)
r, info = http._post_file(url,
data=fields,
files={'file': (fname, data, mime_type)})
if r is None and info.need_retry():
if info.connect_failed:
if config.get_default('default_zone').up_host_backup:
url = config.get_default('default_zone').up_host_backup
else:
url = config.get_default(
'default_zone').get_up_host_backup_by_token(up_token)
if hasattr(data, 'read') is False:
pass
elif hasattr(data, 'seek') and (not hasattr(data, 'seekable')
or data.seekable()):
data.seek(0)
else:
return r, info
r, info = http._post_file(url,
data=fields,
files={'file': (fname, data, mime_type)})
return r, info
def _delete_with_qiniu_mac_and_headers(url, params, auth, headers):
try:
post_headers = _headers.copy()
if headers is not None:
for k, v in headers.items():
post_headers.update({k: v})
r = requests.delete(url,
params=params,
auth=qiniu.auth.QiniuMacRequestsAuth(auth)
if auth is not None else None,
timeout=config.get_default('connection_timeout'),
headers=post_headers)
except Exception as e:
return None, ResponseInfo(None, e)
return __return_wrapper(r)
def _put(url, data, files, auth, headers=None):
if _session is None:
_init()
try:
post_headers = _headers.copy()
if headers is not None:
for k, v in headers.items():
post_headers.update({k: v})
r = _session.put(url,
data=data,
files=files,
auth=auth,
headers=post_headers,
timeout=config.get_default('connection_timeout'))
except Exception as e:
return None, ResponseInfo(None, e)
return __return_wrapper(r)
def execute(self, key, fops, force=None):
"""执行持久化处理:
Args:
key: 待处理的源文件
fops: 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop
force: 强制执行持久化处理开关
Returns:
一个dict变量，返回持久化处理的persistentId，类似{"persistentId": 5476bedf7823de4068253bae};
一个ResponseInfo对象
"""
ops = ';'.join(fops)
data = {'bucket': self.bucket, 'key': key, 'fops': ops}
if self.pipeline:
data['pipeline'] = self.pipeline
if self.notify_url:
data['notifyURL'] = self.notify_url
if force == 1:
data['force'] = 1
url = '{0}/pfop'.format(config.get_default('default_api_host'))
return http._post_with_auth(url, data, self.auth)