def list(self, prefix=None, marker=None, limit=None):
"""前缀查询:
* bucket => str
* prefix => str
* marker => str
* limit => int
* return ret => {'items': items, 'marker': markerOut}, err => str
1. 首次请求 marker = None
2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容
3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束）
"""
options = {
'bucket': self.bucket,
}
if marker is not None:
options['marker'] = marker
if limit is not None:
options['limit'] = limit
if prefix is not None:
options['prefix'] = prefix
url = 'http://{0}/list'.format(config.RSF_HOST)
r = self.__get(url, options)
ret = _ret(r)
eof = False
if ret and not ret.get('marker'):
eof = True
return ret, eof
def make_block(self, block, block_size):
crc = crc32(block)
url = self.block_url(config.get_default('default_up_host'), block_size)
r = None
exception = None
try:
r = self.post(url, block)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = self.block_url(config.UPBACKUP_HOST, block_size)
try:
r = self.post(url, block)
except Exception as e:
raise QiniuClientException(str(e))
ret = _ret(r)
if ret['crc32'] != crc:
raise QiniuServiceException(r.status_code, 'unmatch crc checksum',
r.headers['X-Reqid'])
return ret
def fetch(self, url, key):
to = self.__entry(key)
resource = urlsafe_base64_encode(url)
cmd = 'http://{0}/fetch/{1}/to/{2}'.format(config.IO_HOST, resource,
to)
r = self.__post(cmd)
return _ret(r)
def _put(up_token, key, data, params, mime_type, crc32, is_file=False):
fields = {}
if params:
for k, v in params.items():
fields[k] = str(v)
if crc32:
fields['crc32'] = crc32
if key is not None:
fields['key'] = key
fields['token'] = up_token
url = 'http://' + config.get_default('default_up_host') + '/'
name = key if key else 'filename'
r = None
exception = None
headers = {'User-Agent': config.USER_AGENT}
try:
r = _post(url, data=fields, files={'file': (name, data, mime_type)}, headers=headers)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = 'http://' + config.UPBACKUP_HOST + '/'
if is_file:
data.seek(0)
try:
r = _post(url, data=fields, files={'file': (name, data, mime_type)}, headers=headers)
except Exception as e:
raise QiniuClientException(str(e))
return _ret(r)
def make_block(self, block, block_size):
crc = crc32(block)
url = self.block_url(config.get_default('default_up_host'), block_size)
r = None
exception = None
try:
r = self.post(url, block)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = self.block_url(config.UPBACKUP_HOST, block_size)
try:
r = self.post(url, block)
except Exception as e:
raise QiniuClientException(str(e))
ret = _ret(r)
if ret['crc32'] != crc:
raise QiniuServiceException(
r.status_code, 'unmatch crc checksum', r.headers['X-Reqid'])
return ret
def copy(self, key_pairs, target_bucket=None):
ops = []
url = 'http://{0}/batch'.format(config.RS_HOST)
for k, v in key_pairs.items():
to = entry(v, target_bucket) if target_bucket else self.__entry(v)
ops.append("/copy/{0}/{1}".format(self.__entry(k), to))
r = self.__post(url, dict(op=ops))
return _ret(r)
def delete(self, keys):
url = None
params = None
if isinstance(keys, str):
resource = self.__entry(keys)
url = 'http://{0}/delete/{1}'.format(config.RS_HOST, resource)
else:
ops = []
for key in keys:
ops.append("/delete/{0}".format(self.__entry(key)))
url = 'http://{0}/batch'.format(config.RS_HOST)
params = dict(op=ops)
r = self.__post(url, params)
return _ret(r)
def pfop(auth, bucket, key, fops, pipeline=None, notify_url=None):
ops = '|'.join(fops)
data = {'bucket': bucket, 'key': key, 'fops': ops}
if pipeline:
data['pipeline'] = pipeline
if notify_url:
data['notifyURL'] = notify_url
headers = {'User-Agent': config.USER_AGENT}
url = 'http://{0}/pfop'.format(config.API_HOST)
r = requests.post(
url, data=data, auth=RequestsAuth(auth),
timeout=config.get_default('connection_timeout'), headers=headers)
return _ret(r)
def pfop(auth, bucket, key, fops, pipeline=None, notify_url=None):
ops = '|'.join(fops)
data = {'bucket': bucket, 'key': key, 'fops': ops}
if pipeline:
data['pipeline'] = pipeline
if notify_url:
data['notifyURL'] = notify_url
headers = {'User-Agent': config.USER_AGENT}
url = 'http://{0}/pfop'.format(config.API_HOST)
r = requests.post(url,
data=data,
auth=RequestsAuth(auth),
timeout=config.get_default('connection_timeout'),
headers=headers)
return _ret(r)
def make_file(self):
url = self.make_file_url(config.get_default('default_up_host'))
body = ','.join([status['ctx'] for status in self.blockStatus])
r = None
exception = None
try:
r = self.post(url, body)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = self.make_file_url(config.UPBACKUP_HOST)
try:
r = self.post(url, body)
except Exception as e:
raise QiniuClientException(str(e))
return _ret(r)
def make_file(self):
url = self.make_file_url(config.get_default('default_up_host'))
body = ','.join([status['ctx'] for status in self.blockStatus])
r = None
exception = None
try:
r = self.post(url, body)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = self.make_file_url(config.UPBACKUP_HOST)
try:
r = self.post(url, body)
except Exception as e:
raise QiniuClientException(str(e))
return _ret(r)
def _put(up_token, key, data, params, mime_type, crc32, is_file=False):
fields = {}
if params:
for k, v in params.items():
fields[k] = str(v)
if crc32:
fields['crc32'] = crc32
if key is not None:
fields['key'] = key
fields['token'] = up_token
url = 'http://' + config.get_default('default_up_host') + '/'
name = key if key else 'filename'
r = None
exception = None
headers = {'User-Agent': config.USER_AGENT}
try:
r = _post(url,
data=fields,
files={'file': (name, data, mime_type)},
headers=headers)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = 'http://' + config.UPBACKUP_HOST + '/'
if is_file:
data.seek(0)
try:
r = _post(url,
data=fields,
files={'file': (name, data, mime_type)},
headers=headers)
except Exception as e:
raise QiniuClientException(str(e))
return _ret(r)
def buckets(self):
url = 'http://{0}/buckets'.format(config.RS_HOST)
r = self.__post(url)
return _ret(r)
def prefetch(self, key):
resource = self.__entry(key)
url = 'http://{0}/prefetch/{1}'.format(config.IO_HOST, resource)
r = self.__post(url)
return _ret(r)
