def put_data(up_token,
key,
data,
params=None,
mime_type='application/octet-stream',
check_crc=False,
progress_handler=None,
fname=None):
"""上传二进制流到七牛
Args:
up_token: 上传凭证
key: 上传文件名
data: 上传二进制流
params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
mime_type: 上传数据的mimeType
check_crc: 是否校验crc32
progress_handler: 上传进度
Returns:
一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
一个ResponseInfo对象
"""
crc = crc32(data) if check_crc else None
return _form_put(up_token, key, data, params, mime_type, crc,
progress_handler, fname)
def upload(self):
"""上传操作"""
self.blockStatus = []
host = config.get_default('default_zone').get_up_host_by_token(
self.up_token)
offset = self.recovery_from_record()
for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset):
length = len(block)
crc = crc32(block)
ret, info = self.make_block(block, length, host)
if ret is None and not info.need_retry():
return ret, info
if info.connect_failed():
host = config.get_default(
'default_zone').get_up_host_backup_by_token(self.up_token)
if info.need_retry() or crc != ret['crc32']:
ret, info = self.make_block(block, length, host)
if ret is None or crc != ret['crc32']:
return ret, info
self.blockStatus.append(ret)
offset += length
self.record_upload_progress(offset)
if (callable(self.progress_handler)):
self.progress_handler(
((len(self.blockStatus) - 1) * config._BLOCK_SIZE) +
length, self.size)
return self.make_file(host)
def make_block(self, block, block_size):
crc = crc32(block)
url = self.block_url(config.get_default('default_up_host'), block_size)
r = None
exception = None
try:
r = self.post(url, block)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = self.block_url(config.UPBACKUP_HOST, block_size)
try:
r = self.post(url, block)
except Exception as e:
raise QiniuClientException(str(e))
ret = _ret(r)
if ret['crc32'] != crc:
raise QiniuServiceException(r.status_code, 'unmatch crc checksum',
r.headers['X-Reqid'])
return ret
def make_block(self, block, block_size):
crc = crc32(block)
url = self.block_url(config.get_default('default_up_host'), block_size)
r = None
exception = None
try:
r = self.post(url, block)
except Exception as e:
exception = e
finally:
retry = _need_retry(r, exception)
if retry:
url = self.block_url(config.UPBACKUP_HOST, block_size)
try:
r = self.post(url, block)
except Exception as e:
raise QiniuClientException(str(e))
ret = _ret(r)
if ret['crc32'] != crc:
raise QiniuServiceException(
r.status_code, 'unmatch crc checksum', r.headers['X-Reqid'])
return ret
def put_data(
up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None):
''' put data to Qiniu
If key is None, the server will generate one.
data may be str or read()able object.
'''
crc = crc32(data) if check_crc else None
return _form_put(up_token, key, data, params, mime_type, crc, False, progress_handler)
def put(up_token,
key,
data,
params=None,
mime_type='application/octet-stream',
check_crc=False):
''' put data to Qiniu
If key is None, the server will generate one.
data may be str or read()able object.
'''
crc = crc32(data) if check_crc else None
return _put(up_token, key, data, params, mime_type, crc)
def put_data(
up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None):
"""上传二进制流到七牛
Args:
up_token: 上传凭证
key: 上传文件名
data: 上传二进制流
params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
mime_type: 上传数据的mimeType
check_crc: 是否校验crc32
progress_handler: 上传进度
Returns:
一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
一个ResponseInfo对象
"""
crc = crc32(data) if check_crc else None
return _form_put(up_token, key, data, params, mime_type, crc, progress_handler)
def upload(self):
self.blockStatus = []
host = config.get_default('default_up_host')
for block in _file_iter(self.input_stream, config._BLOCK_SIZE):
length = len(block)
crc = crc32(block)
ret, info = self.make_block(block, length, host)
if ret is None and not info.need_retry:
return ret, info
if info.connect_failed:
host = config.UPBACKUP_HOST
if info.need_retry or crc != ret['crc32']:
ret, info = self.make_block(block, length, host)
if ret is None or crc != ret['crc32']:
return ret, info
self.blockStatus.append(ret)
if(callable(self.progress_handler)):
self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size)
return self.make_file(host)
def put_data(up_token,
key,
data,
params=None,
mime_type='application/octet-stream',
check_crc=False,
progress_handler=None,
fname=None,
hostscache_dir=None):
"""上传二进制流到七牛
Args:
up_token: 上传凭证
key: 上传文件名
data: 上传二进制流
params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
mime_type: 上传数据的mimeType
check_crc: 是否校验crc32
progress_handler: 上传进度
hostscache_dir： host请求 缓存文件保存位置
Returns:
一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
一个ResponseInfo对象
"""
final_data = b''
if hasattr(data, 'read'):
while True:
tmp_data = data.read(config._BLOCK_SIZE)
if len(tmp_data) == 0:
break
else:
final_data += tmp_data
else:
final_data = data
crc = crc32(final_data)
return _form_put(up_token, key, final_data, params, mime_type, crc,
hostscache_dir, progress_handler, fname)
def upload(self):
self.blockStatus = []
host = config.get_default('default_up_host')
for block in _file_iter(self.input_stream, config._BLOCK_SIZE):
length = len(block)
crc = crc32(block)
ret, info = self.make_block(block, length, host)
if ret is None and not info.need_retry:
return ret, info
if info.connect_failed:
host = config.UPBACKUP_HOST
if info.need_retry or crc != ret['crc32']:
ret, info = self.make_block(block, length, host)
if ret is None or crc != ret['crc32']:
return ret, info
self.blockStatus.append(ret)
if (callable(self.progress_handler)):
self.progress_handler(
((len(self.blockStatus) - 1) * config._BLOCK_SIZE) +
length, self.size)
return self.make_file(host)
def upload(self):
"""上传操作"""
self.blockStatus = []
host = config.get_default('default_up_host')
offset = self.recovery_from_record()
for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset):
length = len(block)
crc = crc32(block)
ret, info = self.make_block(block, length, host)
if ret is None and not info.need_retry():
return ret, info
if info.connect_failed():
host = config.get_default('default_up_host_backup')
if info.need_retry() or crc != ret['crc32']:
ret, info = self.make_block(block, length, host)
if ret is None or crc != ret['crc32']:
return ret, info
self.blockStatus.append(ret)
offset += length
self.record_upload_progress(offset)
if(callable(self.progress_handler)):
self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size)
return self.make_file(host)
def upload(self):
"""上传操作"""
self.blockStatus = []
self.recovery_index = 1
self.expiredAt = None
self.uploadId = None
self.get_bucket()
host = self.get_up_host()
if self.version == 'v1':
offset = self.recovery_from_record()
self.part_size = config._BLOCK_SIZE
elif self.version == 'v2':
offset, self.uploadId, self.expiredAt = self.recovery_from_record()
if offset > 0 and self.blockStatus != [] and self.uploadId is not None \
and self.expiredAt is not None:
self.recovery_index = self.blockStatus[-1]['partNumber'] + 1
else:
self.recovery_index = 1
init_url = self.block_url_v2(host, self.bucket_name)
self.uploadId, self.expiredAt = self.init_upload_task(init_url)
else:
raise ValueError("version must choose v1 or v2 !")
for index, block in enumerate(
_file_iter(self.input_stream, self.part_size, offset)):
length = len(block)
if self.version == 'v1':
crc = crc32(block)
ret, info = self.make_block(block, length, host)
elif self.version == 'v2':
index_ = index + self.recovery_index
url = self.block_url_v2(
host,
self.bucket_name) + '/%s/%d' % (self.uploadId, index_)
ret, info = self.make_block_v2(block, url)
if ret is None and not info.need_retry():
return ret, info
if info.connect_failed():
if config.get_default('default_zone').up_host_backup:
host = config.get_default('default_zone').up_host_backup
else:
host = config.get_default(
'default_zone').get_up_host_backup_by_token(
self.up_token, self.hostscache_dir)
if self.version == 'v1':
if info.need_retry() or crc != ret['crc32']:
ret, info = self.make_block(block, length, host)
if ret is None or crc != ret['crc32']:
return ret, info
elif self.version == 'v2':
if info.need_retry():
url = self.block_url_v2(
host, self.bucket_name) + '/%s/%d' % (self.uploadId,
index + 1)
ret, info = self.make_block_v2(block, url)
if ret is None:
return ret, info
del ret['md5']
ret['partNumber'] = index_
self.blockStatus.append(ret)
offset += length
self.record_upload_progress(offset)
if (callable(self.progress_handler)):
self.progress_handler(
((len(self.blockStatus) - 1) * self.part_size) +
len(block), self.size)
if self.version == 'v1':
return self.make_file(host)
elif self.version == 'v2':
make_file_url = self.block_url_v2(
host, self.bucket_name) + '/%s' % self.uploadId
return self.make_file_v2(self.blockStatus, make_file_url,
self.file_name, self.mime_type,
self.params)
