def test_put_quote_key():
data = r(100)
key = 'a\\b\\c"你好' + r(9)
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'].encode('utf8') == key
data = r(100)
key = u'a\\b\\c"你好' + r(9)
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'] == key
def test_put_quote_key():
data = r(100)
key = 'a\\b\\c"你好' + r(9)
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'].encode('utf8') == key
data = r(100)
key = u'a\\b\\c"你好' + r(9)
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'] == key
def post(self):
mimeType = 'application/json'
if self.request.headers.get('Content-Type', '') == mimeType:
data = self.request.body
dataObj = json.loads(data)
pid = dataObj.get('id', 'default')
conf.ACCESS_KEY = STORAGE_ACCESS_KEY
conf.SECRET_KEY = STORAGE_SECRET_KEY
storagePolicy = rs.PutPolicy(STORAGE_BUCKET)
storageToken = storagePolicy.token()
extra = io.PutExtra()
extra.mime_type = mimeType
io.put(storageToken, str(pid), data, extra)
return
def test_put_same_crc():
key = "test_%s" % r(9)
data = "hello bubby!"
extra.check_crc = 2
ret, err = io.put(policy.token(), key, data, extra)
assert err is None
assert ret['key'] == key
def test_put_same_crc(): key = "test_%s" % r(9) params = "op=3" data = "hello bubby!" extra.check_crc = 2 ret, err = io.put(policy.token(), key, data, extra) assert err is None
def test_put_crc_fail(self):
key = "test_%s" % r(9)
data = "hello bubby!"
extra.check_crc = 2
extra.crc32 = "wrong crc32"
ret, err = io.put(policy.token(), key, data, extra)
assert err is not None
def test_put_unicode3():
key = "test_%s" % r(9) + '你好'
data = key
key = key.decode('utf8')
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret[u'key'].endswith(u'你好')
def test_put_unicode3():
key = "test_%s" % r(9) + '你好'
data = key
key = key.decode('utf8')
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret[u'key'].endswith(u'你好')
def post(self):
mimeType = 'application/json'
if self.request.headers.get('Content-Type', '') == mimeType:
data = self.request.body
dataObj = json.loads(data)
pid = dataObj.get('id', 'default')
conf.ACCESS_KEY = STORAGE_ACCESS_KEY
conf.SECRET_KEY = STORAGE_SECRET_KEY
storagePolicy = rs.PutPolicy(STORAGE_BUCKET)
storageToken = storagePolicy.token()
extra = io.PutExtra()
extra.mime_type = mimeType
io.put(storageToken, str(pid), data, extra)
return
def test_put_same_crc():
key = "test_%s" % r(9)
data = "hello bubby!"
extra.check_crc = 2
ret, err = io.put(policy.token(), key, data, extra)
assert err is None
assert ret['key'] == key
def put(self, img_url, key=None, rename=None):
"""
put your data to qiniu. If key is None, the server will generate one.
Note: img_url is web url.
"""
r = None
if img_url is None:
raise ValueError('upload img path to qiniu error!')
for i in range(5):
r = requests.get(img_url, timeout=10.0)
if r is not None:
break
policy = rs.PutPolicy(bucket_name)
policy.saveKey = rename
up_token = policy.token()
ret, err = io.put(up_token, key, StringIO(r.content))
if err is not None:
sys.stderr.write('error: %s ' % err)
return ret['key']
else:
# print 'upload img ok.'
pass
return ret
def test_put_crc_fail(self):
key = "test_%s" % r(9)
data = "hello bubby!"
extra.check_crc = 2
extra.crc32 = "wrong crc32"
ret, err = io.put(policy.token(), key, data, extra)
assert err is not None
def process_item(self, item, spider):
# 获得上传权限
qiniu.conf.ACCESS_KEY = "QBsaz_MsErywKS2kkQpwJlIIvBYmryNuPzoGvHJF"
qiniu.conf.SECRET_KEY = "OTi4GrXf8CQQ0ZLit6Wgy3P8MxFIueqMOwBJhBti"
# 配置上传策略。
# 其中lvxingpai是上传空间的名称（或者成为bucket名称）
policy = qiniu.rs.PutPolicy('lvxingpai-img-store')
# 取得上传token
uptoken = policy.token()
# 上传的额外选项
extra = io.PutExtra()
# 文件自动校验crc
extra.check_crc = 1
upload_stream = False
# 将相应的数据存入mongo中
client = pymongo.MongoClient('zephyre.me', 27017)
db = client.imagestore
# 检查是否已经入mongo库
if db.Hotel.find_one({'url_hash': str(item['hash_value'])}) is None:
# 先生成本地文件
localfile = str(time.time()) + str(random.random())
with open(localfile, 'wb') as f:
f.write(item['pic'])
# 上传
if upload_stream:
# 上传流
with open(localfile, 'rb') as f:
body = f.read()
ret, err = io.put(uptoken, str(item['key']), body, extra)
else:
# 上传本地文件
ret, err = io.put_file(uptoken, str(item['key']), localfile,
extra)
if err is not None:
sys.stderr.write('error: %s ' % err)
return
# 计算文件大小，进入mongo
file_size = int(getsize(localfile))
db.Hotel.save({
'url': item['url'],
'key': item['key'],
'url_hash': item['hash_value'],
'ret_hash': ret['hash'],
'size': file_size
})
# 增加索引
db.Hotel.create_index('url_hash')
# 删除上传成功的文件
os.remove(localfile)
return item
def writeQnData(self, key, mf): while True: ret, err = io.put(uptoken, key, mf, extra) if err is not None: logger.error(err) else: logger.info(ret) break
def test_put(): key = "test_%s" % r(9) params = "op=3" data = "hello bubby!" extra.check_crc = 2 extra.crc32 = zlib.crc32(data) & 0xFFFFFFFF ret, err = io.put(policy.token(), key, data, extra) assert err is None
def put_file(self, name, content):
bucket = BUCKET_KEY
policy = rs.PutPolicy(bucket)
token = policy.token()
ret, err = io.put(token, name, content)
if err is not None:
raise IOError("QiniuStorageError: %s", err)
return self.img_url(name)
def test_put():
key = "test_%s" % r(9)
# params = "op=3"
data = "hello bubby!"
extra.check_crc = 2
extra.crc32 = binascii.crc32(data) & 0xFFFFFFFF
ret, err = io.put(policy.token(), key, data, extra)
assert err is None
assert ret['key'] == key
def qvttUpload(self, key):
domain = self.conf.get('qiniu', 'domain')
accessKey = self.conf.get('qiniu', 'accesskey')
secretKey = self.conf.get('qiniu', 'secretkey')
bucket = self.conf.get('qiniu', 'bucket')
qConf.ACCESS_KEY = accessKey
qConf.SECRET_KEY = secretKey
policy = qRs.PutPolicy(bucket)
uploadToken = policy.token()
url = '%s/%s' % (domain, key)
key = '%s.vtt' % (key, )
data = makeContent(url)
qIo.put(uploadToken, key, data)
return
def _put_file(self, name, content):
if self.encrypt_func:
content = self.encrypt_func(content)
policy = rs.PutPolicy(self.bucket)
uptoken = policy.token()
ret, err = io.put(uptoken, name, content)
if err is not None:
raise IOError("QiniuStorageError: %s", err)
def save_file_qiniu(binary, filename, mime="application/octet-stream"):
policy = rs.PutPolicy(QiniuConf.BUCKET_NAME)
uptoken = policy.token()
extra = io.PutExtra()
extra.mime_type = mime
res, err = io.put(uptoken, filename, binary, extra)
if err is not None:
return 'none'
url = rs.make_base_url(QiniuConf.BUCKET_DOMAIN, filename)
return url
def _put_file(self, name, content):
if self.encrypt_func:
content = self.encrypt_func(content)
policy = rs.PutPolicy(self.bucket)
uptoken = policy.token()
ret, err = io.put(uptoken, name, content)
if err is not None:
raise IOError("QiniuStorageError: %s", err)
def process_item(self, item, spider):
# 获得上传权限
qiniu.conf.ACCESS_KEY = "QBsaz_MsErywKS2kkQpwJlIIvBYmryNuPzoGvHJF"
qiniu.conf.SECRET_KEY = "OTi4GrXf8CQQ0ZLit6Wgy3P8MxFIueqMOwBJhBti"
# 配置上传策略。
# 其中lvxingpai是上传空间的名称（或者成为bucket名称）
policy = qiniu.rs.PutPolicy('lvxingpai-img-store')
# 取得上传token
uptoken = policy.token()
# 上传的额外选项
extra = io.PutExtra()
# 文件自动校验crc
extra.check_crc = 1
upload_stream = False
# 将相应的数据存入mongo中
client = pymongo.MongoClient('zephyre.me', 27017)
db = client.imagestore
# 检查是否已经入mongo库
if db.Hotel.find_one({'url_hash': str(item['hash_value'])}) is None:
# 先生成本地文件
localfile = str(time.time()) + str(random.random())
with open(localfile, 'wb') as f:
f.write(item['pic'])
# 上传
if upload_stream:
# 上传流
with open(localfile, 'rb') as f:
body = f.read()
ret, err = io.put(uptoken, str(item['key']), body, extra)
else:
# 上传本地文件
ret, err = io.put_file(uptoken, str(item['key']), localfile, extra)
if err is not None:
sys.stderr.write('error: %s ' % err)
return
# 计算文件大小，进入mongo
file_size = int(getsize(localfile))
db.Hotel.save({'url': item['url'], 'key': item['key'],
'url_hash': item['hash_value'], 'ret_hash': ret['hash'], 'size': file_size})
# 增加索引
db.Hotel.create_index('url_hash')
# 删除上传成功的文件
os.remove(localfile)
return item
def post(self):
files = self.request.files.get('upload_file')
if not files:
self.write('no file')
return
data = files[0].get('body')
key = 'upload_app'
putPolicy = qRs.PutPolicy(scope='%s:%s' % (BUCKET, key))
uptoken = putPolicy.token()
print 'up up uploading'
ret, err = qIo.put(uptoken, key, data)
if err:
self.write(err)
else:
self.write('http://%s/%s' % (QINIU_DOMAIN, key))
return
def test_put_no_length():
class test_reader(object):
def __init__(self):
self.data = 'abc'
self.pos = 0
def read(self, n=None):
if n is None or n < 0:
newpos = len(self.data)
else:
newpos = min(self.pos+n, len(self.data))
r = self.data[self.pos: newpos]
self.pos = newpos
return r
key = "test_%s" % r(9)
data = test_reader()
extra.check_crc = 2
extra.crc32 = binascii.crc32('abc') & 0xFFFFFFFF
ret, err = io.put(policy.token(), key, data, extra)
assert err is None
assert ret['key'] == key
def save_file_qiniu(binary, filename, mime="application/octet-stream"):
today = datetime.now().strftime("%Y/%m/%d/")
filename = today + filename
policy = rs.PutPolicy(QINIU_SETTINGS.BUCKET_NAME)
uptoken = policy.token()
extra = io.PutExtra()
extra.mime_type = mime
res, err = io.put(uptoken, filename, binary, extra)
if err is not None:
raise Exception("Qiniu save file [%s] error: %s res: %s" % (filename, err, res))
url = rs.make_base_url(QINIU_SETTINGS.BUCKET_DOMAIN, filename)
iv = fop.ImageView()
iv.mode = 2
iv.width = THUMB_SIZE[0]
url_thumb = iv.make_request(url)
return url, url_thumb
def save_file_qiniu(binary, filename, mime="application/octet-stream"):
today = datetime.now().strftime("%Y/%m/%d/")
filename = today + filename
policy = rs.PutPolicy(QINIU_SETTINGS.BUCKET_NAME)
uptoken = policy.token()
extra = io.PutExtra()
extra.mime_type = mime
res, err = io.put(uptoken, filename, binary, extra)
if err is not None:
raise Exception("Qiniu save file [%s] error: %s res: %s" % (filename, err, res))
url = rs.make_base_url(QINIU_SETTINGS.BUCKET_DOMAIN, filename)
iv = fop.ImageView()
iv.mode = 2
iv.width = THUMB_SIZE[0]
url_thumb = iv.make_request(url)
return url, url_thumb
def test_put_no_key():
data = r(100)
extra.check_crc = 0
ret, err = io.put(policy.token(), key=None, data=data, extra=extra)
assert err is None
assert ret['hash'] == ret['key']
def test_put_unicode1():
key = "test_%s" % r(9) + '你好'
data = key
ret, err = io.put(policy.token(), key, data, extra)
assert err is None
assert ret[u'key'].endswith(u'你好')
def test_put_no_key():
data = r(100)
extra.check_crc = 0
ret, err = io.put(policy.token(), key=None, data=data, extra=extra)
assert err is None
assert ret['hash'] == ret['key']
def test_put_StringIO():
key = "test_%s" % r(9)
data = cStringIO.StringIO('hello buddy!')
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'] == key
def test_put_fail_reqid(self):
key = "test_%s" % r(9)
data = "hello bubby!"
ret, err = io.put("", key, data, extra)
assert "reqid" in err
def test_put_urlopen():
key = "test_%s" % r(9)
data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg')
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'] == key
def test_put_urlopen():
key = "test_%s" % r(9)
data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg')
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'] == key
def test_put_StringIO():
key = "test_%s" % r(9)
data = cStringIO.StringIO('hello buddy!')
ret, err = io.put(policy.token(), key, data)
assert err is None
assert ret['key'] == key
def test_put_fail_reqid(self):
key = "test_%s" % r(9)
data = "hello bubby!"
ret, err = io.put("", key, data, extra)
assert "reqid" in err
reload(sys)
sys.setdefaultencoding('utf-8')
sys.path.insert(0, '../')
from qiniu import conf as qConf, rs as qRs, io as qIo
ACCESS_KEY = ''
SECRET_KEY = ''
BUCKET = ''
QINIU_DOMAIN = ''
qConf.ACCESS_KEY = ACCESS_KEY
qConf.SECRET_KEY = SECRET_KEY
if __name__ == '__main__':
putPolicy = qRs.PutPolicy(scope=BUCKET)
uptoken = putPolicy.token()
filePath = '../files/test.jpg'
key = 'upload_put.jpg'
f = open(filePath, 'rb')
data = f.read()
#上传结果
ret, err = qIo.put(uptoken, key, data)
print ret
print err
if err is None:
print 'http://%s/%s' % (QINIU_DOMAIN, key)
def test_put_unicode1():
key = "test_%s" % r(9) + '你好'
data = key
ret, err = io.put(policy.token(), key, data, extra)
assert err is None
assert ret[u'key'].endswith(u'你好')
