def process_item(self, item, spider):
# 获得上传权限
qiniu.conf.ACCESS_KEY = "QBsaz_MsErywKS2kkQpwJlIIvBYmryNuPzoGvHJF"
qiniu.conf.SECRET_KEY = "OTi4GrXf8CQQ0ZLit6Wgy3P8MxFIueqMOwBJhBti"
# 配置上传策略。
# 其中lvxingpai是上传空间的名称（或者成为bucket名称）
policy = qiniu.rs.PutPolicy('lvxingpai-img-store')
# 取得上传token
uptoken = policy.token()
# 上传的额外选项
extra = io.PutExtra()
# 文件自动校验crc
extra.check_crc = 1
upload_stream = False
# 将相应的数据存入mongo中
client = pymongo.MongoClient('zephyre.me', 27017)
db = client.imagestore
# 检查是否已经入mongo库
if db.Hotel.find_one({'url_hash': str(item['hash_value'])}) is None:
# 先生成本地文件
localfile = str(time.time()) + str(random.random())
with open(localfile, 'wb') as f:
f.write(item['pic'])
# 上传
if upload_stream:
# 上传流
with open(localfile, 'rb') as f:
body = f.read()
ret, err = io.put(uptoken, str(item['key']), body, extra)
else:
# 上传本地文件
ret, err = io.put_file(uptoken, str(item['key']), localfile,
extra)
if err is not None:
sys.stderr.write('error: %s ' % err)
return
# 计算文件大小，进入mongo
file_size = int(getsize(localfile))
db.Hotel.save({
'url': item['url'],
'key': item['key'],
'url_hash': item['hash_value'],
'ret_hash': ret['hash'],
'size': file_size
})
# 增加索引
db.Hotel.create_index('url_hash')
# 删除上传成功的文件
os.remove(localfile)
return item
def save_file_qiniu(binary, filename, mime="application/octet-stream"):
policy = rs.PutPolicy(QiniuConf.BUCKET_NAME)
uptoken = policy.token()
extra = io.PutExtra()
extra.mime_type = mime
res, err = io.put(uptoken, filename, binary, extra)
if err is not None:
return 'none'
url = rs.make_base_url(QiniuConf.BUCKET_DOMAIN, filename)
return url
def post(self):
mimeType = 'application/json'
if self.request.headers.get('Content-Type', '') == mimeType:
data = self.request.body
dataObj = json.loads(data)
pid = dataObj.get('id', 'default')
conf.ACCESS_KEY = STORAGE_ACCESS_KEY
conf.SECRET_KEY = STORAGE_SECRET_KEY
storagePolicy = rs.PutPolicy(STORAGE_BUCKET)
storageToken = storagePolicy.token()
extra = io.PutExtra()
extra.mime_type = mimeType
io.put(storageToken, str(pid), data, extra)
return
def save_file_qiniu(binary, filename, mime="application/octet-stream"):
today = datetime.now().strftime("%Y/%m/%d/")
filename = today + filename
policy = rs.PutPolicy(QINIU_SETTINGS.BUCKET_NAME)
uptoken = policy.token()
extra = io.PutExtra()
extra.mime_type = mime
res, err = io.put(uptoken, filename, binary, extra)
if err is not None:
raise Exception("Qiniu save file [%s] error: %s res: %s" % (filename, err, res))
url = rs.make_base_url(QINIU_SETTINGS.BUCKET_DOMAIN, filename)
iv = fop.ImageView()
iv.mode = 2
iv.width = THUMB_SIZE[0]
url_thumb = iv.make_request(url)
return url, url_thumb
binascii = zlib
except ImportError:
zlib = None
import binascii
import cStringIO
from qiniu import conf
from qiniu import rs
from qiniu import io
conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY")
conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY")
bucket_name = os.getenv("QINIU_TEST_BUCKET")
policy = rs.PutPolicy(bucket_name)
extra = io.PutExtra()
extra.mime_type = "text/plain"
extra.params = {'x:a': 'a'}
def r(length):
lib = string.ascii_uppercase
return ''.join([random.choice(lib) for i in range(0, length)])
class TestUp(unittest.TestCase):
def test(self):
def test_put():
key = "test_%s" % r(9)
# params = "op=3"
data = "hello bubby!"