def process_item(self, item, spider):
# 获得上传权限
qiniu.conf.ACCESS_KEY = "QBsaz_MsErywKS2kkQpwJlIIvBYmryNuPzoGvHJF"
qiniu.conf.SECRET_KEY = "OTi4GrXf8CQQ0ZLit6Wgy3P8MxFIueqMOwBJhBti"
# 配置上传策略。
# 其中lvxingpai是上传空间的名称（或者成为bucket名称）
policy = qiniu.rs.PutPolicy('lvxingpai-img-store')
# 取得上传token
uptoken = policy.token()
# 上传的额外选项
extra = io.PutExtra()
# 文件自动校验crc
extra.check_crc = 1
upload_stream = False
# 将相应的数据存入mongo中
client = pymongo.MongoClient('zephyre.me', 27017)
db = client.imagestore
# 检查是否已经入mongo库
if db.Hotel.find_one({'url_hash': str(item['hash_value'])}) is None:
# 先生成本地文件
localfile = str(time.time()) + str(random.random())
with open(localfile, 'wb') as f:
f.write(item['pic'])
# 上传
if upload_stream:
# 上传流
with open(localfile, 'rb') as f:
body = f.read()
ret, err = io.put(uptoken, str(item['key']), body, extra)
else:
# 上传本地文件
ret, err = io.put_file(uptoken, str(item['key']), localfile,
extra)
if err is not None:
sys.stderr.write('error: %s ' % err)
return
# 计算文件大小，进入mongo
file_size = int(getsize(localfile))
db.Hotel.save({
'url': item['url'],
'key': item['key'],
'url_hash': item['hash_value'],
'ret_hash': ret['hash'],
'size': file_size
})
# 增加索引
db.Hotel.create_index('url_hash')
# 删除上传成功的文件
os.remove(localfile)
return item
def test_put_file(self):
localfile = "%s" % __file__
key = "test_%s" % r(9)
extra.check_crc = 1
ret, err = io.put_file(policy.token(), key, localfile, extra)
assert err is None
assert ret['key'] == key
def test_put_file(self):
localfile = "%s" % __file__
key = "test_%s" % r(9)
extra.check_crc = 1
ret, err = io.put_file(policy.token(), key, localfile, extra)
assert err is None
assert ret['key'] == key
def putFile(self, key, localFile, mimeType=None):
putPolicy = qRs.PutPolicy('%s:%s' % (self.bucket, key))
uptoken = putPolicy.token()
putExtra = qIo.PutExtra()
if mimeType is not None:
putExtra.mime_type = '%s' % (mimeType,)
statInfo = os.stat(localFile)
fsize = statInfo.st_size
ret, err = qIo.put_file(uptoken, key, localFile, putExtra)
return ret, err
def process_item(self, item, spider):
# 获得上传权限
qiniu.conf.ACCESS_KEY = "QBsaz_MsErywKS2kkQpwJlIIvBYmryNuPzoGvHJF"
qiniu.conf.SECRET_KEY = "OTi4GrXf8CQQ0ZLit6Wgy3P8MxFIueqMOwBJhBti"
# 配置上传策略。
# 其中lvxingpai是上传空间的名称（或者成为bucket名称）
policy = qiniu.rs.PutPolicy('lvxingpai-img-store')
# 取得上传token
uptoken = policy.token()
# 上传的额外选项
extra = io.PutExtra()
# 文件自动校验crc
extra.check_crc = 1
upload_stream = False
# 将相应的数据存入mongo中
client = pymongo.MongoClient('zephyre.me', 27017)
db = client.imagestore
# 检查是否已经入mongo库
if db.Hotel.find_one({'url_hash': str(item['hash_value'])}) is None:
# 先生成本地文件
localfile = str(time.time()) + str(random.random())
with open(localfile, 'wb') as f:
f.write(item['pic'])
# 上传
if upload_stream:
# 上传流
with open(localfile, 'rb') as f:
body = f.read()
ret, err = io.put(uptoken, str(item['key']), body, extra)
else:
# 上传本地文件
ret, err = io.put_file(uptoken, str(item['key']), localfile, extra)
if err is not None:
sys.stderr.write('error: %s ' % err)
return
# 计算文件大小，进入mongo
file_size = int(getsize(localfile))
db.Hotel.save({'url': item['url'], 'key': item['key'],
'url_hash': item['hash_value'], 'ret_hash': ret['hash'], 'size': file_size})
# 增加索引
db.Hotel.create_index('url_hash')
# 删除上传成功的文件
os.remove(localfile)
return item
def putData(self):
import socket
socket.setdefaulttimeout(3)
try:
ret,err=io.put_file(self.token,self.filename,Db)
if err:
rs.Client().delete(ZONENAME,self.filename)
self.putData()
else:
wx.CallAfter(self.showMsg,"数据库上传成功")
except:
wx.CallAfter(self.showMsg,"数据库上传失败")
def put_files(self, key=None, local_file=None, rename=None):
"""
put local file to qiniu. If key is None, the server will generate one.
Note: local_file if local file path.
"""
if local_file is None:
raise NameError('upload file to qiniu error!')
policy = rs.PutPolicy(bucket_name)
policy.saveKey = rename
up_token = policy.token()
ret, err = io.put_file(up_token, key, local_file)
if err is not None:
sys.stderr.write('error: %s ' % err)
return ret['key']
else:
# print 'upload local: %s file ok.' % local_file
pass
return ret
def Xh_writeData(data,image_name_suffix,filename, html5):
ALLOW_SUFFIX =['.jpg','.png','.jpeg','.gif']
if image_name_suffix in ALLOW_SUFFIX:
image_name = str(int(time.time())) + image_name_suffix
try:
with open(filename,'wb') as destination:
if html5:
destination.write(data)#写文件流
else:
for c in data.chunks():
destination.write(c)
key = filename.replace(BASE_DIR+"/","");
ret, err = io.put_file(policy.token(), key, filename)
if(err == None):
return str(QINIU_XHURL)+key;
else:
return ""
except:
return "";
else:
return "";
def save2hasEditedDB(request):
if request.method == 'POST':
responseAll = []
pid = Pid(pid=0)
pid.save()
_id = len(Pid.objects)
thisTime = str(int(time.time())) + ""
fReadTxt = request.POST.get("elm1","");
fReadTxtAll = "";
if(fReadTxt!=""):
contentHtmlOut = ""
contentHtmlStore= ""
contentHtmlDir = str(MEDIA_ROOT)+"content/"+str(_id)
os.makedirs(contentHtmlDir);
contentHtmlOut = str(MEDIA_ROOT)+"content/"+str(_id)+"/"+thisTime+".html";
contentHtmlStore= str(QINIU_URL)+str(_id)+"/contentHtml/"+thisTime+".html";
fWriteTxt = open(contentHtmlOut,"wb+");
fCode = "<style>@media only screen and (min-device-width : 320px) and (max-device-width : 568px) and (orientation : portrait) { img { width: 300px; } p { font-size: 1.5em; text-align: justify; padding: 4px;}} </style>" # for auto adjust format by liaojundong
fReadTxtAll = "<html xmlns=\"http://www.w3.org/1999/xhtml\"> <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf8\"><body> \n" + fCode + "\n" + fReadTxt + "</body></html>"
fWriteTxt.write(fReadTxtAll.encode("utf8") )
fWriteTxt.close();
filename = contentHtmlOut
key = filename.replace(BASE_DIR+"/","");
ret, err = io.put_file(policy.token(), key, filename)
responseData = {}
responseData["fReadTxt"] = fReadTxt
responseData["fCodeHtml"] = fReadTxtAll.encode("utf8");
responseData["post_info"] = request.POST
responseData["pId"] = _id;
title = request.POST['title']
aItem = LiwuItem_hasEditedDB(title=title)
aItem.pId = _id;
aItem.itemURL = request.POST.get('itemURL',"")
aItem.liwuReason = fReadTxtAll;
aItem.price = request.POST.get('price',"");
aItem.priceRank = request.POST.get('priceRank',"")
rateStr = request.POST.get('rate',"")
try:
aItem.rate = float(rateStr)
except:
aItem.rate = 4; #default, 0, 4 7
aItem.soldAmout = request.POST.get('soldAmout',"")
aItem.imageUrl = request.POST.get('imageUrl',"")
aItem.liwuPersonTag = request.POST.get('liwuPersonTag',"")
aItem.liwuSeasonTag = request.POST.get('liwuSeasonTag',"")
aItem.liwuTags = request.POST.get('liwuTags',"")
aItem.liwuGender= request.POST.get('liwuGender',"")
scoreAsLiwu = request.POST.get('scoreAsLiwu',"")
try:
aItem.scoreAsLiwu = float(scoreAsLiwu);
except:
aItem.scoreAsLiwu = 0.5;
aItem.save()
old_pId = request.POST.get('pId',"");
try:
old_pId = int(old_pId);
except:
old_pId = -1;
found = LiwuItem_toBeEdited.objects.filter(pId = old_pId);
if(len(found)>0):
bItem = found[0];
bItem.editOrNot = 1;
bItem.save();
return HttpResponse(json.dumps(responseData), content_type="application/json")
def upFile(self,filepath,key):
try:
ret,err=io.put_file(self.token,key,filepath)
except:
pass
#coding=utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
sys.path.insert(0, '../')
from qiniu import conf as qConf, rs as qRs, io as qIo
ACCESS_KEY = ''
SECRET_KEY = ''
BUCKET = ''
QINIU_DOMAIN = ''
qConf.ACCESS_KEY = ACCESS_KEY
qConf.SECRET_KEY = SECRET_KEY
if __name__ == '__main__':
putPolicy = qRs.PutPolicy(scope=BUCKET)
uptoken = putPolicy.token()
filePath = '../files/test.jpg'
key = 'upload_basic.jpg'
#上传结果
ret, err = qIo.put_file(uptoken, key, filePath)
print ret
print err
if err is None:
print 'http://%s/%s' % (QINIU_DOMAIN, key)
def upFile(self, filepath, key):
try:
ret, err = io.put_file(self.token, key, filepath)
except:
pass
