def WriteZipFile(filename):
"""
保存所有kvdb数据，压缩成zip然后aes cbc加密
"""
FileBuffer = io.BytesIO()
datalist = FindKVDBKeys()
if datalist:
zfile = zipfile.ZipFile(FileBuffer,mode='w')
for data in datalist:
# bytedata = (data + "tttttttt").encode(encoding="utf-8")
bytedata = kv.get(str(data))
if bytedata:
# print(bytedata)
zfile.writestr(str(data),bytedata)
zfile.close()
key = config.keyDataBackUp
iv = Random.new().read(AES.block_size)
cipher = AES.new(key, AES.MODE_ECB)
CryptIV = cipher.encrypt(iv)
cipher = AES.new(key, AES.MODE_CBC, iv)
bytebuffer = FileBuffer.getvalue()
lendata = 16 - len(bytebuffer)%16
bytebuffer = bytebuffer + chr(lendata)*lendata
#print(bytebuffer)
CryptData = CryptIV + cipher.encrypt(bytebuffer)
bucket = Bucket('backup')
# print(FileBuffer.getvalue())
bucket.put_object(filename,CryptData)
FileBuffer.close()
def put_obj2storage(file_name="", data="", expires="365", type=None, encoding=None, domain_name=STORAGE_DOMAIN_NAME):
whichStorage = int(getAttr("DEFAULT_STORAGE"))
if whichStorage == 0:
bucket = Bucket(domain_name)
bucket.put_object(file_name, data, content_type=type, content_encoding=encoding)
return bucket.generate_url(file_name)
elif whichStorage == 1:
import qiniu.conf, qiniu.rs, qiniu.io
qiniu.conf.ACCESS_KEY = getAttr("QNKEY")
qiniu.conf.SECRET_KEY = getAttr("QNSECRET")
policy = qiniu.rs.PutPolicy(getAttr("QNSTORAGE"))
uptoken = policy.token()
key = file_name
if key[0] == "/":
key = key[1:]
ret, err = qiniu.io.put(uptoken, key, data)
if err is not None:
print "err = ", err
return ""
###下面返回的网址有可能不同，有的是 xxxx.u.qiniudn.com 请改为自己的
return "http://%s/%s" % (getAttr("QNDOMAIN"), key)
else:
# 可后续扩展其他云存储
pass
return ""
def delPic(request,id):
if isUser(request):
if request.method == "POST":
try:
pic_id = request.POST.get('del_pic_id','')
pic = Pictures.objects.get(id=pic_id)
except Exception, e:
raise e
if 'SERVER_SOFTWARE' in os.environ:
try:
from sae.storage import Bucket
bucket = Bucket('media')
pic_path = str(id) + '/' + pic.pic_src[pic.pic_src.rindex("/")+1:]
bucket.delete_object(pic_path)
pic.delete()
except Exception, e:
raise e
else:
try:
os.remove(pic.pic_src[1:])
# print pic.pic_src[1:pic.pic_src.rindex("/")+1]
if not os.listdir(pic.pic_src[1:pic.pic_src.rindex("/")+1]):
os.rmdir(pic.pic_src[1:pic.pic_src.rindex("/")+1])
pic.delete()
except Exception, e:
raise e
def store_image(self):
"""
download image from WeChat url and store in storage. update stored_url
"""
try:
response = urllib2.urlopen(self.input_url)
if self.is_local_storage:
# just use the current url
self.stored_url = self.input_url
else:
# save to remote storage
from sae.storage import Bucket
# print ("in else "+self.storage_domain_name)
bucket = Bucket('wechatimgsave')
filename = self.definefilename(self.input_url, '.jpeg')
bucket.put_object(obj=filename,
contents=response.read(),
content_type='image/jpeg',
content_encoding=None,
metadata=None)
self.stored_url = bucket.generate_url(filename)
except urllib2.HTTPError as e:
logging.error("Error in opening " + self.input_url +
"with error code: " + e.code)
def inPage(self, site, filename):
try:
conn = Connection(accesskey='ym51nzx10z', secretkey='h0kxmzj2ly13jjj1m0jjly41li1wimizzz2w2m32', retries=3)
bucket = Bucket(site, conn)
page = bucket.get_object_contents(filename)
except Exception, e:
print e
def ReadZipFile(filename):
"""
从storage中读取数据，还原到kvdb中
参数 filename 要还原数据的文件名
"""
bucket = Bucket('backup')
# print(filename)
CryptData = bucket.get_object_contents(filename)
# print(CryptData)
# -FileBuffer.close()
key = config.keyDataBackUp
cipher = AES.new(key, AES.MODE_ECB)
iv = cipher.decrypt(CryptData[:16])
# print(str(iv))
cipher = AES.new(key, AES.MODE_CBC, iv)
bytebuffer = cipher.decrypt(CryptData[16:])
lendata = ord(bytebuffer[-1])
FileBuffer = io.BytesIO(bytebuffer[:-lendata])
zfile = zipfile.ZipFile(FileBuffer,mode='r')
namelist = zfile.namelist()
for name in namelist:
bytedata = zfile.read(name)
kv.set(name,bytedata.decode("utf-8"))
return u"数据已还原"
def sae_save_file( f , storage_name , file_name = None ):
from sae.storage import Bucket
bucket = Bucket(storage_name)
print bucket
if file_name is None:
file_name = f._get_name()
return bucket.put_object(file_name, f)
def inPageAmazon(self, params):
try:
conn = Connection(accesskey='ym51nzx10z', secretkey='h0kxmzj2ly13jjj1m0jjly41li1wimizzz2w2m32', retries=3)
bucket = Bucket('amazon', conn)
page = bucket.get_object_contents(params['category'] + '/' + params['in_page'])
except Exception, e:
print e
def get(self, sub_path):
from sae.storage import Bucket
bucket = Bucket('oerp') # 从云平台拿到一个Bucket容器
#imagebinary = meta['body']
response = bucket.get_object_contents(sub_path, chunk_size=10) # 取文件 bucket.get_object_contents(u'oerp', r'/uploadimg/' + sub_path)
self.set_header('Content-Type', 'text/xml; charset=utf-8')
self.write(response.next())
def put_obj2storage(file_name='',
data='',
expires='365',
type=None,
encoding=None,
domain_name=STORAGE_DOMAIN_NAME):
whichStorage = int(getAttr('DEFAULT_STORAGE'))
if whichStorage == 0:
bucket = Bucket(domain_name)
bucket.put_object(file_name,
data,
content_type=type,
content_encoding=encoding)
return bucket.generate_url(file_name)
elif whichStorage == 1:
import qiniu.conf, qiniu.rs, qiniu.io
qiniu.conf.ACCESS_KEY = getAttr('QNKEY')
qiniu.conf.SECRET_KEY = getAttr('QNSECRET')
policy = qiniu.rs.PutPolicy(getAttr('QNSTORAGE'))
uptoken = policy.token()
key = file_name
if key[0] == "/":
key = key[1:]
ret, err = qiniu.io.put(uptoken, key, data)
if err is not None:
print 'err = ', err
return ''
###下面返回的网址有可能不同，有的是 xxxx.u.qiniudn.com 请改为自己的
return "http://%s/%s" % (getAttr('QNDOMAIN'), key)
else:
#可后续扩展其他云存储
pass
return ''
def GetAllFilesOfStorage():
global domain_name
try:
bucket = Bucket(domain_name)
fileList = bucket.list()
L = []
count = 0
for f in fileList:
#fileTime = f.last_modified.encode()
#fileTime = fileTime.replace("T", " ").split(".")[0]
#timeArray = time.strptime(fileTime, "%Y-%m-%d %H:%M:%S")
#timeStamp = int(time.mktime(timeArray))
#startTime = "2015-8-2 00:00:00"
#if timeStamp > time.mktime(time.strptime(startTime,'%Y-%m-%d %H:%M:%S')):
L.append({
"name": f.name,
"size": f.bytes,
"url": bucket.generate_url(f.name),
"last_modified": f.last_modified
})
count = count + 1
Log("GetAllFilesOfStorage Count %s" % str(count))
return L
except Exception, e:
Log("Error when GetAllFilesOfStorage: %s" % e)
return None
def exportImport(filename,tp):
result={}
# for client debug
if settings.DEBUG:
data = xlrd.open_workbook(settings.MEDIA_ROOT+filename)
# for sae
else:
bucket = Bucket('resources')
obj = bucket.get_object_contents(filename)
data=xlrd.open_workbook(file_contents=obj)
table = data.sheets()[0]
# check the column
ncols=table.ncols
nrows=table.nrows
# for student
if (tp==0 and (not ncols==11)) or (tp==1 and (not ncols==9)):
result['status']='failured'
result['tip']='excel列数不对'
elif nrows<2:
result['status']='failured'
result['tip']='至少需要一条记录'
else:
statistic=executeImport(table,tp)
result['status']='success'
result['tip']='导入成功，共 %d 人，成功导入 %d 人，跳过 %d 人' \
% (statistic['sum'],statistic['count'],statistic['existed'])
result['usernames']=statistic['usernames']
# delete the uploaded temp file
# for client debug
if settings.DEBUG:
os.remove(settings.MEDIA_ROOT+filename)
# for sae
else:
bucket.delete_object(filename)
return result
def UploadFile(file_name, file_content):
global domain_name, maxSize
#Log("UploadFile")
#import sae.const
#access_key = sae.const.ACCESS_KEY
#secret_key = sae.const.SECRET_KEY
#appname = sae.const.APP_NAME
try:
from sae.storage import Bucket
bucket = Bucket(domain_name)
file_content.seek(0, 2)
length = file_content.tell()
if (length > maxSize):
Log("UploadFile oversize")
return "oversize"
else:
file_name = file_name.replace(' ', '')
file_content.seek(0)
bucket.put_object(file_name, file_content.read())
#url = bucket.generate_url(file_name)
url = "http://applinzi.ddianke.com/%s" % file_name
Log("UploadFile url %s" % url)
return url
except Exception, e:
Log("Error when UploadFile: %s" % e)
return None
def add_object(bucket_name, folder_path, object_name, object_data):
"""add object
Args:
bucket_name: name of bucket
folder_path: folder path
object_name: name of object
object_data: object data
Returns:
True if add successfully
False if object existed
"""
bucket = Bucket(bucket_name)
folder_path = complete_folder(folder_path)
object_path = "%s%s" % (folder_path, object_name)
virtual_object_path = "/s/%s/%s" % (bucket_name, object_path)
print 'class StorageInterface add_object() virtual_object_path: %s' % virtual_object_path
if is_file_existed(virtual_object_path):
print 'class StorageInterface add_object() file existed! object_path: %s' % object_path
return False
print 'class StorageInterface add_object() start put_object() object_path: %s' % object_path
# print 'class StorageInterface add_object() start put_object() object_data[0]: %s' % str(object_data)[0]
bucket.put_object(object_path, object_data)
print 'end'
return True
def POST(self):
context = web.input().context.encode('utf-8')
taskid = web.input().taskid
file_name = web.input().file_name
# Only segement the title and keyword
seg_content = []
lines = context.split('\r')
for line in lines:
tmp = line.split('\t')
seg_content.append(tmp[1])
seg_content.append(tmp[2])
results = get_seg_result('\t'.join(seg_content), keep_return=True)
newcontent = assemble_result(context, results['filtered_result'], taskid=taskid, file_name=file_name)
# log_message('Task ' + taskid + ' is excecuted (post), result: ' + '<br /> '.join(newcontent))
# submit to server
try:
# payload = urllib.urlencode([('seg_result', ''.join(newcontent).encode('utf-8')), ('taskid', taskid), ])
# req = urllib2.urlopen(SUBMIT_RESULT_URL, payload)
# log_message('submit task ' + str(taskid) + ' to server' + req.read())
from sae.storage import Bucket
bucket = Bucket('mozillaup')
bucket.put_object(file_name + '/' + str(taskid), ''.join(newcontent).encode('utf-8'))
# log_message('submit task ' + str(taskid) + ' to bucket')
except Exception, ex:
print ex
result = 'Encounter an error when submit task' + type(ex).__name__ + ' ' + str(ex.args)
log_message(result)
def get(self, request, *args, **kwargs):
from sae.deferredjob import MySQLExport, DeferredJob
from sae.storage import Bucket as SBucket
import time
import datetime
export_bucket = 'xkongbackup'
bucket = SBucket(export_bucket)
now = time.strftime('%Y_%m_%d_%H_%M_%S')
filename = 'app_ninan_%s.zip' % now
deferred_job = DeferredJob()
job = MySQLExport(export_bucket, filename, 'note_note',
'backends/backupsuccess/')
deferred_job.add(job)
resp = {'touch': filename}
# Delete all files in this bucket created a month ago
a_month_ago = datetime.datetime.now() - datetime.timedelta(days=30)
for object_ in bucket.list():
last_modified = object_['last_modified']
if last_modified:
mtime = datetime.datetime.strptime(last_modified,
'%Y-%m-%dT%H:%M:%S.%f')
else:
continue
if object_['content_type'] is not None and mtime < a_month_ago:
bucket.delete_object(object_['name'])
return self.render_to_response(resp)
def get_max_file_index(bucket_name, folder_path):
"""get the max file index in specific folder
Args:
bucket_name: name of bucket
folder_path: folder path
Returns:
-1 if no files
an positive int (including zero) indicating the max file index
"""
bucket = Bucket(bucket_name)
folder_path = complete_folder(folder_path)
# read files
files = [i for i in bucket.list(path=folder_path)]
# if the folder not exists or folder has no file
if len(files)==0:
return -1
# if has files, return the max file index
max_index = 0
for i in files:
num = int(i['name'].split(folder_path)[1].split('.')[0])
if num>max_index:
max_index = num
return max_index
def post(self, id):
id = int(id)
from sae.storage import Bucket
bucket = Bucket('audio')
fid = str(self.current_user) + datetime.now().strftime('/%Y-%m-%dT%H-%M-%S')
name = fid + '.wav'
#length = int(request.get_header('content-length',0))
bucket.put_object(name, self.request.body)
name = bucket.generate_url(name)
sss = Session()
model = RoomChat()
model.room_id = id
model.content_type = 'audio'
model.content = name
model.user_name = self.user.name
model.user_id = self.current_user
model.create_at = datetime.now()
sss.add(model)
sss.commit()
ls = GetTop.ajax(id)
self.render_list(*ls)
def upload_file_handle(f,user_id):
from sae.storage import Bucket
bucket=Bucket('image')
bucket.put()
file_postfix=f.name.split('.')[-1]
file_name=str(user_id)+'.'+file_postfix
bucket.put_object(file_name, f.read())
return 'http://sumioo-image.stor.sinaapp.com/'+file_name
def get(self, request, *args, **kwargs):
from utils.kvdb.bucket import Bucket as SBucket
filekey = kwargs.get('filekey')
bucket = SBucket()
contents = bucket.get_object_contents(filekey)
r = HttpResponse(contents)
r['Content-Disposition'] = 'attachment; filename={}'.format(filekey)
return r
def save_with_bucket(self, _bucket, fname, fcontent):
'''
_bucket is customer bucket name
'''
bk = Bucket(_bucket)
import StringIO
b = StringIO.StringIO(fcontent)
bk.put_object(fname, b.read())
return bk.generate_url(fname)
def savePageAmazon(self, params):
url = self.amazonUrlBase + self.amazonUrlMap.get(params['category']) + self.amazonUrlTail + params['id']
page = self.getPage(url)
try:
conn = Connection(accesskey='ym51nzx10z', secretkey='h0kxmzj2ly13jjj1m0jjly41li1wimizzz2w2m32', retries=3)
spider = Bucket('amazon', conn)
spider.put_object(params['category'] + '/' + params['id'], page)
except Exception, e:
print e
def save_img(uid,device_name,file1):
filename = file1.name
timestamp=filename.split(".")[0]
url="www.baidu.com"
if 'SERVER_SOFTWARE' in os.environ:
bucket = Bucket('domain2')
bucket.put_object(filename, file1)
url = bucket.generate_url(filename)
image=Img.create(uid=uid,timestamp=timestamp,device_name=device_name,url=url)
image.save()
def delete_img(con, pid, title):
if 'SERVER_SOFTWARE' in os.environ:
p = get_product_detail(con, title)
bucket = Bucket('domain1')
for url in p.img_list:
img_name = url.split("/")[-1]
bucket.delete_object(img_name)
print "delete bucket object", img_name
sql = "delete from {0} where pid ={1}".format(IMG_TABLE, pid)
execute_non_query(con, sql)
def DeleteFileFromStorage(fileName):
global domain_name
try:
from sae.storage import Bucket
bucket = Bucket(domain_name)
bucket.delete_object(fileName)
Log("DeleteFileFromStorage fileName %s" % fileName)
except Exception, e:
Log("Error when DeleteFileFromStorage: %s" % e)
return None
def edit0(request):
pname = pcomment = pmood = newname = 0
from sae.storage import Bucket
bucket = Bucket('abc')
if request.POST:
if request.POST.has_key('correct'):
if request.GET.has_key('atitle'):
pname = request.GET['atitle']
pn = t.objects.all()
if (len(pn)!= 0):
pname = pn[0].title
for i in pn:
i.delete()
we = imagess.objects.filter(title = pname)
if (len(we)!= 0):
img = bucket.get_object_contents('stati/'+pname)
im = Image.open(StringIO.StringIO(img))
imgout = StringIO.StringIO()
im.save(imgout,"jpeg")
img_data = imgout.getvalue()
we[0].title = request.POST['cname']+'.jpg'
newname = we[0].title
if (newname != pname):
ne = t(title = newname)
ne.save()
bucket.put_object('stati/'+newname, img_data)
im = Image.open(StringIO.StringIO(img))
out = im.resize((128, 128))
imgout = StringIO.StringIO()
out.save(imgout,"jpeg")
img_data = imgout.getvalue()
bucket.put_object('manage/'+newname, img_data)
bucket.delete_object('manage/'+pname)
bucket.delete_object('stati/'+pname)
pname = newname
we[0].comment = request.POST['ccomment']
we[0].mood = request.POST['cmood']
we[0].save()
pname = request.POST['cname']+'.jpg'
pcomment = request.POST['ccomment']
pmood = request.POST['cmood']
elif request.GET.has_key('atitle'):
if (pname == 0):
pname = request.GET['atitle']
p = t(title = pname)
p.save()
we = imagess.objects.filter(title = pname)
if (len(we)!= 0):
pcomment = we[0].comment
pmood = we[0].mood
if (pname!=0):
pname = pname[:-4]
return render_to_response('editt.html',{'pname':pname,'newname':newname, \
'pmood': pmood, 'pcomment':pcomment},context_instance=RequestContext(request))
def checktest(tt):
def gpng(txt):
url = 'https://api.handwriting.io/render/png'
params = {
'handwriting_id': '2D5S18M00002',
'handwriting_size': 'auto',
'width': '500px',
'height': '320px',
'line_spacing': '1.5',
'line_spacing_variance': 0.1,
'text': 'Though bladed corn be lodged and trees blown down'
}
params['text'] = txt
r = requests.get(url, auth=TOKEN_PAIR, params=params)
return r
correct_time = time.strftime('%m_%d_%H_%M_%S', time.localtime(time.time()))
bucket = Bucket('tempic') #修改成你自己的storage名
bucket.put()
bucket.post(acl='.r:*', metadata={'expires': '1d'})
f = gpng(tt)
data = f.content
bucket.put_object(correct_time + ".png", data)
file_url = bucket.generate_url(correct_time + ".png")
return file_url
def imgupload(req):
username = req.session.get("username", '')
if username == '':
return HttpResponseRedirect('/admin')
else:
if req.POST:
post = req.POST
if req.FILES:
img = req.FILES['img']
# s = Client()
bucket = Bucket('img')
# st = bucket.conn.__dict__
# if bucket.stat()['bytes'] == '0':
bucket.put()
bucket.post(acl='.r:.sinaapp.com,.r:sae.sina.com.cn')
tut = img._name.split('.')[-1]
dt_str = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S')
filename = dt_str + str(randint(100, 999)) + '.' + tut
bucket.put_object(filename, img)
# s.put('hitmstcweb',filename,img)
image = Img(
name = post.get('name','dt_str'), \
descripe = post.get('descripe',''), \
img = bucket.generate_url(filename), \
author = User.objects.get(username = username), \
)
image.save()
content = {'last_event': get_last_event()}
return render_to_response('imgupload.html',
content,
context_instance=RequestContext(req))
def initialize(*args):
global trie, FREQ, total, min_freq, initialized
if len(args)==0:
dictionary = DICTIONARY
else:
dictionary = args[0]
with DICT_LOCK:
if initialized:
return
if trie:
del trie
trie = None
t1 = time.time()
cache_file = 'jieba.cache'
default_dict = dictionary
default_bucket = getattr(settings, 'STORAGE_BUCKET_NAME')
bucket = Bucket(default_bucket)
cache_file_content = bucket.get_object_contents(dictionary)
dict_stamp = bucket.stat_object(default_dict)['timestamp']
load_from_cache_fail = True
try:
cache_stamp = bucket.stat_object(cache_file)['timestamp']
except:
cache_exists = False
else:
if cache_stamp > dict_stamp:
logger.debug("loading model from cache %s" % cache_file)
try:
cache_content = bucket.get_object_contents(cache_file)
trie,FREQ,total,min_freq = marshal.loads(cache_content)
load_from_cache_fail = False
except:
load_from_cache_fail = True
if load_from_cache_fail:
trie,FREQ,total = gen_trie(cache_file_content)
FREQ = dict([(k,log(float(v)/total)) for k,v in FREQ.iteritems()]) #normalize
min_freq = min(FREQ.itervalues())
logger.debug("dumping model to file cache %s" % cache_file)
try:
tmp_suffix = "."+str(random.random())
cache_file = 'dict' + tmp_suffix + '.cache'
cache_file = os.path.join(tempfile.gettempdir(), cache_file)
with open(cache_file,'wb') as temp_cache_file:
marshal.dump((trie,FREQ,total,min_freq),temp_cache_file)
if cache_exists:
bucket.delete_object('jieba.cache')
bucket.put_object('jieba.cache', open(cache_file, 'rb'))
except:
logger.error("dump cache file failed.")
logger.exception("")
initialized = True
logger.debug("loading model cost %s seconds." % (time.time() - t1))
logger.debug("Trie has been built succesfully.")
def STT(self, filePath): # 输入语音文件（amr格式），输出文字
try:
bucket = Bucket('wechat')
result = self.__client.asr(bucket.get_object_contents(filePath), self.__format, self.__rate, {'cuid': self.__cuid,
'dev_pid': self.__dev_pid})
if 'error_code' in result: # 识别失败
raise Exception("error_code: %s, error_msg: %s" % (result['error_code'], result['error_msg']))
return result['result'][0] # 提供1-5个候选结果，utf-8 编码
except Exception, ex:
print 'BaiduSTT.STT error:%s' % (ex)
return ''
def put_obj2storage(file_name='',
data='',
expires='365',
type=None,
encoding=None,
domain_name=STORAGE_DOMAIN_NAME):
bucket = Bucket(domain_name)
bucket.put_object(file_name,
data,
content_type=type,
content_encoding=encoding)
return bucket.generate_url(file_name)
def upload_storage(bucketname=BUCKET, savename="test.txt", filedata=None):
if bucketname and savename and filedata:
if DEBUG:
return None
from sae.storage import Bucket
bucket = Bucket(bucketname)
if savename[0] == "/":
savename = savename[1:]
bucket.put_object(savename, filedata)
return bucket.generate_url(savename)
else:
return False
def upload_storage(bucketname=BUCKET, savename="test.txt", filedata=None):
if bucketname and savename and filedata:
if DEBUG:
return None
from sae.storage import Bucket
bucket = Bucket(bucketname)
if savename[0]=="/":
savename = savename[1:]
bucket.put_object(savename, filedata)
return bucket.generate_url(savename)
else:
return False
def savePageExceptAmazon(self, site, id):
url = getattr(self, site + 'Url')
if site == 'douban':
url += str( (int(id) - 1) * 10 )
else:
url += str(id)
page = self.getPage(url)
try:
conn = Connection(accesskey='ym51nzx10z', secretkey='h0kxmzj2ly13jjj1m0jjly41li1wimizzz2w2m32', retries=3)
spider = Bucket(site, conn)
spider.put_object(str(id), page)
except Exception, e:
print e
def delete_file(file_path):
from urlparse import urlsplit
from sae.storage import Bucket
res = urlsplit(file_path)
hostname, path = res.hostname, res.path
bucket_name = hostname.split(".", 1)[0].split("-")[-1]
_, object_name = path.split("/", 1)
bucket = Bucket(bucket_name)
bucket.delete_object(object_name)
def get(self, request, *args, **kwargs):
bucket_name = getattr(settings, "SAE_STORAGE_BUCKET_NAME",
'xkong1946')
sae_bucket = SaeBucket(bucket_name)
kv_bucket = Bucket()
files = ['dict.txt', 'jieba.cache']
ret = []
for file_ in files:
contents = sae_bucket.get_object_contents(file_)
kv_bucket.save(file_, contents)
ret.append(file_)
# or return json
return HttpResponse("done")
def delete_file(file_path):
from urlparse import urlsplit
from sae.storage import Bucket
res = urlsplit(file_path)
hostname, path = res.hostname, res.path
bucket_name = hostname.split(".", 1)[0].split("-")[-1]
_, object_name = path.split("/", 1)
bucket = Bucket(bucket_name)
bucket.delete_object(object_name)
def imgupload(req):
username = req.session.get("username",'')
if username == '':
return HttpResponseRedirect('/admin')
else:
if req.POST:
post = req.POST
if req.FILES:
img = req.FILES['img']
# s = Client()
bucket = Bucket('img')
# st = bucket.conn.__dict__
# if bucket.stat()['bytes'] == '0':
bucket.put()
bucket.post(acl='.r:.sinaapp.com,.r:sae.sina.com.cn')
tut = img._name.split('.')[-1]
dt_str = datetime.strftime(datetime.now(),'%Y%m%d%H%M%S')
filename = dt_str + str(randint(100,999)) + '.' + tut
bucket.put_object(filename,img)
# s.put('hitmstcweb',filename,img)
image = Img(
name = post.get('name','dt_str'), \
descripe = post.get('descripe',''), \
img = bucket.generate_url(filename), \
author = User.objects.get(username = username), \
)
image.save()
content = {'last_event':get_last_event()}
return render_to_response('imgupload.html',content,context_instance = RequestContext(req));
def MakeBackup():
"""
定时备份文件任务
"""
dbchgcounter = kv.get("kvdbchg")
if dbchgcounter == None:
dbchgcounter = 0
if dbchgcounter == 0:
return u"数据未改变"
dbchgcounter = 0
kv.set("kvdbchg",dbchgcounter)
bucket = Bucket('backup')
tm = datetime.now()
# 删除过期文件
dellist = []
fdlist = []
fwlist = []
for finf in bucket.list():
last_modified = str(finf[u'last_modified'])
last_modified = last_modified[:last_modified.index(".")]#2013-05-22T05:09:32.259140 -> 2013-05-22T05:09:32
filetime = datetime.strptime(last_modified,"%Y-%m-%dT%H:%M:%S")
fname = str(finf[u"name"])
if "d.zip.data" in fname:
fdlist.append((fname,tm-filetime))
else:
fwlist.append((fname,tm-filetime))
if len(fdlist) > 3:
sorted(fdlist,key = lambda x:x[1])
dellist = fdlist[3:]
if len(fwlist) > 4:
sorted(fwlist,key = lambda x:x[1])
dellist += fdlist[4:]
for fname in dellist:
bucket.delete_object(fname[0])
#备份新文件
filename = tm.strftime(u"%Y-%m-%d_%H_%M_%S")
if tm.weekday() == 5: #周六
filename += "w.zip.data"
else:
filename += "d.zip.data"
WriteZipFile(filename)
return u"已备份"
def __init__(self, **kwargs):
bucket = kwargs['bucket'] if 'bucket' in kwargs else ''
accesskey = kwargs['accesskey'] if 'accesskey' in kwargs else ''
secretkey = kwargs['secretkey'] if 'secretkey' in kwargs else ''
account = kwargs['account'] if 'account' in kwargs else '' # app name
retries = long(
kwargs['retries']) if 'retries' in kwargs else 3 # app name
self.prefix = kwargs['prefix'] if 'prefix' in kwargs else ''
if accesskey and secretkey and account:
conn = Connection(accesskey, secretkey, account, retries)
self.kv = conn.get_bucket(bucket)
else:
self.kv = Bucket(bucket)
def generate_city_list():
values = []
#read citylist
BUCKET = 'citylist'
# read citylist
bucket = Bucket(BUCKET)
citylist_content = bucket.get_object_contents('cities.json')
citylist = json.loads(citylist_content)
values.append('<ul>')
for city in citylist:
line = '''<li><a href="%s/%s">%s</li>''' % ('http://aqidatapuller.applinzi.com', city['city'].lower(), city['city'])
values.append(line)
values.append('</ul>')
return ''.join(values)
def cron_task():
BUCKET = 'citylist'
# read citylist
bucket = Bucket(BUCKET)
citylist_content = bucket.get_object_contents('cities.json')
cities = json.loads(citylist_content)
# datepuller
data_puller = DataPuller()
# mysql
mysql = Mysql()
for city in cities:
value = data_puller.pull_data(int(city['cid']))
if value:
mysql.insert_data(city['city'], int(value))
def thumbnail(request, filename):
bucket = Bucket('upload')
bucket.put()
bucket.put_object("image/" + filename, request.FILES['file'])
obj = bucket.get_object_contents("image/" + filename)
image = Image.open(StringIO(obj))
image.thumbnail((160, 120), Image.ANTIALIAS)
imgOut = StringIO()
image.save(imgOut, 'jpeg')
img_data = imgOut.getvalue()
bucket.put_object('thumbnail/' + filename, img_data)
imgOut.close()
def get(self, request, *args, **kwargs):
# Uncomment the following two lines to allow initial.
done = {'done': '0', 'ret': '0'}
# return self.render_to_response(done)
bucket_name = getattr(settings, "SAE_STORAGE_BUCKET_NAME",
'xkong1946')
sae_bucket = SaeBucket(bucket_name)
kv_bucket = Bucket()
files = ['dict.txt', 'jieba.cache']
ret = []
for file_ in files:
contents = sae_bucket.get_object_contents(file_)
kv_bucket.save(file_, contents)
ret.append(file_)
done = {'done': '0', 'ret': ret}
return self.render_to_response(done)
def retrieve(url, bucket, filename, retry=5):
"""保存url的图片到storage"""
from sae.storage import Bucket
for x in xrange(1, 6):
try:
img = urllib2.urlopen(url)
b = Bucket(bucket)
mime = RE_MIME.search(str(img.info()))
if mime:
mime = mime.groups()[0]
b.put_object(filename, img.read(), mime)
print 'Save "%s" to "%s/%s", MimeType:"%s"' % (url, bucket,
filename, mime)
return "http://akbquiz-%s.stor.sinaapp.com/" % (
bucket, quote(filename.encode('utf-8')))
except Exception, e:
print 'Save image file "%s" failed, retrying: %d. (%s)' % (url, x,
e)
pass
def add_objects(bucket_name, folder_path, objects):
"""add objects
Args:
bucket: name of bucket
folder_path: folder path
objects: dictionary of filename and its data, for example:
objects['sample.txt']="hello world"
"""
bucket = Bucket(bucket_name)
folder_path = complete_folder(folder_path)
for key in objects.keys():
object_path = "%s%s" % (folder_path, key)
virtual_object_path = "/s/%s/%s" % (bucket_name, object_path)
if is_file_existed(virtual_object_path):
print 'class StorageInterface add_objects() file existed! object_path: %s' % object_path
continue
bucket.put_object(object_path, objects[key])
def ImageClassify(self, filePath):
try:
""" 读取图片 """
bucket = Bucket('wechat')
image = bucket.get_object_contents(filePath)
""" 调用通用物体识别 """
self.__client.advancedGeneral(image)
""" 如果有可选参数 """
options = {}
options["baike_num"] = 0
""" 带参数调用通用物体识别 """
info = self.__client.advancedGeneral(image, options)
#print info
if 'error_code' in info: # 识别失败
raise Exception("error_code: %s, error_msg: %s" %
(info['error_code'], info['error_msg']))
print "image:%s" % (filePath)
for item in info['result']:
print 'root:%s,keyword:%s' % (item['root'], item['keyword'])
except Exception, ex:
print 'BaiduImageClassify.ImageClassify error:%s' % (ex)
def upload_file_handle(f, user_id):
from sae.storage import Bucket
bucket = Bucket('image')
bucket.put()
file_postfix = f.name.split('.')[-1]
file_name = str(user_id) + '.' + file_postfix
bucket.put_object(file_name, f.read())
return 'http://sumioo-image.stor.sinaapp.com/' + file_name
def UploadFileFromUrl(file_name, url, values=None):
#url=url+'//'
global domain_name
Log("Upload File From Url")
try:
if values is not None:
data = json.dumps(values, ensure_ascii=False).encode('utf-8')
req = urllib2.Request(url, data)
sock = urllib2.urlopen(req)
else:
sock = urllib.urlopen(url)
f = sock.read()
stream = cStringIO.StringIO(f)
sock.close()
bucket = Bucket(domain_name)
file_name = file_name.replace(' ', '')
bucket.put_object(file_name, stream)
return_url = bucket.generate_url(file_name)
Log("Upload File From Url url %s" % return_url)
return return_url
except Exception, e:
Log("Error when Upload File From Url: %s" % e)
def delete_blog(request):
if request.is_ajax() and request.method == 'POST':
blog_id = request.POST.get('blog_id')
if blog_id is None:
result = {'status': '0', 'info': '出现错误：待删除的博客ID不能为空！'}
else:
try:
b = Blog.objects.get(id=blog_id)
# 删除该博客对应的图片
import sae.storage
from sae.storage import Bucket
bucket = Bucket('img')
img_file = b.image.url
file_name = img_file[-14:]
if file_name != 'default000.jpg':
bucket.delete_object(file_name)
b.delete()
result = {'status': '1', 'info': '已成功删除该博客！'}
except ObjectDoesNotExist:
result = {'status': '1', 'info': '待删除的博客不存在！'}
return HttpResponse(json.dumps(result),
content_type='application/json')
return render(request, 'blogs/index.html')
def delete_object(bucket_name, folder_path, object_name):
"""delete object
Args:
bucket_name: name of bucket
folder_path: folder path
object_name: name of object
Returns:
True if delete successfully
False if object unexisted
"""
bucket = Bucket(bucket_name)
folder_path = complete_folder(folder_path)
object_path = "%s%s" % (folder_path, object_name)
virtual_object_path = "/s/%s/%s" % (bucket_name, object_path)
if not is_file_existed(virtual_object_path):
print 'class StorageInterface delete_object() file unexisted! object_path: %s' % object_path
return False
bucket.delete_object(object_path)
print 'class StorageInterface delete_object delete successfully!'
return True
def get_urls(bucket_name, folder_path):
"""get resource urls from storage
Args:
bucket_name: the name of bucket
folder_path: resources folder path
Returns:
list of resource urls
"""
bucket = Bucket(bucket_name)
folder_path+='/' # folder should be end with '/'
# if no file
files = [i for i in bucket.list(path=folder_path)]
if len(files) == 0:
return []
# else has files
filenames = []
for item in [i['name'] for i in bucket.list(path=folder_path) if i['name'][-3:]=="jpg"]:
filenames.append(item)
return [bucket.generate_url(i) for i in filenames]
def listDirOfSAEStorage(path=''):
monkey.patch_all()
bucket = Bucket('media')
bucket.put()
bucket.post(
acl=
'.r:.sinaapp.com,.r:sae.sina.com.cn,.r:.vipsinaapp.com,.r:.qq.com,.r:.wx.qq.com',
metadata={'expires': '7d'})
if not path.endswith('/'):
if path == '':
pass
else:
path = path + '/'
return [i for i in bucket.list(path)]
def setToSAEStorage(filename, data):
monkey.patch_all()
bucket = Bucket('media')
bucket.put()
bucket.post(
acl=
'.r:.sinaapp.com,.r:sae.sina.com.cn,.r:.vipsinaapp.com,.r:.qq.com,.r:.wx.qq.com',
metadata={'expires': '7d'})
tmp = __method_get_namelist(filename)
if tmp == None:
return None
filetype = tmp[1]
if filetype.lower() in ('jpg', 'jpeg', 'bmp', 'gif', 'png'):
filepath = 'image/' + filename
elif filetype.lower() in ("swf", "wmv"):
filepath = 'video/' + filename
elif filetype.lower() in ("wma", "mp3"):
filepath = 'music/' + filename
else:
filepath = filename
bucket.put_object(filepath, data)
return bucket.generate_url(filepath)
def img_upload(request):
try:
reqfile = request.FILES['upload_file']
if reqfile is None:
result = {'status': '1', 'info': '图片为空！'}
return HttpResponse(json.dumps(result),
content_type='application/json')
else:
from os import environ
online = environ.get("APP_NAME", "")
if online:
import sae.const
access_key = sae.const.ACCESS_KEY
secret_key = sae.const.SECRET_KEY
appname = sae.const.APP_NAME
domain_name = "img" # 刚申请的domain
str_time = str(time.mktime(
datetime.datetime.now().timetuple()))
str_time = str_time[:-2]
reqfile.name = str_time + ".png"
import sae.storage
from sae.storage import Bucket
bucket = Bucket('img')
bucket.put()
# http://www.sinacloud.com/doc/sae/python/storage.html
bucket.put_object(
reqfile.name,
reqfile,
content_type='application/x-www-form-urlencoded ')
url = bucket.generate_url(reqfile.name)
result = {'status': '1', 'info': '图片上传成功！', 'url': url}
return HttpResponse(json.dumps(result),
content_type='application/json')
else:
result = {'status': '0', 'info': '图片上传出现错误！'}
return HttpResponse(json.dumps(result),
content_type='application/json')
except Exception, e:
result = {'status': '0', 'info': '图片上传出现错误：%s' % e}
return HttpResponse(json.dumps(result),
content_type='application/json')
