def initialize(*args):
global trie, FREQ, total, min_freq, initialized
if len(args)==0:
dictionary = DICTIONARY
else:
dictionary = args[0]
with DICT_LOCK:
if initialized:
return
if trie:
del trie
trie = None
t1 = time.time()
cache_file = 'jieba.cache'
default_dict = dictionary
default_bucket = getattr(settings, 'STORAGE_BUCKET_NAME')
bucket = Bucket(default_bucket)
cache_file_content = bucket.get_object_contents(dictionary)
dict_stamp = bucket.stat_object(default_dict)['timestamp']
load_from_cache_fail = True
try:
cache_stamp = bucket.stat_object(cache_file)['timestamp']
except:
cache_exists = False
else:
if cache_stamp > dict_stamp:
logger.debug("loading model from cache %s" % cache_file)
try:
cache_content = bucket.get_object_contents(cache_file)
trie,FREQ,total,min_freq = marshal.loads(cache_content)
load_from_cache_fail = False
except:
load_from_cache_fail = True
if load_from_cache_fail:
trie,FREQ,total = gen_trie(cache_file_content)
FREQ = dict([(k,log(float(v)/total)) for k,v in FREQ.iteritems()]) #normalize
min_freq = min(FREQ.itervalues())
logger.debug("dumping model to file cache %s" % cache_file)
try:
tmp_suffix = "."+str(random.random())
cache_file = 'dict' + tmp_suffix + '.cache'
cache_file = os.path.join(tempfile.gettempdir(), cache_file)
with open(cache_file,'wb') as temp_cache_file:
marshal.dump((trie,FREQ,total,min_freq),temp_cache_file)
if cache_exists:
bucket.delete_object('jieba.cache')
bucket.put_object('jieba.cache', open(cache_file, 'rb'))
except:
logger.error("dump cache file failed.")
logger.exception("")
initialized = True
logger.debug("loading model cost %s seconds." % (time.time() - t1))
logger.debug("Trie has been built succesfully.")
def initialize(dictionary=None):
global pfdict, FREQ, total, min_freq, initialized, DICTIONARY, DICT_LOCK
if not dictionary:
dictionary = DICTIONARY
with DICT_LOCK:
if initialized:
return
logger.debug("Building prefix dict from %s ..." % X_CACHE_FILE)
t1 = time.time()
cache_file = X_CACHE_FILE
if settings.DEBUG:
bucket = Bucket('xkong1946')
else:
bucket = Bucket()
dict_stamp = bucket.stat_object(dictionary)['timestamp']
cache_stamp = bucket.stat_object(cache_file)['timestamp']
load_from_cache_fail = True
if cache_stamp > dict_stamp:
logger.debug("Loading model from cache %s" % cache_file)
try:
cf = bucket.get_object_contents(cache_file)
pfdict, FREQ, total = marshal.loads(cf)
# prevent conflict with old version
load_from_cache_fail = not isinstance(pfdict, set)
except Exception, e:
print e
load_from_cache_fail = True
if load_from_cache_fail:
dict_file_content = bucket.get_object_contents(dictionary)
pfdict, FREQ, total = gen_pfdict(dict_file_content)
logger.debug("Dumping model to file cache %s" % cache_file)
try:
import StringIO
fd = StringIO.StringIO()
fd.write(marshal.dumps((pfdict, FREQ, total)))
if bucket.stat_object(X_CACHE_FILE):
bucket.delete_object(X_CACHE_FILE)
bucket.put_object(X_CACHE_FILE, fd.getvalue())
except Exception:
logger.exception("Dump cache file failed.")
initialized = True
logger.debug("Loading model cost %s seconds." % (time.time() - t1))
logger.debug("Prefix dict has been built succesfully.")
class AvatarManager(object):
def __init__(self,app):
self.app = app
self.bucket = Bucket(app.config['STORAGE_NAME'])
self.bucket.put()
self.bucket.post(acl='.r:.sinaapp.com,.r:sae.sina.com.cn', metadata={'expires': '1d'})
def _saveAvatar(self,file,filename,size,sizecode):
filetype = file.mimetype.split('/')[1]
img = Image.open(StringIO(file.read()))
file.seek(0)
ratio = float(img.size[0]) / img.size[1]
if ratio > 1:
length = int(img.size[1]/2)
else:
length = int(img.size[0]/2)
x0,y0 = int(img.size[0]/2),int(img.size[1]/2)
img = img.crop((x0-length,y0-length,x0+length,y0+length))
img = img.resize((size,size),Image.ANTIALIAS)
output = StringIO()
img.save(output,filetype)
content = output.getvalue()
output.close()
self.bucket.put_object('avatar/'+filename, content, content_type=file.mimetype)
def saveAvatar(self,file,id):
filename = str(id)
self._saveAvatar(file,filename,self.app.config['AVATAR_SIZE_S'],'s')
self._saveAvatar(file,filename,self.app.config['AVATAR_SIZE_M'],'m')
self._saveAvatar(file,filename,self.app.config['AVATAR_SIZE_L'],'l')
def getAvatarUrl(self,id,sizecode):
try:
self.bucket.stat_object('avatar/'+str(id)+sizecode)
except :
return self.app.config['DEFAULT_AVATAR_URL']+'/'+sizecode
return self.bucket.generate_url('avatar/'+str(id)+sizecode)
class SaeStorage(Storage):
"""实现存储在SAE上的Storage
"""
def __init__(self, bucket_name, path):
self.bucket_name = bucket_name
self.folder = path
self._bucket = Bucket(bucket_name)
self.locks = {}
def __repr__(self):
return "%s(%r)(%r)" % (self.__class__.__name__, self.bucket_name,
self.folder)
def create(self):
return self
def destroy(self):
# Remove all files
self.clean()
# REMOVE locks
del self.locks
def create_file(self, name, **kwargs):
def onclose_fn(sfile):
self._bucket.put_object(self._fpath(name), sfile.file.getvalue())
f = StructFile(BytesIO(), name=name, onclose=onclose_fn)
return f
def open_file(self, name, **kwargs):
if self._bucket.stat_object(self._fpath(name)) is None:
raise NameError(name)
content = self._bucket.get_object_contents(self._fpath(name))
def onclose_fn(sfile):
self._bucket.put_object(self._fpath(name), sfile.file.getvalue())
return StructFile(BytesIO(content), name=name, onclose=onclose_fn)
def _fpath(self, fname):
return os.path.join(self.folder, fname)
def clean(self):
files = self.list()
for fname in files:
self._bucket.delete_object(self._fpath(fname))
def list(self):
file_generate = self._bucket.list(path=self._fpath(""))
file_names = []
for f in file_generate:
file_names.append(f['name'][len(self.folder) + 1:])
return file_names
def file_exists(self, name):
return name in self.list()
def file_modified(self, name):
return self._bucket.stat_object(self._fpath(name))\
.get('last_modified', '')
def file_length(self, name):
return int(self._bucket.stat_object(self._fpath(name))['bytes'])
def delete_file(self, name):
self._bucket.delete_object(self._fpath(name))
def rename_file(self, name, newname, safe=False):
if name not in self.list():
raise NameError(name)
if safe and newname in self.list():
raise NameError("File %r exists" % newname)
content = self._bucket.get_object_contents(self._fpath(name))
self._bucket.delete_object(self._fpath(name))
self._bucket.put_object(self._fpath(newname), content)
def lock(self, name):
if name not in self.locks:
self.locks[name] = Lock()
return self.locks[name]
def temp_storage(self, name=None):
name = name or "%s.tmp" % random_name()
path = os.path.join(self.folder, name)
tempstore = SaeStorage(self.bucket_name, path)
return tempstore.create()
def manage(request):
from sae.storage import Bucket
bucket = Bucket('abc')
sa = t.objects.all()
for x in sa:
x.delete()
if request.GET: #delete
if request.GET.has_key('ctitle'):
name = request.GET["ctitle"]
bucket.delete_object('manage/'+name)
bucket.delete_object('stati/'+name)
ta = imagess.objects.filter(title = name)
if (len(ta)!= 0):
for i in ta:
i.delete()
if request.GET.has_key('stitle'): #save beautify
name = request.GET["stitle"]
if (name != '0'):
new_name = name[2:]
new_comment = '..'
new_mood = '..'
ta = imagess.objects.filter(title = new_name)
if (len(ta)!= 0):
new_comment = ta[0].comment
new_mood = ta[0].mood
new_lat = ta[0].lat
new_lon = ta[0].lon
new_photo = imagess(picture = 0, comment = new_comment, mood = new_mood, \
title = name, lat = new_lat, lon = new_lon)
new_photo.save()
obj = bucket.get_object_contents('meihua/'+name)
im = Image.open(StringIO.StringIO(obj))
imgout = StringIO.StringIO()
im.save(imgout,"jpeg")
img_data = imgout.getvalue()
bucket.put_object('stati/'+name, img_data)
im = Image.open(StringIO.StringIO(obj))
out = im.resize((128, 128))
imgout = StringIO.StringIO()
out.save(imgout,"jpeg")
img_data = imgout.getvalue()
bucket.put_object('manage/'+name, img_data)
#陈列部分
A = []
a = bucket.list(path='manage/')
for i in a:
dic = []
s = i.name.split('/')[-1]
dic.append(s)
dic.append(i.last_modified)
ta = imagess.objects.filter(title = s)
if (len(ta)!= 0):
dic.append(ta[0].mood)
dic.append(ta[0].comment)
A.append(dic)
if request.GET:
if request.GET.has_key('search'):#search
if request.GET['writesearch'] != '':
A=[]
wcomment = request.GET['writesearch']
result = imagess.objects.filter(comment = wcomment)
for i in range(0, len(result)):
a = bucket.stat_object('manage/'+result[i].title)
dic = []
dic.append(result[i].title)
dic.append(a.last_modified)
dic.append(result[i].mood)
dic.append(result[i].comment)
A.append(dic)
return render_to_response('manage.html',{'A':A },\
context_instance=RequestContext(request))
class SaeStorage(Storage):
"""实现存储在SAE上的Storage
"""
def __init__(self, bucket_name, path):
self.bucket_name = bucket_name
self.folder = path
self._bucket = Bucket(bucket_name)
self.locks = {}
def __repr__(self):
return "%s(%r)(%r)" % (self.__class__.__name__, self.bucket_name,
self.folder)
def create(self):
return self
def destroy(self):
# Remove all files
self.clean()
# REMOVE locks
del self.locks
def create_file(self, name, **kwargs):
def onclose_fn(sfile):
self._bucket.put_object(self._fpath(name), sfile.file.getvalue())
f = StructFile(BytesIO(), name=name, onclose=onclose_fn)
return f
def open_file(self, name, **kwargs):
if self._bucket.stat_object(self._fpath(name)) is None:
raise NameError(name)
content = self._bucket.get_object_contents(self._fpath(name))
def onclose_fn(sfile):
new_content = sfile.file.getvalue()
if new_content != content:
self._bucket.put_object(self._fpath(name), new_content)
return StructFile(BytesIO(content), name=name, onclose=onclose_fn)
def _fpath(self, fname):
return os.path.join(self.folder, fname)
def clean(self):
files = self.list()
for fname in files:
self._bucket.delete_object(self._fpath(fname))
def list(self):
file_generate = self._bucket.list(path=self._fpath(""))
file_names = []
for f in file_generate:
file_names.append(f['name'][len(self.folder)+1:])
return file_names
def file_exists(self, name):
return name in self.list()
def file_modified(self, name):
return self._bucket.stat_object(self._fpath(name))\
.get('last_modified', '')
def file_length(self, name):
return int(self._bucket.stat_object(self._fpath(name))['bytes'])
def delete_file(self, name):
self._bucket.delete_object(self._fpath(name))
def rename_file(self, name, newname, safe=False):
name_list = self.list()
if name not in name_list:
raise NameError(name)
if safe and newname in name_list:
raise NameError("File %r exists" % newname)
content = self._bucket.get_object_contents(self._fpath(name))
self._bucket.delete_object(self._fpath(name))
self._bucket.put_object(self._fpath(newname), content)
def lock(self, name):
if name not in self.locks:
self.locks[name] = Lock()
return self.locks[name]
def temp_storage(self, name=None):
temp_store = RamStorage()
return temp_store.create()
class SAEStorage(LocalStorage):
def __init__(self, bucket):
from sae.storage import Bucket
self.bucket = Bucket(bucket)
bucket_stat = self.bucket.stat()
#self.last_mark = bucket_stat.objects + \
# bucket_stat.bytes
self.last_mark = 0
def list(self):
articles = self.bucket.list()
filter_func = lambda x : self.is_article(x.name)
articles = filter(filter_func, articles)
articles = self._sorted_files(articles)
rst = []
for article in articles:
article_name = article.name
content = self.bucket.get_object_contents(article_name)
content = content.decode('utf-8')
art_meta = self._get_metadatas(content)
art_meta['filename'] = article_name
if type(article.name) == unicode:
adjust_name = article_name.encode('utf-8')
else :
adjust_name = article_name
art_meta['filename_url_encode'] = \
base64.urlsafe_b64encode(adjust_name)
if not art_meta['date']:
art_meta['date'] = article.last_modified
if not art_meta["slug"]:
art_meta['slug'] = article_name.rpartition(".")[0]
art_meta['slug'] = art_meta['slug'].replace("_", " ")
rst.append(art_meta)
return rst
def get(self, article, cut = -1):
content = self.bucket.get_object_contents(article)
content = content.decode('utf-8')
content = unicode(content)
mdparse = MDParse()
if cut != -1:
content = content[:cut]
content += "\n....."
content = self._clean_metadatas(content)
return mdparse.parse(content)
def save(self, name, content):
self.bucket.put_object(name, content)
def delete(self, name):
self.bucket.delete_object(name)
def update_time(self, article):
stat = self.bucket.stat_object(article)
tmp = float(stat.timestamp)
d = datetime.datetime.fromtimestamp(tmp)
return d.strftime("%Y-%m-%d %H:%M:%S")
def has_last(self):
bucket_stat = self.bucket.stat()
curr_mark = bucket_stat.objects + bucket_stat.bytes
res = self.last_mark == curr_mark
self.last_mark = curr_mark
return not res
def _sorted_files(self, articles):
def key_func(x):
stat = self.bucket.stat_object(x.name)
return float(stat.timestamp)
return sorted(articles, key=key_func, reverse=True)
class SAEStorageKVDB(object):
'''
A bridge for pyutils kvdb wrapper to access sae storage like a kvdb.
e.g.
from pyutils import KVDBWrapper
from sae_storage import SAEStorageKVDB
kvclient = KVDBWrapper(SAEStorageKVDB, bucket='mybucket', accesskey='xxx', secretkey='yyy', account='myapp', prefix='PRE:', ...)
kvclient.set(key,value)
kvclient.get(key)
'''
def __init__(self, **kwargs):
bucket = kwargs['bucket'] if 'bucket' in kwargs else ''
accesskey = kwargs['accesskey'] if 'accesskey' in kwargs else ''
secretkey = kwargs['secretkey'] if 'secretkey' in kwargs else ''
account = kwargs['account'] if 'account' in kwargs else '' # app name
retries = long(
kwargs['retries']) if 'retries' in kwargs else 3 # app name
self.prefix = kwargs['prefix'] if 'prefix' in kwargs else ''
if accesskey and secretkey and account:
conn = Connection(accesskey, secretkey, account, retries)
self.kv = conn.get_bucket(bucket)
else:
self.kv = Bucket(bucket)
def info(self):
return self.kv.stat()
def get(self, key, **kwargs):
k = self.prefix + key
return self.kv.get_object(k, **kwargs)
def set(self, key, value, **kwargs):
k = self.prefix + key
return self.kv.put_object(k, value, **kwargs)
def delete(self, key, **kwargs):
k = self.prefix + key
return self.kv.delete_object(k, **kwargs)
def exist(self, key, **kwargs):
k = self.prefix + key
rc = False
st = self.kv.stat_object(k)
if st: rc = True
return rc
def scan(self, cursor=None, count=100, **kwargs):
'''
Retrieve keys by given arguments
:param kwargs:
'count' for retrieve count;
'cursor' is the key for next time retrieve
:return:
'''
return self.kv.c(prefix=self.prefix,
marker=cursor,
limit=count,
**kwargs)
def scanv(self, cursor=None, count=100, **kwargs):
'''
Retrieve key-values by given arguments
:param kwargs:
'count' for retrieve count;
'cursor' is the key for next time retrieve
:return:
'''
return self.kv.list(prefix=self.prefix,
marker=cursor,
limit=count,
**kwargs)
def mget(self, keys, **kwargs):
for key in keys:
k = self.prefix + key
val = self.get(k, **kwargs)
yield val
