def GetImageRecords(photo_ids):
"""Queries the ImageRecord db, w/ memcaching. Returns photo_id -> rec dict"""
# check if we've got the whole thing in memcache
keys_no_prefix = photo_ids[:]
if MEMCACHE_ENABLED:
multi_key = VERSION + 'MIR' + ','.join(photo_ids)
recs = memcache.get(multi_key)
if recs: return recs
key_prefix = VERSION + 'IR'
record_map = memcache.get_multi(keys_no_prefix, key_prefix=key_prefix)
missing_ids = list(set(keys_no_prefix) - set(record_map.keys()))
if not missing_ids: return record_map
else:
missing_ids = keys_no_prefix
record_map = {}
config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
db_recs = ImageRecord.get_by_key_name(missing_ids, config=config)
memcache_map = {}
for id, r in zip(missing_ids, db_recs):
record_map[id] = r
memcache_map[id] = r
if MEMCACHE_ENABLED and memcache_map:
memcache.add_multi(memcache_map, key_prefix=key_prefix)
memcache.add(multi_key, record_map)
return record_map
def get_seeds(self):
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY)
results = q.get(config=config)
return results.seeds
def GetImageRecords(photo_ids):
"""Queries the ImageRecord db, w/ memcaching. Returns photo_id -> rec dict"""
# check if we've got the whole thing in memcache
multi_key = VERSION + 'MIR' + ','.join(photo_ids)
recs = memcache.get(multi_key)
if recs: return recs
keys_no_prefix = photo_ids[:]
key_prefix = VERSION + 'IR'
record_map = memcache.get_multi(keys_no_prefix, key_prefix=key_prefix)
missing_ids = list(set(keys_no_prefix) - set(record_map.keys()))
if not missing_ids: return record_map
config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
db_recs = ImageRecord.get_by_key_name(missing_ids, config=config)
memcache_map = {}
for id, r in zip(missing_ids, db_recs):
record_map[id] = r
memcache_map[id] = r
if memcache_map:
memcache.add_multi(memcache_map, key_prefix=key_prefix)
memcache.add(multi_key, record_map)
return record_map
def get_seeds(self):
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5,
read_policy=db.EVENTUAL_CONSISTENCY)
results = q.get(config=config)
return results.seeds
def add_to_seeds(self,seed):
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY)
results = q.get(config=config)
results.seeds.append(seed)
results.seeds = list(set(p.seeds))
results.put()
def add_to_seeds(self, seed):
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5,
read_policy=db.EVENTUAL_CONSISTENCY)
results = q.get(config=config)
results.seeds.append(seed)
results.seeds = list(set(p.seeds))
results.put()
def wrapper(x, *args, **kwds):
if not kwds.has_key('config'):
if f.__name__ == 'get':
params = {
'read_policy':DATASTORE_CONSISTENCY,
'deadline': DATASTORE_DEADLINE,}
else:
params = {
'deadline': DATASTORE_DEADLINE,}
kwds['config'] = db.create_config(**params)
return f(x, *args, **kwds)
def wrapper(x, *args, **kwds):
if not kwds.has_key('config'):
if f.__name__ == 'get':
params = {
'read_policy':DATASTORE_CONSISTENCY,
'deadline': DATASTORE_DEADLINE,}
else:
params = {
'deadline': DATASTORE_DEADLINE,}
kwds['config'] = db.create_config(**params)
return f(x, *args, **kwds)
def force_set_seeds(self,seeds):
logging.info("passed seeds: %s" % seeds)
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY)
results = q.get(config=config)
logging.info("result seeds: %s" % results.seeds)
results.seeds = seeds
results.put()
def init_seed_model(self):
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY)
result_count = q.count(config=config)
if result_count == 0:
try:
seed_model = NewsMeDigestionSeedUsers(seeds=self._static_known_seeds)
seed_model.put()
except Exception, exception:
# ERROR 2012-06-02 15:07:56,629 newmedigester.py:321] 'ascii' codec can't decode byte 0xe2 in position 7: ordinal not in range(128)
logging.error("failed to save static seeds to model: %s" % (self._static_known_seeds))
logging.error(exception)
def force_set_seeds(self, seeds):
logging.info("passed seeds: %s" % seeds)
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5,
read_policy=db.EVENTUAL_CONSISTENCY)
results = q.get(config=config)
logging.info("result seeds: %s" % results.seeds)
results.seeds = seeds
results.put()
def init_seed_model(self):
q = NewsMeDigestionSeedUsers.all()
config = db.create_config(deadline=5,
read_policy=db.EVENTUAL_CONSISTENCY)
result_count = q.count(config=config)
if result_count == 0:
try:
seed_model = NewsMeDigestionSeedUsers(
seeds=self._static_known_seeds)
seed_model.put()
except Exception, exception:
# ERROR 2012-06-02 15:07:56,629 newmedigester.py:321] 'ascii' codec can't decode byte 0xe2 in position 7: ordinal not in range(128)
logging.error("failed to save static seeds to model: %s" %
(self._static_known_seeds))
logging.error(exception)
def post(self):
recs = self.request.get_all('d')
photo_id_to_dims = {}
for rec in recs:
photo_id, width_str, height_str = rec.split(',')
width = int(width_str)
height = int(height_str)
photo_id_to_dims[photo_id] = (width, height)
config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
db_recs = ImageRecord.get_by_key_name(photo_id_to_dims.keys(), config=config)
for image in db_recs:
width, height = photo_id_to_dims[image.photo_id]
if image.width == width and image.height == height:
continue
image.width = width
image.height = height
image.put()
self.response.out.write('Added %d dimensions' % len(db_recs))
def post(self):
recs = self.request.get_all('d')
photo_id_to_dims = {}
for rec in recs:
photo_id, width_str, height_str = rec.split(',')
width = int(width_str)
height = int(height_str)
photo_id_to_dims[photo_id] = (width, height)
config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
db_recs = ImageRecord.get_by_key_name(photo_id_to_dims.keys(),
config=config)
for image in db_recs:
width, height = photo_id_to_dims[image.photo_id]
if image.width == width and image.height == height:
continue
image.width = width
image.height = height
image.put()
self.response.out.write('Added %d dimensions' % len(db_recs))
def _get_db_run_config_eventual(self):
# https://developers.google.com/appengine/docs/python/datastore/queries?hl=en#Data_Consistency
return db.create_config(deadline=10, read_policy=db.EVENTUAL_CONSISTENCY)
import re
from google.appengine.api import users, memcache, mail
from google.appengine.ext import db, deferred
from setting import *
from common import *
class EntityNotFound(object):
def __nonzero__(self):
return False
ENTITY_NOT_FOUND = EntityNotFound()
EVENTUAL_CONSISTENCY_CONFIG = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
QUICK_LIMITED_EVENTUAL_CONSISTENCY_CONFIG = db.create_config(deadline=0.5, read_policy=db.EVENTUAL_CONSISTENCY)
USER_FLAGS = {
'active': 1, # not be banned
'verified': 2,
'mute': 4 # whether or not receive email notification
}
class User(db.Model):
# key name: email
name = db.StringProperty(required=True)
site = db.StringProperty(indexed=False)
flag = db.IntegerProperty(default=USER_FLAGS['active'] | USER_FLAGS['verified'], indexed=False)
def to_dict(self):
def _get_db_run_config_eventual(self):
# https://developers.google.com/appengine/docs/python/datastore/queries?hl=en#Data_Consistency
return db.create_config(deadline=10,
read_policy=db.EVENTUAL_CONSISTENCY)
from google.appengine.api import users, memcache, mail
from google.appengine.ext import db, deferred
from setting import *
from common import *
class EntityNotFound(object):
def __nonzero__(self):
return False
ENTITY_NOT_FOUND = EntityNotFound()
EVENTUAL_CONSISTENCY_CONFIG = db.create_config(
read_policy=db.EVENTUAL_CONSISTENCY)
QUICK_LIMITED_EVENTUAL_CONSISTENCY_CONFIG = db.create_config(
deadline=0.5, read_policy=db.EVENTUAL_CONSISTENCY)
USER_FLAGS = {
'active': 1, # 可用，未禁言
'verified': 2, # 已验证
'mute': 4, # 是否不接收邮件通知
'noua': 8, # 是否不显示UA(默认显示，0)
}
class User(db.Model):
# key name: email 邮箱
name = db.StringProperty(required=True) # 用户名
site = db.StringProperty(indexed=False) # 主页
