def GetImageRecords(photo_ids): """Queries the ImageRecord db, w/ memcaching. Returns photo_id -> rec dict""" # check if we've got the whole thing in memcache keys_no_prefix = photo_ids[:] if MEMCACHE_ENABLED: multi_key = VERSION + 'MIR' + ','.join(photo_ids) recs = memcache.get(multi_key) if recs: return recs key_prefix = VERSION + 'IR' record_map = memcache.get_multi(keys_no_prefix, key_prefix=key_prefix) missing_ids = list(set(keys_no_prefix) - set(record_map.keys())) if not missing_ids: return record_map else: missing_ids = keys_no_prefix record_map = {} config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY) db_recs = ImageRecord.get_by_key_name(missing_ids, config=config) memcache_map = {} for id, r in zip(missing_ids, db_recs): record_map[id] = r memcache_map[id] = r if MEMCACHE_ENABLED and memcache_map: memcache.add_multi(memcache_map, key_prefix=key_prefix) memcache.add(multi_key, record_map) return record_map
def get_seeds(self): q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) results = q.get(config=config) return results.seeds
def GetImageRecords(photo_ids): """Queries the ImageRecord db, w/ memcaching. Returns photo_id -> rec dict""" # check if we've got the whole thing in memcache multi_key = VERSION + 'MIR' + ','.join(photo_ids) recs = memcache.get(multi_key) if recs: return recs keys_no_prefix = photo_ids[:] key_prefix = VERSION + 'IR' record_map = memcache.get_multi(keys_no_prefix, key_prefix=key_prefix) missing_ids = list(set(keys_no_prefix) - set(record_map.keys())) if not missing_ids: return record_map config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY) db_recs = ImageRecord.get_by_key_name(missing_ids, config=config) memcache_map = {} for id, r in zip(missing_ids, db_recs): record_map[id] = r memcache_map[id] = r if memcache_map: memcache.add_multi(memcache_map, key_prefix=key_prefix) memcache.add(multi_key, record_map) return record_map
def add_to_seeds(self,seed): q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) results = q.get(config=config) results.seeds.append(seed) results.seeds = list(set(p.seeds)) results.put()
def add_to_seeds(self, seed): q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) results = q.get(config=config) results.seeds.append(seed) results.seeds = list(set(p.seeds)) results.put()
def wrapper(x, *args, **kwds): if not kwds.has_key('config'): if f.__name__ == 'get': params = { 'read_policy':DATASTORE_CONSISTENCY, 'deadline': DATASTORE_DEADLINE,} else: params = { 'deadline': DATASTORE_DEADLINE,} kwds['config'] = db.create_config(**params) return f(x, *args, **kwds)
def force_set_seeds(self,seeds): logging.info("passed seeds: %s" % seeds) q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) results = q.get(config=config) logging.info("result seeds: %s" % results.seeds) results.seeds = seeds results.put()
def init_seed_model(self): q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) result_count = q.count(config=config) if result_count == 0: try: seed_model = NewsMeDigestionSeedUsers(seeds=self._static_known_seeds) seed_model.put() except Exception, exception: # ERROR 2012-06-02 15:07:56,629 newmedigester.py:321] 'ascii' codec can't decode byte 0xe2 in position 7: ordinal not in range(128) logging.error("failed to save static seeds to model: %s" % (self._static_known_seeds)) logging.error(exception)
def force_set_seeds(self, seeds): logging.info("passed seeds: %s" % seeds) q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) results = q.get(config=config) logging.info("result seeds: %s" % results.seeds) results.seeds = seeds results.put()
def init_seed_model(self): q = NewsMeDigestionSeedUsers.all() config = db.create_config(deadline=5, read_policy=db.EVENTUAL_CONSISTENCY) result_count = q.count(config=config) if result_count == 0: try: seed_model = NewsMeDigestionSeedUsers( seeds=self._static_known_seeds) seed_model.put() except Exception, exception: # ERROR 2012-06-02 15:07:56,629 newmedigester.py:321] 'ascii' codec can't decode byte 0xe2 in position 7: ordinal not in range(128) logging.error("failed to save static seeds to model: %s" % (self._static_known_seeds)) logging.error(exception)
def post(self): recs = self.request.get_all('d') photo_id_to_dims = {} for rec in recs: photo_id, width_str, height_str = rec.split(',') width = int(width_str) height = int(height_str) photo_id_to_dims[photo_id] = (width, height) config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY) db_recs = ImageRecord.get_by_key_name(photo_id_to_dims.keys(), config=config) for image in db_recs: width, height = photo_id_to_dims[image.photo_id] if image.width == width and image.height == height: continue image.width = width image.height = height image.put() self.response.out.write('Added %d dimensions' % len(db_recs))
def _get_db_run_config_eventual(self): # https://developers.google.com/appengine/docs/python/datastore/queries?hl=en#Data_Consistency return db.create_config(deadline=10, read_policy=db.EVENTUAL_CONSISTENCY)
import re from google.appengine.api import users, memcache, mail from google.appengine.ext import db, deferred from setting import * from common import * class EntityNotFound(object): def __nonzero__(self): return False ENTITY_NOT_FOUND = EntityNotFound() EVENTUAL_CONSISTENCY_CONFIG = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY) QUICK_LIMITED_EVENTUAL_CONSISTENCY_CONFIG = db.create_config(deadline=0.5, read_policy=db.EVENTUAL_CONSISTENCY) USER_FLAGS = { 'active': 1, # not be banned 'verified': 2, 'mute': 4 # whether or not receive email notification } class User(db.Model): # key name: email name = db.StringProperty(required=True) site = db.StringProperty(indexed=False) flag = db.IntegerProperty(default=USER_FLAGS['active'] | USER_FLAGS['verified'], indexed=False) def to_dict(self):
from google.appengine.api import users, memcache, mail from google.appengine.ext import db, deferred from setting import * from common import * class EntityNotFound(object): def __nonzero__(self): return False ENTITY_NOT_FOUND = EntityNotFound() EVENTUAL_CONSISTENCY_CONFIG = db.create_config( read_policy=db.EVENTUAL_CONSISTENCY) QUICK_LIMITED_EVENTUAL_CONSISTENCY_CONFIG = db.create_config( deadline=0.5, read_policy=db.EVENTUAL_CONSISTENCY) USER_FLAGS = { 'active': 1, # 可用,未禁言 'verified': 2, # 已验证 'mute': 4, # 是否不接收邮件通知 'noua': 8, # 是否不显示UA(默认显示,0) } class User(db.Model): # key name: email 邮箱 name = db.StringProperty(required=True) # 用户名 site = db.StringProperty(indexed=False) # 主页