def get_entity_from_protobuf(data): """Converts one or more encoded Protocol Buffers to ``db.Model`` instances. This is used to de-serialize entities previously serialized using :func:`get_protobuf_from_entity`. After retrieving an entity protobuf from memcache, this converts it back to a ``db.Model`` instance. Example:: from google.appengine.api import memcache from tipfy.appengine.db import get_entity_from_protobuf # Get the protobuf from cache and de-serialize it. protobuf = memcache.get('my-cache-key') if protobuf: entity = get_entity_from_protobuf(protobuf) This function derives from `Nick's Blog <http://blog.notdot.net/2009/9/Efficient-model-memcaching>`_. :param data: One or more entities serialized to Protocol Buffer (a string or a list). :returns: One or more entities de-serialized from Protocol Buffers (a ``db.Model`` inatance or a list of ``db.Model`` instances). """ if not data: return None elif isinstance(data, str): return db.model_from_protobuf(data) elif isinstance(data, dict): return dict((k, db.model_from_protobuf(v)) for k, v in data.iteritems()) else: return [db.model_from_protobuf(x) for x in data]
def get_entity_from_protobuf(data): """Converts one or more encoded Protocol Buffers to ``db.Model`` instances. This is used to de-serialize entities previously serialized using :func:`get_protobuf_from_entity`. After retrieving an entity protobuf from memcache, this converts it back to a ``db.Model`` instance. Example:: from google.appengine.api import memcache from tipfy.appengine.db import get_entity_from_protobuf # Get the protobuf from cache and de-serialize it. protobuf = memcache.get('my-cache-key') if protobuf: entity = get_entity_from_protobuf(protobuf) This function derives from `Nick's Blog <http://blog.notdot.net/2009/9/Efficient-model-memcaching>`_. :param data: One or more entities serialized to Protocol Buffer (a string or a list). :returns: One or more entities de-serialized from Protocol Buffers (a ``db.Model`` inatance or a list of ``db.Model`` instances). """ if not data: return None elif isinstance(data, str): return db.model_from_protobuf(data) elif isinstance(data, dict): return dict( (k, db.model_from_protobuf(v)) for k, v in data.iteritems()) else: return [db.model_from_protobuf(x) for x in data]
def deserialize_entities(data): if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_entities(data): if data is None: return None elif isinstance(data, str): return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def protobuf_decoded(*args, **kwargs): v = func(*args, **kwargs) if type(v) == list: return map(lambda pb: db.model_from_protobuf( entity_pb.EntityProto(pb)), v) else: return db.model_from_protobuf(entity_pb.EntityProto(v))
def deserialize_entities(self, data): # deserialize from protocol buffer if not data: return None elif isinstance(data, str): return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def unpack(data): if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def protobuf_decoded(*args, **kwargs): v = func(*args, **kwargs) if type(v) == list: return map( lambda pb: db.model_from_protobuf(entity_pb.EntityProto(pb)), v) else: return db.model_from_protobuf(entity_pb.EntityProto(v))
def deserialize_entities(data): """Deserializes a protobuf into either a single db.Model or a list of them.""" if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(data) else: return [db.model_from_protobuf(item) for item in data]
def _deserialize(data): '''Improve memcache performance by converting from protobuf''' if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def _deserialize(data): '''Improve memcache performance by converting from protobuf''' if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_entities(data): from google.appengine.datastore import entity_pb if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_entities(data): # http://blog.notdot.net/2009/9/Efficient-model-memcaching if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_models(data): """Deserialize a list of models efficiently for memcached.""" if data is None: return None elif isinstance(data, str): return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_entities(data): """Given the representation of a serialized entity, or entities, return the deserialized form.""" if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_entities(self, data): # deserialize from protocol buffer if not data: return None elif isinstance(data, str): return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [ db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data ]
def deserialize_entities(data): """Given the representation of a serialized entity, or entities, return the deserialized form.""" if data is None: return None elif isinstance(data, str): # Just one instance return db.model_from_protobuf(entity_pb.EntityProto(data)) else: return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]
def deserialize_instances(data, _search_class=None): """ Get and deserializes the data from the cache, return a instance of the model """ if data is None: return None elif _search_class is not None: if type(data) == type(list): return [search.SearchableEntity.FromPb(x) for x in data]#data contains a list of objects return _search_class.from_entity(search.SearchableEntity.FromPb(data)) elif isinstance(data, str): return db.model_from_protobuf(entity_pb.EntityProto(data))#just one instance in data return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]#data contains a list of objects
def deserialize_instances(data): """ Get and deserializes the data from the cache, return a instance of the model """ if data is None: return None elif isinstance(data, str): return db.model_from_protobuf(entity_pb.EntityProto(data))#just one instance in data return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data]#data contains a list of objects
def get(path): """Return the StaticContent object for the given path. Args: path: The path to get the content from. Return: A StaticContent object or None. """ # Nick Johnson explains the logic of the code here: # http://blog.notdot.net/2009/9/Efficient-model-memcaching # Using Protocol Buffers is way faster and more efficient than # pickling. # Addresses are case-insensitive. path = path.lower() # Try to retrieve the content from memcache. entity = memcache.get(path) if entity is not None: # Yeah, it was there. Decode it. entity = db.model_from_protobuf(entity_pb.EntityProto(entity)) else: # Bad luck. Have to get that entity from its key (the path). entity = StaticContent.get_by_key_name(path) if entity is not None: # Now store it for next time, after encoding. memcache.set(path, db.model_to_protobuf(entity).Encode()) return entity
def deserialize_instances(data, _search_class=None): """ Get and deserializes the data from the cache, return a instance of the model """ if data is None: return None elif _search_class is not None: if type(data) == type(list): return [search.SearchableEntity.FromPb(x) for x in data] #data contains a list of objects return _search_class.from_entity(search.SearchableEntity.FromPb(data)) elif isinstance(data, str): return db.model_from_protobuf( entity_pb.EntityProto(data)) #just one instance in data return [db.model_from_protobuf(entity_pb.EntityProto(x)) for x in data] #data contains a list of objects
def _query_work(self, index, cursor): """Queries for work in memcache.""" if cursor: try: cursor = int(cursor) except ValueError: # This is an old style task that resides in the Datastore, not # memcache. Use the parent implementation instead. return super(MemcacheForkJoinQueue, self)._query_work(index, cursor) else: cursor = 0 key_list = [self._create_index_key(index, n) for n in xrange(cursor, cursor + self.batch_size)] results = memcache.get_multi(key_list) result_list = [] for key in key_list: proto = results.get(key) if not proto: continue try: result_list.append(db.model_from_protobuf(proto)) except ProtocolBuffer.ProtocolBufferDecodeError: logging.exception('Could not decode EntityPb at memcache key %r: %r', key, proto) return result_list, cursor + self.batch_size
def _query_work(self, index, cursor): """Queries for work in memcache.""" if cursor: try: cursor = int(cursor) except ValueError: # This is an old style task that resides in the Datastore, not # memcache. Use the parent implementation instead. return super(MemcacheForkJoinQueue, self)._query_work(index, cursor) else: cursor = 0 key_list = [ self._create_index_key(index, n) for n in xrange(cursor, cursor + self.batch_size) ] results = memcache.get_multi(key_list) result_list = [] for key in key_list: proto = results.get(key) if not proto: continue try: result_list.append(db.model_from_protobuf(proto)) except ProtocolBuffer.ProtocolBufferDecodeError: logging.exception( 'Could not decode EntityPb at memcache key %r: %r', key, proto) return result_list, cursor + self.batch_size
def get_filter_models(self, bridge_name): if bridge_name in self.filters: return self.filter_models.setdefault(bridge_name, [db.model_from_protobuf(entity_pb.EntityProto(filter)) for filter in self.filters[bridge_name]]) else: return None
def get_bridge_model(self, bridge_name): if bridge_name in self.bridges: return self.bridge_models.setdefault(bridge_name, db.model_from_protobuf( entity_pb.EntityProto(self.bridges[bridge_name]))) else: return None
def batch_put(mbc_name, bucket_key, list_keys, decrementing=False): from apps.user.models import * logging.info("Batch putting %s to memcache: %s" % (mbc_name, list_keys)) mbc = MemcacheBucketConfig.get_or_create(mbc_name) entities_to_put = [] had_error = False object_dict = memcache.get_multi(list_keys) for key in list_keys: data = object_dict.get(key) try: entity = db.model_from_protobuf(entity_pb.EntityProto(data)) if entity: entities_to_put.append(entity) except AssertionError, e: old_key = mbc.get_bucket(mbc.count) if bucket_key != old_key and not decrementing and not had_error: old_count = mbc.count mbc.decrement_count() logging.warn( 'encounted error, going to decrement buckets from %s to %s' % (old_count, mbc.count), exc_info=True) last_keys = memcache.get(old_key) or [] memcache.set(old_key, [], time=MEMCACHE_TIMEOUT) deferred.defer(batch_put, mbc_name, old_key, last_keys, decrementing=True, _queue='slow-deferred') had_error = True except Exception, e: logging.error('error getting object: %s' % e, exc_info=True)
def getAggregatedError(self, project, environment, server, backtraceText, errorHash, message, timestamp): """Gets (and updates) the error matching the given report, or None if no matching error is found.""" error = None key = '%s|%s' % (project, errorHash) serialized = memcache.get(key) if serialized: error = db.model_from_protobuf(serialized) else: q = LoggedError.all() q.filter('project =', project) q.filter('hash =', errorHash) q.filter('active =', True) for possibility in q: if backtrace.normalizeBacktrace(possibility.backtrace) == backtrace.normalizeBacktrace(backtraceText): error = possibility break if error: error.count += 1 error.firstOccurrence = min(error.firstOccurrence, timestamp) if timestamp > error.lastOccurrence: error.lastOccurrence = timestamp error.backtrace = backtraceText error.lastMessage = message[:300] if environment not in error.environments: error.environments.append(environment) if server not in error.servers: error.servers.append(server) error.put() memcache.set(key, db.model_to_protobuf(error)) return error
def deserialize_models(data): if data is None: return None elif isinstance(data, str): # Just one instance return db.protobuf_to_model(entity_pb.EntityProto(data)) return [db.model_from_protobuf(entity_pb.EntityProto(x)) if x is not None else None for x in data]
def get_document(request): """Decodes document from prospective_search result POST request. Args: request: received POST request Returns: document: original datastore.Entity or db.Model document from match call. Raises: DocumentTypeError: """ from google.appengine.ext import db doc_class = request.get('python_document_class') if not doc_class: return None entity = entity_pb.EntityProto() entity.ParseFromString( base64.urlsafe_b64decode(request.get('document').encode('utf-8'))) doc_class = int(doc_class) if doc_class is _doc_class.ENTITY: return datastore.Entity('temp-kind').FromPb(entity) elif doc_class is _doc_class.MODEL: return db.model_from_protobuf(entity) else: raise DocumentTypeError()
def getProject(name): """Gets the project with the given name.""" serialized = memcache.get('project:%s' % name) if serialized: return db.model_from_protobuf(serialized) else: return Project.get_or_insert(name)
def get_document(request): """Decodes document from prospective_search result POST request. Args: request: received POST request Returns: document: original datastore.Entity or db.Model document from match call. Raises: DocumentTypeError: """ from google.appengine.ext import db doc_class = request.get('python_document_class') if not doc_class: return None entity = entity_pb.EntityProto() entity.ParseFromString(base64.urlsafe_b64decode( request.get('document').encode('utf-8'))) doc_class = int(doc_class) if doc_class is _doc_class.ENTITY: return datastore.Entity('temp-kind').FromPb(entity) elif doc_class is _doc_class.MODEL: return db.model_from_protobuf(entity) else: raise DocumentTypeError()
def get_document(request): """Decodes document from matcher result POST request. Args: request: received POST request Returns: document: document which was used to generate this match. Raises: ProtocolBufferDecodeError: DocumentTypeError: """ from google.appengine.ext import db doc_class = request.get('python_document_class') if not doc_class: return None entity = entity_pb.EntityProto() entity.ParseFromString(base64.urlsafe_b64decode( request.get('document').encode('utf-8'))) doc_class = int(doc_class) if doc_class is _doc_class.DICT: return dict(datastore.Entity('temp-kind').FromPb(entity)) elif doc_class is _doc_class.ENTITY: return datastore.Entity('temp-kind').FromPb(entity) elif doc_class is _doc_class.MODEL: return db.model_from_protobuf(entity) else: raise DocumentTypeError()
def getProject(name): """Gets the project with the given name.""" serialized = memcache.get('project:%s' % name) if serialized: return db.model_from_protobuf(serialized) else: return Project.get_or_insert(name)
def get_document(request): """Decodes document from matcher result POST request. Args: request: received POST request Returns: document: document which was used to generate this match. Raises: ProtocolBufferDecodeError: DocumentTypeError: """ from google.appengine.ext import db doc_class = request.get('python_document_class') if not doc_class: return None entity = entity_pb.EntityProto() entity.ParseFromString( base64.urlsafe_b64decode(request.get('document').encode('utf-8'))) doc_class = int(doc_class) if doc_class is _doc_class.DICT: return dict(datastore.Entity('temp-kind').FromPb(entity)) elif doc_class is _doc_class.ENTITY: return datastore.Entity('temp-kind').FromPb(entity) elif doc_class is _doc_class.MODEL: return db.model_from_protobuf(entity) else: raise DocumentTypeError()
def get_bridge_model(self, bridge_name): if bridge_name in self.bridges: return self.bridge_models.setdefault( bridge_name, db.model_from_protobuf( entity_pb.EntityProto(self.bridges[bridge_name]))) else: return None
def get_alternatives(self, experiment_name): if experiment_name not in self.alternative_models: if experiment_name in self.alternatives: self.alternative_models[experiment_name] = [] for alternative_number in self.alternatives[experiment_name]: self.alternative_models[experiment_name].append(db.model_from_protobuf(entity_pb.EntityProto(self.alternatives[experiment_name][alternative_number]))) return self.alternative_models.get(experiment_name) or []
def get_filter_models(self, bridge_name): if bridge_name in self.filters: return self.filter_models.setdefault(bridge_name, [ db.model_from_protobuf(entity_pb.EntityProto(filter)) for filter in self.filters[bridge_name] ]) else: return None
def get_experiment(self, experiment_name): if experiment_name not in self.experiment_models: if experiment_name in self.experiments: self.experiment_models[experiment_name] = db.model_from_protobuf( entity_pb.EntityProto(self.experiments[experiment_name]) ) return self.experiment_models.get(experiment_name)
def getProject(name): """Gets the project with the given name.""" serialized = memcache.get(name, namespace='projects') if serialized: return db.model_from_protobuf(serialized) else: result = Project.get_or_insert(name) memcache.set(name, db.model_to_protobuf(result), namespace='projects') return result
def get_experiment(self, experiment_name): if experiment_name not in self.experiment_models: if experiment_name in self.experiments: self.experiment_models[ experiment_name] = db.model_from_protobuf( entity_pb.EntityProto( self.experiments[experiment_name])) return self.experiment_models.get(experiment_name)
def __decode_data(pdump): """Returns a data dictionary after decoding it from "pickled+" form.""" try: eP, eO = pickle.loads(pdump) for k, v in eP.iteritems(): eO[k] = db.model_from_protobuf(v) except Exception, e: logging.warn("failed to decode session data: %s" % e) eO = {}
def __decode_data(pdump): """Returns a data dictionary after decoding it from "pickled+" form.""" try: eP, eO = pickle.loads(pdump) for k, v in eP.iteritems(): eO[k] = db.model_from_protobuf(v) except Exception, e: logging.warn("failed to decode session data: %s" % e) eO = {}
def decode_data(pdump): """Returns a data dictionary after decoding it from "pickled+" form.""" try: eP, eO = pickle.loads(pdump) for k, v in eP.iteritems(): eO[k] = db.model_from_protobuf(v) except Exception: eO = {} return eO
def getProject(name): """Gets the project with the given name.""" serialized = memcache.get(name, namespace = 'projects') if serialized: return db.model_from_protobuf(serialized) else: result = Project.get_or_insert(name) memcache.set(name, db.model_to_protobuf(result), namespace = 'projects') return result
def decode_data(pdump): """Returns a data dictionary after decoding it from "pickled+" form.""" try: eP, eO = pickle.loads(pdump) for k, v in eP.iteritems(): eO[k] = db.model_from_protobuf(v) except Exception: eO = {} return eO
def run(): # Set your downloaded folder's path here (must be readable by dev_appserver) mypath = '/local_target' # Set your app's name here appname = "dev~yourappnamehere" # Do the harlem shake onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] ec = datastore_pbs.get_entity_converter() for file in onlyfiles: i = 0 try: raw = open(mypath + "/" + file, 'r') reader = records.RecordsReader(raw) to_put = list() for record in reader: entity_proto = entity_pb.EntityProto(contents=record) entity_proto.key_.app_ = appname entity = db.model_from_protobuf(entity_proto) a = db.model_from_protobuf(entity_proto) for pp in dir(a): try: ppp = getattr(a, "_" + pp) if isinstance(ppp, db.Key): ppp._Key__reference.set_app(appname) ppp except AttributeError: """ It's okay """ to_put.append(a) i += 1 if i % 100 == 0: print "Saved %d %ss" % (i, entity.kind()) db.put(to_put) to_put = list() db.put(to_put) to_put = list() print "Saved %d" % i except ProtocolBufferDecodeError: """ All good """
def getInfoPage(self, pageKey): entity = memcache.get(pageKey) if entity: entity = db.model_from_protobuf(entity_pb.EntityProto(entity)) else: entity = models.PagePost.get_by_key_name(pageKey) if entity: memcache.set(pageKey, db.model_to_protobuf(entity).Encode()) return entity
def getInfoPage(self, pageKey): entity = memcache.get(pageKey) if entity: entity = db.model_from_protobuf(entity_pb.EntityProto(entity)) else: entity = models.PagePost.get_by_key_name(pageKey) if entity: memcache.set(pageKey, db.model_to_protobuf(entity).Encode()) return entity
def get_filter_models(self, bridge_name): """Callers must hold the lock returned by get_lock() when calling this method for thread-safe access.""" if bridge_name in self.filter_models: return self.filter_models[bridge_name] elif bridge_name in self.filters: models = [db.model_from_protobuf(entity_pb.EntityProto(filter)) for filter in self.filters[bridge_name]] self.filter_models[bridge_name] = models return models else: return None
def getStats(): protobuf = memcache.get('stats') if protobuf is not None: return db.model_from_protobuf(protobuf) else: stats = Stats.get_by_key_name('stats') if stats is not None: protobuf = db.model_to_protobuf(stats) memcache.set('stats', protobuf) return stats return createStats()
def get_filter_models(self, bridge_name): """Callers must hold the lock returned by get_lock() when calling this method for thread-safe access.""" if bridge_name in self.filter_models: return self.filter_models[bridge_name] elif bridge_name in self.filters: models = [db.model_from_protobuf(entity_pb.EntityProto(filter)) for filter in self.filters[bridge_name]] self.filter_models[bridge_name] = models return models else: return None
def get_bridge_model(self, bridge_name): """Callers must hold the lock returned by get_lock() when calling this method for thread-safe access.""" if bridge_name in self.bridge_models: return self.bridge_models[bridge_name] elif bridge_name in self.bridges: model_proto_contents = self.bridges[bridge_name] model = db.model_from_protobuf(entity_pb.EntityProto(model_proto_contents)) self.bridge_models[bridge_name] = model return model else: return None
def getPlayerSession(playerId): protobuf = memcache.get(prefix + playerId) if protobuf is not None: return db.model_from_protobuf(protobuf) else: session = PlayerSession.get_by_key_name(playerId) if session is not None: protobuf = db.model_to_protobuf(session) memcache.set(prefix + playerId, protobuf, DEFAULT_MAX_SESSION_LIFE_TIME.seconds) # TODO : set expiry time depending of creationDateTime return session return None
def get_bridge_model(self, bridge_name): """Callers must hold the lock returned by get_lock() when calling this method for thread-safe access.""" if bridge_name in self.bridge_models: return self.bridge_models[bridge_name] elif bridge_name in self.bridges: model_proto_contents = self.bridges[bridge_name] model = db.model_from_protobuf( entity_pb.EntityProto(model_proto_contents)) self.bridge_models[bridge_name] = model return model else: return None
def getAdmin(): protobuf = memcache.get('admin') if protobuf is not None: return db.model_from_protobuf(protobuf) else: admin = Admin.get_by_key_name('admin') if admin is not None: protobuf = db.model_to_protobuf(admin) memcache.set('admin', protobuf) return admin return createAdmin()
def LoadFromMemcache(self, key): """Try loading the game from memcache. Sets self.game, returns true if self.game was successfully set. If false, self.game was not set.""" encoded = memcache.get(key) if not encoded: logging.warn("Memcache miss.") return False try: self.game = db.model_from_protobuf(encoded) return True except db.Error, e: logging.warn("Game Model decode from protobuf error: %s" % e) return False
def testCacheProtobuf(self): """Tries to cache an encoded protocol buffer.""" from google.appengine.ext import db class MyModel(db.Model): name = db.StringProperty() entity = MyModel(name="foobar") os.environ['APPLICATION_ID'] = 'app' memcache.set('protobuf', db.model_to_protobuf(entity).Encode()) encoded_entity = memcache.get('protobuf') cached_entity = db.model_from_protobuf(encoded_entity) assert cached_entity.name == 'foobar'
def unserialize(data, _search_class=None): """ Get and deserializes the data from the cache, return a instance of the model """ if data is None: return None if _search_class is not None: if type(data) == type(list): return [unserialize(x, search_class=_search_class) for x in data]#data contains a list of objects return _search_class.from_entity(search.SearchableEntity.FromPb(data)) if isinstance(data, basestring): return db.model_from_protobuf(datastore.entity_pb.EntityProto(data)) elif isinstance(data, list): return [unserialize (x) for x in data] elif isinstance(data, dict): return dict([(k, unserialize(x)) for k, x in data.items()])