def serialize_models(models):
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(models):
if models is None:
return None
elif isinstance(models, db.Model):
return db.model_to_protobuf(models).Encode()
else:
return [db.model_to_protobuf(x).Encode() for x in models]
def put(self, keyname, value, session):
"""
Insert a keyname/value pair into the datastore for the session.
Args:
keyname: The keyname of the mapping.
value: The value of the mapping.
Returns the model entity key
"""
keyname = session._validate_key(keyname)
if value is None:
raise ValueError(u"You must pass a value to put.")
sessdata = session._get(keyname=keyname)
if sessdata is None:
sessdata = _AppEngineUtilities_SessionData()
# sessdata.session_key = session.session.key()
sessdata.keyname = keyname
try:
db.model_to_protobuf(value)
if not value.is_saved():
value.put()
sessdata.model = value
except:
sessdata.content = pickle.dumps(value)
sessdata.model = None
sessdata.session = session.session
session.cache[keyname] = value
return sessdata.put()
def put(self, keyname, value, session):
"""
Insert a keyname/value pair into the datastore for the session.
Args:
keyname: The keyname of the mapping.
value: The value of the mapping.
Returns the model entity key
"""
keyname = session._validate_key(keyname)
if value is None:
raise ValueError(u"You must pass a value to put.")
sessdata = session._get(keyname=keyname)
if sessdata is None:
sessdata = _AppEngineUtilities_SessionData()
# sessdata.session_key = session.session.key()
sessdata.keyname = keyname
try:
db.model_to_protobuf(value)
if not value.is_saved():
value.put()
sessdata.model = value
except:
sessdata.content = pickle.dumps(value)
sessdata.model = None
sessdata.session = session.session
session.cache[keyname] = value
return sessdata.put()
def test_put_entities(self):
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
db_batch = flexmock()
db_batch.should_receive("batch_delete").and_return(None)
db_batch.should_receive("batch_put_entity").and_return(None)
db_batch.should_receive("batch_get_entity").and_return({
"key1": {},
"key2": {}
})
zookeeper = flexmock()
zookeeper.should_receive("acquire_lock").and_return(True)
zookeeper.should_receive("release_lock").and_return(True)
dd = DatastoreDistributed(db_batch, zookeeper)
dd.put_entities("hello", [key1, key2], {})
db_batch = flexmock()
db_batch.should_receive("batch_delete").and_return(None)
db_batch.should_receive("batch_put_entity").and_return(None)
db_batch.should_receive("batch_get_entity").and_return({
"key1": {
"entity": key1.Encode()
},
"key2": {
"entity": key2.Encode()
}
})
dd = DatastoreDistributed(db_batch, zookeeper)
dd.put_entities("hello", [key1, key2], {})
def serialize_models(models):
"""Serialize a list of models efficiently for memcached."""
if models is None:
return None
elif isinstance(models, db.Model):
return db.model_to_protobuf(models).Encode()
else:
return [db.model_to_protobuf(x).Encode() for x in models]
def test_get_index_kv_from_tuple(self):
dd = DatastoreDistributed(None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a/b",key1),("a/b",key2)]
assert dd.get_index_kv_from_tuple(tuples_list) == (['a/b/Item/name/Bob\x00/Item:Bob!', 'a/b/Item:Bob!'], ['a/b/Item/name/Sally\x00/Item:Sally!', 'a/b/Item:Sally!'])
def serialize_entities(self, models):
# serialize using protocol buffer.
if not models:
return None
elif isinstance(models, db.Model):
return db.model_to_protobuf(models).Encode()
else:
return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(self, models):
# serialize using protocol buffer.
if not models:
return None
elif isinstance(models, db.Model):
return db.model_to_protobuf(models).Encode()
else:
return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(models):
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
else:
# A list
return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(models):
"""Encode/Decode to Protocol Buffers before/after pickling.
from http://blog.notdot.net/2009/9/Efficient-model-memcaching"""
if models is None:
return None
elif isinstance(models, db.Model): # Just one instance
return db.model_to_protobuf(models).Encode()
else: # A list
return [db.model_to_protobuf(x).Encode() for x in models]
def test_insert_index_entries(self):
db_batch = flexmock()
db_batch.should_receive("batch_put_entity").and_return(None)
dd = DatastoreDistributed(db_batch)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
dd.insert_index_entries([key1,key2])
def serialize_entities(self, models):
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
else:
# A list of models
return [db.model_to_protobuf(x).Encode() for x in models]
def test_insert_index_entries(self):
db_batch = flexmock()
db_batch.should_receive("batch_put_entity").and_return(None)
dd = DatastoreDistributed(db_batch, self.get_zookeeper())
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
dd.insert_index_entries([key1, key2])
def pack(models): if models is None: return None elif isinstance(models, db.Model): # Just one instance return db.model_to_protobuf(models).Encode() else: # A list return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(models):
"""Encode/Decode to Protocol Buffers before/after pickling.
from http://blog.notdot.net/2009/9/Efficient-model-memcaching"""
if models is None:
return None
elif isinstance(models, db.Model):# Just one instance
return db.model_to_protobuf(models).Encode()
else:# A list
return [db.model_to_protobuf(x).Encode() for x in models]
def test_insert_entities(self):
db_batch = flexmock()
db_batch.should_receive("batch_put_entity").and_return(None)
dd = DatastoreDistributed(db_batch, self.get_zookeeper())
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
dd.insert_entities([key1,key2], {"hello\x00\x00Item:Sally!": 1, "hello\x00\x00Item:Bob!":1})
def test_get_index_kv_from_tuple(self):
dd = DatastoreDistributed(None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a/b", key1), ("a/b", key2)]
assert dd.get_index_kv_from_tuple(tuples_list) == ([
'a/b/Item/name/Bob\x00/Item:Bob!', 'a/b/Item:Bob!'
], ['a/b/Item/name/Sally\x00/Item:Sally!', 'a/b/Item:Sally!'])
def test_get_index_kv_from_tuple(self):
dd = DatastoreDistributed(None, None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a\x00b",key1),("a\x00b",key2)]
self.assertEquals(dd.get_index_kv_from_tuple(
tuples_list), (['a\x00b\x00Item\x00name\x00\x9aBob\x00\x00Item:Bob\x01', 'a\x00b\x00Item:Bob\x01'],
['a\x00b\x00Item\x00name\x00\x9aSally\x00\x00Item:Sally\x01', 'a\x00b\x00Item:Sally\x01']))
def _serialize(models):
'''Improve memcache performance converting to protobuf'''
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
else:
# A list
return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(models):
# http://blog.notdot.net/2009/9/Efficient-model-memcaching
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
else:
# A list
return [db.model_to_protobuf(x).Encode() for x in models]
def serialize_entities(models):
"""Serializes either a db.Model instance or a list of them as protobufs."""
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
else:
# A list
return [db.model_to_protobuf(item).Encode() for item in models]
def _serialize(models):
'''Improve memcache performance converting to protobuf'''
if models is None:
return None
elif isinstance(models, db.Model):
# Just one instance
return db.model_to_protobuf(models).Encode()
else:
# A list
return [db.model_to_protobuf(x).Encode() for x in models]
def test_get_index_kv_from_tuple(self):
dd = DatastoreDistributed(None, None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a\x00b",key1),("a\x00b",key2)]
self.assertEquals(dd.get_index_kv_from_tuple(
tuples_list), (['a\x00b\x00Item\x00name\x00\x9aBob\x00\x00Item:Bob!', 'a\x00b\x00Item:Bob!'],
['a\x00b\x00Item\x00name\x00\x9aSally\x00\x00Item:Sally!', 'a\x00b\x00Item:Sally!']))
def serialize_instances(instances):
"""
Serializes the data before saving it in the memcache
"""
if instances is None:
return None
elif isinstance(instances, db.Model):
return db.model_to_protobuf(instances).Encode()#instances is only one
return [db.model_to_protobuf(x).Encode() for x in instances]
def testGetIndexKVFromTuple(self):
dd = DatastoreDistributed(None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a/b", key1), ("a/b", key2)]
assert dd.GetIndexKVFromTuple(tuples_list) == (
["a/b/Item/name/Bob\x00/Item:Bob!", "a/b/Item:Bob!"],
["a/b/Item/name/Sally\x00/Item:Sally!", "a/b/Item:Sally!"],
)
def serialize_instances(instances):
"""
Serializes the data before saving it in the memcache
"""
if instances is None:
return None
elif isinstance(instances, search.SearchableModel):
return instances._populate_entity(_entity_class=search.SearchableEntity)._ToPb()
elif isinstance(instances, db.Model):
return db.model_to_protobuf(instances).Encode()#instances is only one
return [db.model_to_protobuf(x).Encode() for x in instances]
def test_get_index_kv_from_tuple(self):
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a\x00b",key1),("a\x00b",key2)]
self.assertEquals(get_index_kv_from_tuple(
tuples_list), (['a\x00b\x00Item\x00name\x00\x9aBob\x01\x01\x00Item:Bob\x01',
'a\x00b\x00Item:Bob\x01'],
['a\x00b\x00Item\x00name\x00\x9aSally\x01\x01\x00Item:Sally\x01',
'a\x00b\x00Item:Sally\x01']))
def serialize_instances(instances):
"""
Serializes the data before saving it in the memcache
"""
if instances is None:
return None
elif isinstance(instances, search.SearchableModel):
return instances._populate_entity(
_entity_class=search.SearchableEntity)._ToPb()
elif isinstance(instances, db.Model):
return db.model_to_protobuf(instances).Encode() #instances is only one
return [db.model_to_protobuf(x).Encode() for x in instances]
def serialize_entities(entities):
"""Given a list of datastore entities, or a single entity, return a string (or list of strings)
that represents that entity. Using protobuf is substantially faster than pickling and leads to
much more compact representations when storing in memcache."""
if entities is None:
return None
elif isinstance(entities, db.Model):
# Just one instance
return db.model_to_protobuf(entities).Encode()
else:
# A list
return [db.model_to_protobuf(x).Encode() for x in entities]
def serialize_entities(entities):
"""Given a list of datastore entities, or a single entity, return a string (or list of strings)
that represents that entity. Using protobuf is substantially faster than pickling and leads to
much more compact representations when storing in memcache."""
if entities is None:
return None
elif isinstance(entities, db.Model):
# Just one instance
return db.model_to_protobuf(entities).Encode()
else:
# A list
return [db.model_to_protobuf(x).Encode() for x in entities]
def testFetchKeys(self):
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
db_batch = flexmock()
db_batch.should_receive("batch_delete").and_return(None)
db_batch.should_receive("batch_put_entity").and_return(None)
db_batch.should_receive("batch_get_entity").and_return(['aaa'])
dd = DatastoreDistributed(db_batch)
assert dd.fetch_keys([key1.key(), key2.key()]) == (['aaa'], ['hello//Item:Bob!', 'hello//Item:Sally!'])
def testKindKey(self):
dd = DatastoreDistributed(None)
item = Item(name="Dyan", _app="hello")
key = db.model_to_protobuf(item)
assert dd.GetKindKey("howdy", key.key().path()) == "howdy/Item:0000000000!"
item1 = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item1)
assert dd.GetKindKey("howdy", key.key().path()) == "howdy/Item:Bob!"
item2 = Item(key_name="Frank", name="Frank", _app="hello", parent=item1)
key = db.model_to_protobuf(item2)
assert dd.GetKindKey("howdy", key.key().path()) == "howdy/Item:Frank!Item:Bob!"
def test_kind_key(self):
item = Item(name="Dyan", _app="hello")
key = db.model_to_protobuf(item)
self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:0000000000\x01")
item1 = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item1)
self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:Bob\x01")
item2 = Item(key_name="Frank", name="Frank", _app="hello", parent = item1)
key = db.model_to_protobuf(item2)
self.assertEquals(get_kind_key("howdy", key.key().path()),
"howdy\x00Item\x01Item:Bob\x01Item:Frank\x01")
def test_get_index_kv_from_tuple(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a\x00b",key1),("a\x00b",key2)]
self.assertEquals(get_index_kv_from_tuple(
tuples_list), (['a\x00b\x00Item\x00name\x00\x9aBob\x01\x01\x00Item:Bob\x01',
'a\x00b\x00Item:Bob\x01'],
['a\x00b\x00Item\x00name\x00\x9aSally\x01\x01\x00Item:Sally\x01',
'a\x00b\x00Item:Sally\x01']))
def test_kind_key(self):
dd = DatastoreDistributed(None, None)
item = Item(name="Dyan", _app="hello")
key = db.model_to_protobuf(item)
self.assertEquals(dd.get_kind_key("howdy", key.key().path()), "howdy\x00Item!Item:0000000000!")
item1 = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item1)
self.assertEquals(dd.get_kind_key("howdy", key.key().path()), "howdy\x00Item!Item:Bob!")
item2 = Item(key_name="Frank", name="Frank", _app="hello", parent = item1)
key = db.model_to_protobuf(item2)
self.assertEquals(dd.get_kind_key("howdy", key.key().path()),
"howdy\x00Item!Item:Bob!Item:Frank!")
def test_get_index_kv_from_tuple(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, None)
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
tuples_list = [("a\x00b",key1),("a\x00b",key2)]
self.assertEquals(dd.get_index_kv_from_tuple(
tuples_list), (['a\x00b\x00Item\x00name\x00\x9aBob\x01\x01\x00Item:Bob\x01',
'a\x00b\x00Item:Bob\x01'],
['a\x00b\x00Item\x00name\x00\x9aSally\x01\x01\x00Item:Sally\x01',
'a\x00b\x00Item:Sally\x01']))
def testFetchKeys(self):
item1 = Item(key_name="Bob", name="Bob", _app="hello")
item2 = Item(key_name="Sally", name="Sally", _app="hello")
key1 = db.model_to_protobuf(item1)
key2 = db.model_to_protobuf(item2)
db_batch = flexmock()
db_batch.should_receive("batch_delete").and_return(None)
db_batch.should_receive("batch_put_entity").and_return(None)
db_batch.should_receive("batch_get_entity").and_return(['aaa'])
dd = DatastoreDistributed(db_batch)
assert dd.fetch_keys([key1.key(), key2.key()]) == (['aaa'], [
'hello//Item:Bob!', 'hello//Item:Sally!'
])
def create_backup_info_file(filename, backup_info):
"""Creates a backup_info_file for the given BackupInformation model."""
info_file = files.open(files.gs.create(filename), 'a', exclusive_lock=True)
try:
with records.RecordsWriter(info_file) as writer:
writer.write('1')
writer.write(db.model_to_protobuf(backup_info).SerializeToString())
for kind_files in backup_info.get_kind_backup_files():
writer.write(db.model_to_protobuf(kind_files).SerializeToString())
finally:
info_file.close(finalize=True)
def create_backup_info_file(filename, backup_info):
"""Creates a backup_info_file for the given BackupInformation model."""
info_file = files.open(files.gs.create(filename), 'a', exclusive_lock=True)
try:
with records.RecordsWriter(info_file) as writer:
writer.write('1')
writer.write(db.model_to_protobuf(backup_info).SerializeToString())
for kind_files in backup_info.get_kind_backup_files():
writer.write(db.model_to_protobuf(kind_files).SerializeToString())
finally:
info_file.close(finalize=True)
def test_get_entity_key(self):
dd = DatastoreDistributed(None, None)
item = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item)
self.assertEquals(str(dd.get_entity_key("howdy",
key.key().path())),
"howdy/Item:Bob!")
def test_configure_namespace(self):
db_batch = flexmock()
db_batch.should_receive("batch_put_entity").and_return(None)
dd = DatastoreDistributed(db_batch)
item = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item)
assert dd.get_table_prefix(key) == "hello/"
def test_get_entity_key(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, None)
item = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item)
self.assertEquals(str(get_entity_key("howdy", key.key().path())), "howdy\x00Item:Bob\x01")
def _fetch_latest_from_datastore(app_version):
"""Get the latest configuration data for this app-version from the datastore.
Args:
app_version: the major version you want configuration data for.
Side Effects:
We populate memcache with whatever we find in the datastore.
Returns:
A config class instance for most recently set options or None if the
query could not complete due to a datastore exception.
"""
rpc = db.create_rpc(deadline=DATASTORE_DEADLINE,
read_policy=db.EVENTUAL_CONSISTENCY)
key = _get_active_config_key(app_version)
config = None
try:
config = Config.get(key, rpc=rpc)
logging.debug('Loaded most recent conf data from datastore.')
except:
logging.warning('Tried but failed to fetch latest conf data from the '
'datastore.')
if config:
memcache.set(app_version,
db.model_to_protobuf(config).Encode(),
namespace=NAMESPACE)
logging.debug('Wrote most recent conf data into memcache.')
return config
def test_kind_key(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, None)
item = Item(name="Dyan", _app="hello")
key = db.model_to_protobuf(item)
self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:0000000000\x01")
item1 = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item1)
self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:Bob\x01")
item2 = Item(key_name="Frank", name="Frank", _app="hello", parent = item1)
key = db.model_to_protobuf(item2)
self.assertEquals(get_kind_key("howdy", key.key().path()),
"howdy\x00Item\x01Item:Bob\x01Item:Frank\x01")
def test_get_entity_kind(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, None)
item = Item(name="Bob", _app="hello")
key = db.model_to_protobuf(item)
self.assertEquals(get_entity_kind(key), "Item")
def createAdmin():
admin = Admin(key_name='admin')
admin.put()
protobuf = db.model_to_protobuf(admin)
memcache.set('admin', protobuf)
return admin
def getAggregatedError(self, project, environment, server, backtraceText, errorHash, message, timestamp):
"""Gets (and updates) the error matching the given report, or None if no matching error is found."""
error = None
key = '%s|%s' % (project, errorHash)
serialized = memcache.get(key)
if serialized:
error = db.model_from_protobuf(serialized)
else:
q = LoggedError.all()
q.filter('project =', project)
q.filter('hash =', errorHash)
q.filter('active =', True)
for possibility in q:
if backtrace.normalizeBacktrace(possibility.backtrace) == backtrace.normalizeBacktrace(backtraceText):
error = possibility
break
if error:
error.count += 1
error.firstOccurrence = min(error.firstOccurrence, timestamp)
if timestamp > error.lastOccurrence:
error.lastOccurrence = timestamp
error.backtrace = backtraceText
error.lastMessage = message[:300]
if environment not in error.environments:
error.environments.append(environment)
if server not in error.servers:
error.servers.append(server)
error.put()
memcache.set(key, db.model_to_protobuf(error))
return error
def push_video_log(self, video_log):
self.video_logs.append(db.model_to_protobuf(video_log).Encode())
if len(self.video_logs) > LastActionCache.CACHED_ACTIONS_PER_TYPE:
self.video_logs = self.video_logs[-1 * LastActionCache.
CACHED_ACTIONS_PER_TYPE:]
self.store()
def set(path, body, content_type, indexed=True, **kwargs):
"""Sets the StaticContent for the provided path.
Args:
path: The path to store the content against.
body: The data to serve for that path.
content_type: The MIME type to serve the content as.
indexed: Index this page in the sitemap?
**kwargs: Additional arguments to be passed to the StaticContent constructor
Returns:
A StaticContent object.
"""
now = datetime.datetime.now().replace(second=0, microsecond=0)
defaults = {
"last_modified": now,
}
defaults.update(kwargs)
content = StaticContent(key_name=path,
body=str(body),
content_type=content_type,
indexed=indexed,
**defaults)
content.put()
memcache.replace(path, db.model_to_protobuf(content).Encode())
try:
eta = now.replace(second=0,
microsecond=0) + datetime.timedelta(seconds=65)
if indexed:
deferred.defer(utils._regenerate_sitemap,
_name='sitemap-%s' % (now.strftime('%Y%m%d%H%M'), ),
_eta=eta)
except (taskqueue.taskqueue.TaskAlreadyExistsError,
taskqueue.taskqueue.TombstonedTaskError), e:
pass
def set(path, body, content_type, indexed=True, **kwargs):
"""Sets the StaticContent for the provided path.
Args:
path: The path to store the content against.
body: The data to serve for that path.
content_type: The MIME type to serve the content as.
indexed: Index this page in the sitemap?
**kwargs: Additional arguments to be passed to the StaticContent constructor
Returns:
A StaticContent object.
"""
content = StaticContent(
key_name=path,
body=body,
content_type=content_type,
indexed=indexed,
**kwargs)
content.put()
memcache.replace(path, db.model_to_protobuf(content).Encode())
try:
now = datetime.datetime.now().replace(second=0, microsecond=0)
eta = now.replace(second=0, microsecond=0) + datetime.timedelta(seconds=65)
if indexed:
deferred.defer(
utils._regenerate_sitemap,
_name='sitemap-%s' % (now.strftime('%Y%m%d%H%M'),),
_eta=eta)
except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError), e:
pass
def test_validate_key(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, None)
item = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item)
dd.validate_key(key.key())
def testStoredEntity(self):
"""Initializes a stored entity instance."""
class MyModel(db.Model):
contents = db.StringProperty()
key = MyModel(contents="Some contents.").save()
entity = db.get(key)
protobuf = db.model_to_protobuf(entity)
stored_entity = typhoonae.redis.datastore_redis_stub._StoredEntity(
protobuf)
self.assertEqual(protobuf, stored_entity.protobuf)
self.assertEqual(
'j\x15j\x04testr\r\x0b\x12\x07MyModel\x18\x01\x0cr\x1e\x1a\x08'
'contents \x00*\x10\x1a\x0eSome contents.\x82\x01\r\x0b\x12\x07'
'MyModel\x18\x01\x0c',
stored_entity.encoded_protobuf)
self.assertEqual({u'contents': u'Some contents.'}, stored_entity.native)
self.assertTrue(
isinstance(
stored_entity.key(),
google.appengine.datastore.entity_pb.Reference))
def add_experiment(self, experiment, alternatives):
if not experiment or not alternatives:
raise Exception(
"Cannot add empty experiment or empty alternatives to BingoCache"
)
self.experiment_models[experiment.name] = experiment
self.experiments[experiment.name] = db.model_to_protobuf(
experiment).Encode()
if not experiment.conversion_name in self.experiment_names_by_conversion_name:
self.experiment_names_by_conversion_name[
experiment.conversion_name] = []
self.experiment_names_by_conversion_name[
experiment.conversion_name].append(experiment.name)
if not experiment.canonical_name in self.experiment_names_by_canonical_name:
self.experiment_names_by_canonical_name[
experiment.canonical_name] = []
self.experiment_names_by_canonical_name[
experiment.canonical_name].append(experiment.name)
for alternative in alternatives:
self.update_alternative(alternative)
self.dirty = True
def test_configure_namespace(self):
db_batch = flexmock()
db_batch.should_receive("batch_put_entity").and_return(None)
dd = DatastoreDistributed(db_batch)
item = Item(key_name="Bob", name="Bob", _app="hello")
key = db.model_to_protobuf(item)
assert dd.get_table_prefix(key) == "hello/"
def put(self,
index,
entity_list,
memincr=memcache.incr,
memset=memcache.set_multi):
"""Enqueue a model instance on this queue.
Does not write to the Datastore.
Args:
index: The work index for this entity.
entity_list: List of work entities to insert into the in-memory queue.
memincr, memset: Used for testing.
Raises:
MemcacheError if the entities were not successfully added.
"""
length_key = self._create_length_key(index)
end = memincr(length_key, len(entity_list), initial_value=0)
if end is None:
raise MemcacheError('Could not increment length key %r' % length_key)
start = end - len(entity_list)
key_map = {}
for number, entity in zip(xrange(start, end), entity_list):
key_map[self._create_index_key(index, number)] = db.model_to_protobuf(
entity)
result = memset(key_map, time=self.expiration_seconds)
if result:
raise MemcacheError('Could not set memcache keys %r' % result)
def put(self,
index,
entity_list,
memincr=memcache.incr,
memset=memcache.set_multi):
"""Enqueue a model instance on this queue.
Does not write to the Datastore.
Args:
index: The work index for this entity.
entity_list: List of work entities to insert into the in-memory queue.
memincr, memset: Used for testing.
Raises:
MemcacheError if the entities were not successfully added.
"""
length_key = self._create_length_key(index)
end = memincr(length_key, len(entity_list), initial_value=0)
if end is None:
raise MemcacheError('Could not increment length key %r' %
length_key)
start = end - len(entity_list)
key_map = {}
for number, entity in zip(xrange(start, end), entity_list):
key_map[self._create_index_key(
index, number)] = db.model_to_protobuf(entity)
result = memset(key_map, time=self.expiration_seconds)
if result:
raise MemcacheError('Could not set memcache keys %r' % result)
def createStats():
stats = Stats(key_name='stats', reviewTimeUnit=30 * 24 * 3600 * 1000)
stats.put()
protobuf = db.model_to_protobuf(stats)
memcache.set('stats', protobuf)
return stats
