def get_by_sid(cls, sid):
"""Returns a ``Session`` instance by session id.
:param sid:
A session id.
:returns:
An existing ``Session`` entity.
"""
data = memcache.get(sid)
if not data:
session = model.Key(cls, sid).get()
if session:
data = session.data
memcache.set(sid, data)
return data
def get_key(cls, user, subject, token):
"""Returns a token key.
:param user:
User unique ID.
:param subject:
The subject of the key. Examples:
- 'auth'
- 'signup'
:param token:
Randomly generated token.
:returns:
``model.Key`` containing a string id in the following format:
``{user_id}.{subject}.{token}.``
"""
return model.Key(cls, '%s.%s.%s' % (str(user), subject, token))
def testContext_TransactionRollback(self):
key = model.Key('Foo', 1)
@tasklets.tasklet
def foo():
ent = model.Expando(key=key, bar=1)
@tasklets.tasklet
def callback():
ctx = tasklets.get_context()
key = yield ent.put_async()
raise model.Rollback()
yield self.ctx.transaction(callback)
foo().check_success()
self.assertEqual(key.get(), None)
def create(cls, value):
"""Creates a new unique value.
:param value:
The value to be unique, as a string.
The value should include the scope in which the value must be
unique (ancestor, namespace, kind and/or property name).
For example, for a unique property `email` from kind `User`, the
value can be `User.email:[email protected]`. In this case `User.email`
is the scope, and `[email protected]` is the value to be unique.
:returns:
True if the unique value was created, False otherwise.
"""
entity = cls(key=model.Key(cls, value))
txn = lambda: entity.put() if not entity.key.get() else None
return model.transaction(txn) is not None
def memoizing_fibonacci(n):
"""A memoizing recursive Fibonacci to exercise RPCs."""
if n <= 1:
raise tasklets.Return(n)
key = model.Key(FibonacciMemo, str(n))
memo = yield key.get_async(ndb_should_cache=False)
if memo is not None:
assert memo.arg == n
logging.info('memo hit: %d -> %d', n, memo.value)
raise tasklets.Return(memo.value)
logging.info('memo fail: %d', n)
a = yield memoizing_fibonacci(n - 1)
b = yield memoizing_fibonacci(n - 2)
ans = a + b
memo = FibonacciMemo(key=key, arg=n, value=ans)
logging.info('memo write: %d -> %d', n, memo.value)
yield memo.put_async(ndb_should_cache=False)
raise tasklets.Return(ans)
def testUnindexedProperty(self):
class MyModel(model.Model):
t = model.TextProperty()
b = model.BlobProperty()
ent = MyModel()
MyModel.t._set_value(ent, u'Hello world\u1234')
MyModel.b._set_value(ent, '\x00\xff')
self.assertEqual(MyModel.t._get_value(ent), u'Hello world\u1234')
self.assertEqual(MyModel.b._get_value(ent), '\x00\xff')
pb = ent._to_pb()
self.assertEqual(str(pb), UNINDEXED_PB)
ent = MyModel._from_pb(pb)
self.assertEqual(ent._get_kind(), 'MyModel')
k = model.Key(flat=['MyModel', None])
self.assertEqual(ent.key, k)
self.assertEqual(MyModel.t._get_value(ent), u'Hello world\u1234')
self.assertEqual(MyModel.b._get_value(ent), '\x00\xff')
def testValidation(self):
class All(model.Model):
s = model.StringProperty()
i = model.IntegerProperty()
f = model.FloatProperty()
t = model.TextProperty()
b = model.BlobProperty()
k = model.KeyProperty()
BVE = datastore_errors.BadValueError
a = All()
a.s = None
a.s = 'abc'
a.s = u'def'
a.s = '\xff' # Not UTF-8.
self.assertRaises(BVE, setattr, a, 's', 0)
a.i = None
a.i = 42
a.i = 123L
self.assertRaises(BVE, setattr, a, 'i', '')
a.f = None
a.f = 42
a.f = 3.14
self.assertRaises(BVE, setattr, a, 'f', '')
a.t = None
a.t = 'abc'
a.t = u'def'
a.t = '\xff' # Not UTF-8.
self.assertRaises(BVE, setattr, a, 't', 0)
a.b = None
a.b = 'abc'
a.b = '\xff'
self.assertRaises(BVE, setattr, a, 'b', u'')
self.assertRaises(BVE, setattr, a, 'b', u'')
a.k = None
a.k = model.Key('Foo', 42)
self.assertRaises(BVE, setattr, a, 'k', '')
def testModelRepr(self):
class Address(model.Model):
street = model.StringProperty()
city = model.StringProperty()
class Person(model.Model):
name = model.StringProperty()
address = model.StructuredProperty(Address)
p = Person(name='Google',
address=Address(street='345 Spear', city='SF'))
self.assertEqual(
repr(p),
"Person(address=Address(city='SF', street='345 Spear'), name='Google')"
)
p.key = model.Key(pairs=[('Person', 42)])
self.assertEqual(
repr(p), "Person(key=Key('Person', 42), "
"address=Address(city='SF', street='345 Spear'), name='Google')")
def _hp_callback(self, message):
nickname = 'Anonymous'
if message.userid:
nickname = yield get_nickname(message.userid)
# Check if there's an URL.
body = message.body
m = re.search(r'(?i)\bhttps?://\S+[^\s.,;\]\}\)]', body)
if not m:
escbody = cgi.escape(body)
else:
url = m.group()
pre = body[:m.start()]
post = body[m.end():]
title = ''
key = model.Key(flat=[UrlSummary.GetKind(), url])
summary = yield key.get_async()
if not summary or summary.when < time.time() - UrlSummary.MAX_AGE:
rpc = urlfetch.create_rpc(deadline=0.5)
urlfetch.make_fetch_call(rpc, url, allow_truncated=True)
t0 = time.time()
result = yield rpc
t1 = time.time()
logging.warning('url=%r, status=%r, dt=%.3f', url,
result.status_code, t1 - t0)
if result.status_code == 200:
bodytext = result.content
m = re.search(r'(?i)<title>([^<]+)</title>', bodytext)
if m:
title = m.group(1).strip()
summary = UrlSummary(key=key,
url=url,
title=title,
when=time.time())
yield summary.put_async()
hover = ''
if summary.title:
hover = ' title="%s"' % summary.title
escbody = (cgi.escape(pre) + '<a%s href="%s">' %
(hover, cgi.escape(url)) + cgi.escape(url) + '</a>' +
cgi.escape(post))
text = '%s - %s - %s<br>' % (cgi.escape(nickname),
time.ctime(message.when), escbody)
raise tasklets.Return((-message.when, text))
def testMultipleInStructuredProperty(self):
class Address(model.Model):
label = model.StringProperty()
line = model.StringProperty(repeated=True)
class Person(model.Model):
name = model.StringProperty()
address = model.StructuredProperty(Address)
m = Person(name='Google',
address=Address(label='work',
line=['345 Spear', 'San Francisco']))
m.key = model.Key(flat=['Person', None])
self.assertEqual(m.address.line, ['345 Spear', 'San Francisco'])
pb = m._to_pb()
self.assertEqual(str(pb), MULTIINSTRUCT_PB)
m2 = Person._from_pb(pb)
self.assertEqual(m2, m)
def get_by_refresh_token(cls, user_id, token):
"""Returns a user object based on a user ID and token.
:param user_id:
The user_id of the requesting user.
:param token:
The token string to be verified.
:returns:
A tuple ``(User, timestamp)``, with a user object and
the token timestamp, or ``(None, None)`` if both were not found.
"""
token_key = cls.token_model.get_key(user_id, 'refresh', token)
user_key = model.Key(cls, user_id)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
timestamp = int(time.mktime(valid_token.created.timetuple()))
return user, timestamp
return None, None
def testContext_CachePolicyDisabledLater(self):
# If the cache is disabled after an entity is stored in the cache,
# further get() attempts *must not* return the result stored in cache.
self.ctx.set_cache_policy(lambda key: True)
key1 = model.Key(flat=('Foo', 1))
ent1 = model.Expando(key=key1)
self.ctx.put(ent1).get_result()
# get() uses cache
self.assertTrue(key1 in self.ctx._cache) # Whitebox.
self.assertEqual(self.ctx.get(key1).get_result(), ent1)
# get() uses cache
self.ctx._cache[key1] = None # Whitebox.
self.assertEqual(self.ctx.get(key1).get_result(), None)
# get() doesn't use cache
self.ctx.set_cache_policy(lambda key: False)
self.assertEqual(self.ctx.get(key1).get_result(), ent1)
def get_or_insert(self, model_class, name,
app=None, namespace=None, parent=None,
context_options=None,
**kwds):
# TODO: Test the heck out of this, in all sorts of evil scenarios.
assert isinstance(name, basestring) and name
key = model.Key(model_class, name,
app=app, namespace=namespace, parent=parent)
# TODO: Can (and should) the cache be trusted here?
ent = yield self.get(key)
if ent is None:
@tasklets.tasklet
def txn():
ent = yield key.get_async(options=context_options)
if ent is None:
ent = model_class(**kwds) # TODO: Check for forbidden keys
ent._key = key
yield ent.put_async(options=context_options)
raise tasklets.Return(ent)
ent = yield self.transaction(txn)
raise tasklets.Return(ent)
def testMultipleStructuredProperty(self):
class Address(model.Model):
label = model.StringProperty()
text = model.StringProperty()
class Person(model.Model):
name = model.StringProperty()
address = model.StructuredProperty(Address, repeated=True)
m = Person(name='Google',
address=[
Address(label='work', text='San Francisco'),
Address(label='home', text='Mountain View')
])
m.key = model.Key(flat=['Person', None])
self.assertEqual(m.address[0].label, 'work')
self.assertEqual(m.address[0].text, 'San Francisco')
self.assertEqual(m.address[1].label, 'home')
self.assertEqual(m.address[1].text, 'Mountain View')
[k] = self.conn.put([m])
m.key = k # Connection.put() doesn't do this!
[m2] = self.conn.get([k])
self.assertEqual(m2, m)
def testMultipleStructuredProperty(self):
class Address(model.Model):
label = model.StringProperty()
text = model.StringProperty()
class Person(model.Model):
name = model.StringProperty()
address = model.StructuredProperty(Address, repeated=True)
m = Person(name='Google',
address=[
Address(label='work', text='San Francisco'),
Address(label='home', text='Mountain View')
])
m.key = model.Key(flat=['Person', None])
self.assertEqual(m.address[0].label, 'work')
self.assertEqual(m.address[0].text, 'San Francisco')
self.assertEqual(m.address[1].label, 'home')
self.assertEqual(m.address[1].text, 'Mountain View')
pb = m._to_pb()
self.assertEqual(str(pb), MULTISTRUCT_PB)
m2 = Person._from_pb(pb)
self.assertEqual(m2, m)
def testRecursiveStructuredProperty(self):
class Node(model.Model):
name = model.StringProperty(indexed=False)
Node.left = model.StructuredProperty(Node)
Node.rite = model.StructuredProperty(Node)
Node._fix_up_properties()
class Tree(model.Model):
root = model.StructuredProperty(Node)
k = model.Key(flat=['Tree', None])
tree = Tree()
tree.key = k
tree.root = Node(name='a',
left=Node(name='a1',
left=Node(name='a1a'),
rite=Node(name='a1b')),
rite=Node(name='a2', rite=Node(name='a2b')))
pb = tree._to_pb()
self.assertEqual(str(pb), RECURSIVE_PB)
tree2 = Tree._from_pb(pb)
self.assertEqual(tree2, tree)
def _args_to_val(func, args, bindings):
vals = []
for arg in args:
if isinstance(arg, (int, long, basestring)):
if arg in bindings:
val = bindings[arg]
else:
val = Binding(None, arg)
bindings[arg] = val
elif isinstance(arg, gql.Literal):
val = arg.Get()
else:
assert False, 'Unexpected arg (%r)' % arg
vals.append(val)
if func == 'nop':
assert len(vals) == 1
return vals[0]
if func == 'list':
return vals
if func == 'key':
if len(vals) == 1 and isinstance(vals[0], basestring):
return model.Key(urlsafe=vals[0])
assert False, 'Unexpected key args (%r)' % (vals, )
assert False, 'Unexpected func (%r)' % func
def NastyCallback(self, rpc):
[ent] = rpc.get_result()
key = model.Key(flat=['Expando', 1])
newrpc = self.conn.async_get(None, [key])
def delete_by_id(cls, user_id):
user_key = model.Key(cls, user_id)
# Use delete_multi() to save a RPC call.
model.delete_multi([user_key])
return True
def account_key(userid):
return model.Key(flat=['Account', userid])
def testBasicSetup2(self):
key = model.Key(flat=['Expando', 1])
rpc = self.conn.async_get(None, [key])
[ent] = rpc.get_result()
self.assertTrue(ent is None)
def foo():
# Foo class is declared in query_test, so let's get a unusual class name.
key1 = model.Key(flat=('ThisModelClassDoesntExist', 1))
ent1 = model.Expando(key=key1, foo=42, bar='hello')
key = yield ctx.put(ent1)
a = yield ctx.get(key1)
def create(cls, value):
entity = cls(key=model.Key(cls, value))
txn = lambda: entity.put() if not entity.key.get() else None
return model.transaction(txn) is not None
class ContextTests(test_utils.DatastoreTest):
def setUp(self):
super(ContextTests, self).setUp()
self.set_up_eventloop()
MyAutoBatcher.reset_log()
self.ctx = context.Context(
conn=model.make_connection(default_model=model.Expando),
auto_batcher_class=MyAutoBatcher)
def set_up_eventloop(self):
if eventloop._EVENT_LOOP_KEY in os.environ:
del os.environ[eventloop._EVENT_LOOP_KEY]
self.ev = eventloop.get_event_loop()
self.log = []
def testContext_AutoBatcher_Get(self):
@tasklets.tasklet
def foo():
key1 = model.Key(flat=['Foo', 1])
key2 = model.Key(flat=['Foo', 2])
key3 = model.Key(flat=['Foo', 3])
fut1 = self.ctx.get(key1)
fut2 = self.ctx.get(key2)
fut3 = self.ctx.get(key3)
ent1 = yield fut1
ent2 = yield fut2
ent3 = yield fut3
raise tasklets.Return([ent1, ent2, ent3])
ents = foo().get_result()
self.assertEqual(ents, [None, None, None])
self.assertEqual(len(MyAutoBatcher._log), 1)
@tasklets.tasklet
def create_entities(self):
key0 = model.Key(flat=['Foo', None])
ent1 = model.Model(key=key0)
ent2 = model.Model(key=key0)
ent3 = model.Model(key=key0)
fut1 = self.ctx.put(ent1)
fut2 = self.ctx.put(ent2)
fut3 = self.ctx.put(ent3)
key1 = yield fut1
key2 = yield fut2
key3 = yield fut3
raise tasklets.Return([key1, key2, key3])
def testContext_AutoBatcher_Put(self):
keys = self.create_entities().get_result()
self.assertEqual(len(keys), 3)
self.assertTrue(None not in keys)
self.assertEqual(len(MyAutoBatcher._log), 1)
def testContext_AutoBatcher_Delete(self):
@tasklets.tasklet
def foo():
key1 = model.Key(flat=['Foo', 1])
key2 = model.Key(flat=['Foo', 2])
key3 = model.Key(flat=['Foo', 3])
fut1 = self.ctx.delete(key1)
fut2 = self.ctx.delete(key2)
fut3 = self.ctx.delete(key3)
yield fut1
yield fut2
yield fut3
foo().check_success()
self.assertEqual(len(MyAutoBatcher._log), 1)
def testContext_MultiRpc(self):
# This test really tests the proper handling of MultiRpc by
# queue_rpc() in eventloop.py. It's easier to test from here, and
# gives more assurance that it works.
config = datastore_rpc.Configuration(max_get_keys=3,
max_put_entities=3)
self.ctx._conn = model.make_connection(config,
default_model=model.Expando)
@tasklets.tasklet
def foo():
ents = [model.Expando() for i in range(10)]
futs = [self.ctx.put(ent) for ent in ents]
keys = yield futs
futs = [self.ctx.get(key) for key in keys]
ents2 = yield futs
self.assertEqual(ents2, ents)
raise tasklets.Return(keys)
keys = foo().get_result()
print keys
self.assertEqual(len(keys), 10)
def testContext_Cache(self):
@tasklets.tasklet
def foo():
key1 = model.Key(flat=('Foo', 1))
ent1 = model.Expando(key=key1, foo=42, bar='hello')
key = yield self.ctx.put(ent1)
self.assertTrue(key1 in self.ctx._cache) # Whitebox.
a = yield self.ctx.get(key1)
b = yield self.ctx.get(key1)
self.assertTrue(a is b)
yield self.ctx.delete(key1)
self.assertTrue(self.ctx._cache[key] is None) # Whitebox.
a = yield self.ctx.get(key1)
self.assertTrue(a is None)
foo().check_success()
def testContext_CachePolicy(self):
def should_cache(key):
return False
@tasklets.tasklet
def foo():
key1 = model.Key(flat=('Foo', 1))
ent1 = model.Expando(key=key1, foo=42, bar='hello')
key = yield self.ctx.put(ent1)
self.assertTrue(key1 not in self.ctx._cache) # Whitebox.
a = yield self.ctx.get(key1)
b = yield self.ctx.get(key1)
self.assertTrue(a is not b)
yield self.ctx.delete(key1)
self.assertTrue(key not in self.ctx._cache) # Whitebox.
a = yield self.ctx.get(key1)
self.assertTrue(a is None)
self.ctx.set_cache_policy(should_cache)
foo().check_success()
def testContext_CachePolicyDisabledLater(self):
# If the cache is disabled after an entity is stored in the cache,
# further get() attempts *must not* return the result stored in cache.
self.ctx.set_cache_policy(lambda key: True)
key1 = model.Key(flat=('Foo', 1))
ent1 = model.Expando(key=key1)
self.ctx.put(ent1).get_result()
# get() uses cache
self.assertTrue(key1 in self.ctx._cache) # Whitebox.
self.assertEqual(self.ctx.get(key1).get_result(), ent1)
# get() uses cache
self.ctx._cache[key1] = None # Whitebox.
self.assertEqual(self.ctx.get(key1).get_result(), None)
# get() doesn't use cache
self.ctx.set_cache_policy(lambda key: False)
self.assertEqual(self.ctx.get(key1).get_result(), ent1)
def testContext_Memcache(self):
@tasklets.tasklet
def foo():
key1 = model.Key(flat=('Foo', 1))
key2 = model.Key(flat=('Foo', 2))
ent1 = model.Expando(key=key1, foo=42, bar='hello')
ent2 = model.Expando(key=key2, foo=1, bar='world')
k1, k2 = yield self.ctx.put(ent1), self.ctx.put(ent2)
self.assertEqual(k1, key1)
self.assertEqual(k2, key2)
yield tasklets.sleep(0.01) # Let other tasklet complete.
keys = [k1.urlsafe(), k2.urlsafe()]
results = memcache.get_multi(keys, key_prefix='NDB:')
self.assertEqual(
results, {
key1.urlsafe(): self.ctx._conn.adapter.entity_to_pb(ent1),
key2.urlsafe(): self.ctx._conn.adapter.entity_to_pb(ent2)
})
foo().check_success()
def testContext_MemcachePolicy(self):
badkeys = []
def tracking_set_multi(*args, **kwds):
try:
res = save_set_multi(*args, **kwds)
if badkeys and not res:
res = badkeys
track.append((args, kwds, res, None))
return res
except Exception, err:
track.append((args, kwds, None, err))
raise
@tasklets.tasklet
def foo():
k1, k2 = yield self.ctx.put(ent1), self.ctx.put(ent2)
self.assertEqual(k1, key1)
self.assertEqual(k2, key2)
yield tasklets.sleep(0.01) # Let other tasklet complete.
key1 = model.Key('Foo', 1)
key2 = model.Key('Foo', 2)
ent1 = model.Expando(key=key1, foo=42, bar='hello')
ent2 = model.Expando(key=key2, foo=1, bar='world')
save_set_multi = memcache.set_multi
try:
memcache.set_multi = tracking_set_multi
memcache.flush_all()
track = []
foo().check_success()
self.assertEqual(len(track), 1)
self.assertEqual(track[0][0], ({
key1.urlsafe(): ent1._to_pb(),
key2.urlsafe(): ent2._to_pb()
}, ))
self.assertEqual(track[0][1], {'key_prefix': 'NDB:', 'time': 0})
memcache.flush_all()
track = []
self.ctx.set_memcache_policy(lambda key: False)
foo().check_success()
self.assertEqual(len(track), 0)
memcache.flush_all()
track = []
self.ctx.set_memcache_policy(lambda key: key == key1)
foo().check_success()
self.assertEqual(len(track), 1)
self.assertEqual(track[0][0], ({key1.urlsafe(): ent1._to_pb()}, ))
self.assertEqual(track[0][1], {'key_prefix': 'NDB:', 'time': 0})
memcache.flush_all()
track = []
self.ctx.set_memcache_policy(lambda key: True)
self.ctx.set_memcache_timeout_policy(lambda key: key.id())
foo().check_success()
self.assertEqual(len(track), 2)
self.assertEqual(track[0][0], ({key1.urlsafe(): ent1._to_pb()}, ))
self.assertEqual(track[0][1], {'key_prefix': 'NDB:', 'time': 1})
self.assertEqual(track[1][0], ({key2.urlsafe(): ent2._to_pb()}, ))
self.assertEqual(track[1][1], {'key_prefix': 'NDB:', 'time': 2})
memcache.flush_all()
track = []
badkeys = [key2.urlsafe()]
self.ctx.set_memcache_timeout_policy(lambda key: 0)
foo().check_success()
self.assertEqual(len(track), 1)
self.assertEqual(track[0][2], badkeys)
memcache.flush_all()
finally:
memcache.set_multi = save_set_multi
def foo():
key = model.Key(flat=('Foo', 1))
lo_hi = yield self.ctx.allocate_ids(key, size=10)
self.assertEqual(lo_hi, (1, 10))
lo_hi = yield self.ctx.allocate_ids(key, max=20)
self.assertEqual(lo_hi, (11, 20))
def testBasicSetup1(self):
ent = model.Expando()
ent.foo = 'bar'
rpc = self.conn.async_put(None, [ent])
[key] = rpc.get_result()
self.assertEqual(key, model.Key(flat=['Expando', 1]))
def testIdAndParent(self):
p = model.Key('ParentModel', 'foo')
# key name
m = model.Model(id='bar')
m2 = model.Model._from_pb(m._to_pb())
self.assertEqual(m2.key, model.Key('Model', 'bar'))
# key name + parent
m = model.Model(id='bar', parent=p)
m2 = model.Model._from_pb(m._to_pb())
self.assertEqual(m2.key, model.Key('ParentModel', 'foo', 'Model',
'bar'))
# key id
m = model.Model(id=42)
m2 = model.Model._from_pb(m._to_pb())
self.assertEqual(m2.key, model.Key('Model', 42))
# key id + parent
m = model.Model(id=42, parent=p)
m2 = model.Model._from_pb(m._to_pb())
self.assertEqual(m2.key, model.Key('ParentModel', 'foo', 'Model', 42))
# parent
m = model.Model(parent=p)
m2 = model.Model._from_pb(m._to_pb())
self.assertEqual(m2.key, model.Key('ParentModel', 'foo', 'Model',
None))
# not key -- invalid
self.assertRaises(datastore_errors.BadValueError,
model.Model,
key='foo')
# wrong key kind -- invalid
k = model.Key('OtherModel', 'bar')
self.assertRaises(model.KindError, model.Model, key=k)
# incomplete parent -- invalid
p2 = model.Key('ParentModel', None)
self.assertRaises(datastore_errors.BadArgumentError,
model.Model,
parent=p2)
self.assertRaises(datastore_errors.BadArgumentError,
model.Model,
id='bar',
parent=p2)
# key + id -- invalid
k = model.Key('Model', 'bar')
self.assertRaises(datastore_errors.BadArgumentError,
model.Model,
key=k,
id='bar')
# key + parent -- invalid
k = model.Key('Model', 'bar', parent=p)
self.assertRaises(datastore_errors.BadArgumentError,
model.Model,
key=k,
parent=p)
# key + id + parent -- invalid
self.assertRaises(datastore_errors.BadArgumentError,
model.Model,
key=k,
id='bar',
parent=p)
def subverting_aries_fix():
# Variation by Guido van Rossum.
def setup_context():
ctx = tasklets.get_context()
ctx.set_datastore_policy(True)
ctx.set_memcache_policy(True)
ctx.set_cache_policy(False)
return ctx
key = model.Key(CrashTestDummyModel, 1)
mkey = tasklets.get_context()._memcache_prefix + key.urlsafe()
ent1 = CrashTestDummyModel(key=key, name=u'Brad Roberts')
ent2 = CrashTestDummyModel(key=key, name=u'Ellen Reid')
ctx = setup_context()
# Store an original version of the entity
# NOTE: Do not wish to store this one value in memcache, turning it off
ent1.put(use_memcache=False)
a_lock1 = threading.Lock()
a_lock2 = threading.Lock()
a_lock3 = threading.Lock()
class A(threading.Thread):
def run(self):
ctx = setup_context()
fut = ent2.put_async()
# Get to the point that the lock is written to memcache
wait_on_batcher(ctx._memcache_set_batcher)
# Wait for B to cause a race condition
a_lock2.acquire()
a_lock1.acquire()
wait_on_batcher(ctx._put_batcher)
a_lock2.release()
a_lock1.release()
# Wait for C to read from memcache
a_lock3.acquire()
fut.check_success()
a_lock3.release()
class C(threading.Thread):
def run(self):
setup_context()
result = key.get()
assert result == ent2, result
eventloop.run()
logging.info('A: write lock to memcache')
a = A()
a_lock1.acquire()
a_lock3.acquire()
a.start()
while memcache.get(mkey) != context._LOCKED:
time.sleep(0.1) # Wait for the memcache lock to be set
logging.info('M: evict the lock')
memcache.flush_all()
assert memcache.get(mkey) is None, 'lock was not evicted'
logging.info("B: read from memcache (it's a miss)")
b = key.get_async()
wait_on_batcher(ctx._memcache_get_batcher)
logging.info('B: write lock to memcache')
wait_on_batcher(ctx._memcache_set_batcher)
logging.info("B: read the lock back (it's a success)")
wait_on_batcher(ctx._memcache_get_batcher)
logging.info('B: read from datastore')
wait_on_batcher(ctx._get_batcher)
logging.info('A: write to datastore')
a_lock1.release()
a_lock2.acquire()
a_lock2.release()
logging.info('B: write to memcache (writes a stale value)')
b.get_result()
eventloop.run() # Puts to memcache are still stuck in the eventloop
logging.info('C: read from memcache (sees a stale value)')
c = C()
c.start()
c.join()
logging.info('A: delete from memcache (deletes the stale value!)')
a_lock3.release()
a.join()
pb3 = memcache.get(mkey)
assert pb3 is not context._LOCKED, 'Received _LOCKED value'
if pb3 is not None:
ent3 = ctx._conn.adapter.pb_to_entity(pb3)
assert ent3 == ent2, 'stale value in memcache; %r != %r' % (ent3, ent2)
# Finally check the high-level API.
ent4 = key.get()
assert ent4 == ent2
def foo():
key1 = model.Key(flat=('Foo', 1))
ent1 = model.Expando(key=key1, foo=42, bar='hello')
key = yield ctx.put(ent1)
a = yield ctx.get(key1)
def foo():
parent = model.Key(flat=('Foo', 1))
ent = yield self.ctx.get_or_insert(Mod, 'a', parent=parent, data='hello')
assert isinstance(ent, Mod)
ent2 = yield self.ctx.get_or_insert(Mod, 'a', parent=parent, data='hello')
assert ent2 == ent
