def testMultiFuture_Repr(self):
mf = tasklets.MultiFuture('info')
r1 = repr(mf)
mf.putq(1)
r2 = repr(mf)
f2 = Future()
f2.set_result(2)
mf.putq(2)
r3 = repr(mf)
self.ev.run()
r4 = repr(mf)
f3 = Future()
mf.putq(f3)
r5 = repr(mf)
mf.complete()
r6 = repr(mf)
f3.set_result(3)
self.ev.run()
r7 = repr(mf)
for r in r1, r2, r3, r4, r5, r6, r7:
self.assertTrue(
re.match(
r'<MultiFuture [\da-f]+ created by '
r'(testMultiFuture_Repr\(tasklets_test.py:\d+\)|\?) for info; ',
r))
if r is r7:
self.assertTrue('result' in r)
else:
self.assertTrue('pending' in r)
def map_query(self, query, callback, options=None, merge_future=None):
mfut = merge_future
if mfut is None:
mfut = tasklets.MultiFuture('map_query')
@tasklets.tasklet
def helper():
try:
inq = tasklets.SerialQueueFuture()
query.run_to_queue(inq, self._conn, options)
is_ancestor_query = query.ancestor is not None
while True:
try:
batch, i, ent = yield inq.getq()
except EOFError:
break
if isinstance(ent, model.Key):
pass # It was a keys-only query and ent is really a Key.
else:
key = ent._key
if key in self._cache:
hit = self._cache[key]
if hit is not None and hit.key != key:
# The cached entry has been mutated to have a different key.
# That's a false hit. Get rid of it. See issue #13.
del self._cache[key]
if key in self._cache:
# Assume the cache is more up to date.
if self._cache[key] is None:
# This is a weird case. Apparently this entity was
# deleted concurrently with the query. Let's just
# pretend the delete happened first.
logging.info('Conflict: entity %s was deleted', key)
continue
# Replace the entity the callback will see with the one
# from the cache.
if ent != self._cache[key]:
logging.info('Conflict: entity %s was modified', key)
ent = self._cache[key]
else:
# Cache the entity only if this is an ancestor query;
# non-ancestor queries may return stale results, since in
# the HRD these queries are "eventually consistent".
# TODO: Shouldn't we check this before considering cache hits?
if is_ancestor_query and self._use_cache(key, options):
self._cache[key] = ent
if callback is None:
val = ent
else:
# TODO: If the callback raises, log and ignore.
if options is not None and options.produce_cursors:
val = callback(batch, i, ent)
else:
val = callback(ent)
mfut.putq(val)
except Exception, err:
_, _, tb = sys.exc_info()
mfut.set_exception(err, tb)
raise
else:
def testRunToQueue(self): qry = Foo.query() queue = tasklets.MultiFuture() qry.run_to_queue(queue, self.conn).check_success() results = queue.get_result() self.assertEqual(len(results), 3) self.assertEqual(results[0][2], self.joe) self.assertEqual(results[1][2], self.jill) self.assertEqual(results[2][2], self.moe)
def testMultiFuture_PreCompleted(self):
@tasklets.tasklet
def foo():
yield tasklets.sleep(0.01)
raise tasklets.Return(42)
mfut = tasklets.MultiFuture()
dep = foo()
dep.wait()
mfut.add_dependent(dep)
mfut.complete()
eventloop.run()
self.assertTrue(mfut.done())
self.assertEqual(mfut.get_result(), [42])
def map_query(self, query, callback, options=None, merge_future=None):
mfut = merge_future
if mfut is None:
mfut = tasklets.MultiFuture('map_query')
@tasklets.tasklet
def helper():
inq = tasklets.SerialQueueFuture()
query.run_to_queue(inq, self._conn, options)
is_ancestor_query = query.ancestor is not None
while True:
try:
batch, i, ent = yield inq.getq()
except EOFError:
break
if isinstance(ent, model.Key):
pass # It was a keys-only query and ent is really a Key.
else:
key = ent._key
if key in self._cache:
# Assume the cache is more up to date.
if self._cache[key] is None:
# This is a weird case. Apparently this entity was
# deleted concurrently with the query. Let's just
# pretend the delete happened first.
logging.info('Conflict: entity %s was deleted',
key)
continue
# Replace the entity the callback will see with the one
# from the cache.
if ent != self._cache[key]:
logging.info('Conflict: entity %s was modified',
key)
ent = self._cache[key]
else:
if is_ancestor_query and self.should_cache(key):
self._cache[key] = ent
if callback is None:
val = ent
else:
# TODO: If the callback raises, log and ignore.
if options is not None and options.produce_cursors:
val = callback(batch, i, ent)
else:
val = callback(ent)
mfut.putq(val)
mfut.complete()
helper()
return mfut
def testMultiFuture_ItemException(self):
mf = tasklets.MultiFuture()
f1 = Future()
f2 = Future()
f3 = Future()
f2.set_result(2)
mf.putq(f1)
f1.set_exception(ZeroDivisionError())
mf.putq(f2)
mf.putq(f3)
f3.set_result(3)
self.ev.run()
mf.complete()
self.assertRaises(ZeroDivisionError, mf.get_result)
def testMultiFuture(self):
@tasklets.tasklet
def foo(dt):
yield tasklets.sleep(dt)
raise tasklets.Return('foo-%s' % dt)
@tasklets.tasklet
def bar(n):
for i in range(n):
yield tasklets.sleep(0.01)
raise tasklets.Return('bar-%d' % n)
bar5 = bar(5)
futs = [foo(0.05), foo(0.01), foo(0.03), bar(3), bar5, bar5]
mfut = tasklets.MultiFuture()
for fut in futs:
mfut.add_dependent(fut)
mfut.complete()
results = mfut.get_result()
self.assertEqual(set(results),
set(['foo-0.01', 'foo-0.03', 'foo-0.05',
'bar-3', 'bar-5']))
def testRunToQueueError(self): qry = Foo.query(Foo.name > '', Foo.rate > 0) queue = tasklets.MultiFuture() fut = qry.run_to_queue(queue, self.conn) self.assertRaises(datastore_errors.BadRequestError, fut.check_success) self.assertRaises(datastore_errors.BadRequestError, queue.check_success)
