def _test_spawn_and_join(self):
pair = self.make_cat_pair()
requester = partial(self.requester_with_join, pair)
fetcher = DataFetcher()
result = fetcher.spawn_and_run(requester)
self.check_cat_pair_fetched(pair, result, fetcher)
self.assert_stats(2, 2, 2)
def _test_spawn_and_join(self):
pair = self.make_cat_pair()
requester = partial(self.requester_with_join, pair)
fetcher = DataFetcher()
result = fetcher.spawn_and_run(requester)
self.check_cat_pair_fetched(pair, result, fetcher)
self.assert_stats(2, 2, 2)
def _test_inner_spawn(self):
cat, friend = pair = self.make_cat_pair()
def requester(fetcher):
fetch_inner = partial(self.request_cat_and_friend, cat)
fetcher.spawn(fetch_inner)
yield
cat1 = fetcher.get_obj(Cat, cat.key().id())
self.assert_cat_equals(cat, cat1)
# Verify that after the first pass, we shouldn't have gotten the
# friend's data
self.assertRaises(KeyError, fetcher.get_obj, Cat,
friend.key().id())
def ondone():
# after the second pass, we should have the friend's data
friend1 = fetcher.get_obj(Cat, friend.key().id())
self.assert_cat_equals(friend, friend1)
return (cat1, friend1)
yield ondone
fetcher = DataFetcher()
result = fetcher.spawn_and_run(requester)
self.check_cat_pair_fetched(pair, result, fetcher)
self.assert_stats(2, 2, 2)
def _test_inner_spawn(self):
cat, friend = pair = self.make_cat_pair()
def requester(fetcher):
fetch_inner = partial(self.request_cat_and_friend, cat)
fetcher.spawn(fetch_inner)
yield
cat1 = fetcher.get_obj(Cat, cat.key().id())
self.assert_cat_equals(cat, cat1)
# Verify that after the first pass, we shouldn't have gotten the
# friend's data
self.assertRaises(KeyError, fetcher.get_obj, Cat, friend.key().id())
def ondone():
# after the second pass, we should have the friend's data
friend1 = fetcher.get_obj(Cat, friend.key().id())
self.assert_cat_equals(friend, friend1)
return (cat1, friend1)
yield ondone
fetcher = DataFetcher()
result = fetcher.spawn_and_run(requester)
self.check_cat_pair_fetched(pair, result, fetcher)
self.assert_stats(2, 2, 2)
def do_fetch():
requester_funcs = get_requester_funcs(cat_pairs)
fetcher = DataFetcher()
if type(requester_funcs) is list:
results = fetcher.spawn_and_run_multi(requester_funcs)
else:
results = fetcher.spawn_and_run(requester_funcs)
return (fetcher, results)
def do_fetch():
requester_funcs = get_requester_funcs(cat_pairs)
fetcher = DataFetcher()
if type(requester_funcs) is list:
results = fetcher.spawn_and_run_multi(requester_funcs)
else:
results = fetcher.spawn_and_run(requester_funcs)
return (fetcher, results)
def _test_already_ran_exception(self):
fetcher = DataFetcher().run()
self.assertRaises(AlreadyRanException, fetcher.run)
def requester(fetcher):
fetcher.run()
fetcher = DataFetcher()
fetcher.spawn(requester)
self.assertRaises(AlreadyRanException, fetcher.run)
def test_spawn_with_ondone_func(self):
pair = cat, friend = self.make_cat_pair()
def requester(fetcher):
requester1 = partial(self.request_cat_and_friend, cat)
ref = fetcher.spawn(requester1)
yield ref.get
fetcher = DataFetcher()
res = fetcher.spawn_and_run(requester)
self.check_cat_pair_fetched(pair, res, fetcher)
def test_spawn_with_ondone_func(self):
pair = cat, friend = self.make_cat_pair()
def requester(fetcher):
requester1 = partial(self.request_cat_and_friend, cat)
ref = fetcher.spawn(requester1)
yield ref.get
fetcher = DataFetcher()
res = fetcher.spawn_and_run(requester)
self.check_cat_pair_fetched(pair, res, fetcher)
def _test_obj_single_query(self):
cat = self.make_cat()
id = cat.key().id()
def requester(fetcher):
ref = fetcher.request_obj(Cat, id)
yield ref.get
fetcher = DataFetcher()
cat2 = fetcher.spawn_and_run(requester)
cat3 = fetcher.get_obj(Cat, id)
self.assert_cat_equals(cat, cat3)
self.assert_cat_equals(cat, cat2)
self.assert_stats(1, 1, 1)
def _test_obj_single_query(self):
cat = self.make_cat()
id = cat.key().id()
def requester(fetcher):
ref = fetcher.request_obj(Cat, id)
yield ref.get
fetcher = DataFetcher()
cat2 = fetcher.spawn_and_run(requester)
cat3 = fetcher.get_obj(Cat, id)
self.assert_cat_equals(cat, cat3)
self.assert_cat_equals(cat, cat2)
self.assert_stats(1, 1, 1)
def _test_too_many_yields_exception(self):
def requester(self, fetcher):
yield lambda: "foo"
yield
fetcher = DataFetcher()
self.assertRaises(Exception, fetcher.spawn_and_run, requester)
def do_test_cache(self, do_fetch, num_fetches, num_requests, num_db_fetches):
# On the first fetch, all requests should have been fetched from
# the data store.
self.assert_stats(num_fetches, num_requests, num_db_fetches)
# On the second fetch of the same data, the listener shouldn't
# have any stats incremented as the data should have been fetched
# from local cache.
do_fetch()
self.assert_stats(num_fetches, num_requests, num_db_fetches)
# If we clear the local cache and request the same keys,
# the data should be fetched from memcache.
DataFetcher.reset()
DataFetcher.set_listener(DataFetcherTestListener())
do_fetch()
self.assert_stats(num_fetches, num_requests, 0)
def do_test_cache(self, do_fetch, num_fetches, num_requests,
num_db_fetches):
# On the first fetch, all requests should have been fetched from
# the data store.
self.assert_stats(num_fetches, num_requests, num_db_fetches)
# On the second fetch of the same data, the listener shouldn't
# have any stats incremented as the data should have been fetched
# from local cache.
do_fetch()
self.assert_stats(num_fetches, num_requests, num_db_fetches)
# If we clear the local cache and request the same keys,
# the data should be fetched from memcache.
DataFetcher.reset()
DataFetcher.set_listener(DataFetcherTestListener())
do_fetch()
self.assert_stats(num_fetches, num_requests, 0)
def _test_identical_requests_with_refs(self):
cat = self.make_cat()
requesters = [partial(self.request_cat, cat) for i in xrange(2)]
cats = DataFetcher().spawn_and_run_multi(requesters)
self.assert_stats(1, 1, 1)
for cat1 in cats:
self.assert_cat_equals(cat, cat1)
# Next, test with a sequential request for the same key.
requester = partial(self.request_cat, cat)
cat1 = DataFetcher().spawn_and_run(requester)
# check that the new data returned the data without performing any fetches
# (the stats haven't changed)
self.assert_stats(1, 1, 1)
self.assert_cat_equals(cat, cat1)
def _test_multi_spawn_and_join_with_common_friend(self):
cat1 = self.make_cat()
cat2 = self.make_cat()
friend = self.make_cat()
cat1.friend_id = friend.key().id()
cat2.friend_id = friend.key().id()
db.put([cat1, cat2])
pairs = [(cat1, friend), (cat2, friend)]
requesters = [partial(self.requester_with_join, pair) for pair in pairs]
fetcher = DataFetcher()
results = fetcher.spawn_and_run_multi(requesters)
for i in xrange(len(pairs)):
self.check_cat_pair_fetched(pairs[i], results[i], fetcher)
self.assert_stats(2, 3, 2)
def _test_multi_query(self):
num = 3
cats = [self.make_cat() for i in xrange(num)]
mkey = "cats_test"
memcache.delete(mkey)
def requester(fetcher):
query = db.GqlQuery("select * from Cat order by __key__ desc")
ref = fetcher.request(mkey, query, num)
yield ref.get
fetcher = DataFetcher()
cats1 = fetcher.spawn_and_run(requester)
cats2 = fetcher.get(mkey)
self.assert_obj_list_equals(cats, cats1)
self.assert_obj_list_equals(cats, cats2)
self.assert_stats(1, 1, 1)
def _test_multi_query(self):
num = 3
cats = [self.make_cat() for i in xrange(num)]
mkey = "cats_test"
memcache.delete(mkey)
def requester(fetcher):
query = db.GqlQuery("select * from Cat order by __key__ desc")
ref = fetcher.request(mkey, query, num)
yield ref.get
fetcher = DataFetcher()
cats1 = fetcher.spawn_and_run(requester)
cats2 = fetcher.get(mkey)
self.assert_obj_list_equals(cats, cats1)
self.assert_obj_list_equals(cats, cats2)
self.assert_stats(1, 1, 1)
def _test_multi_spawn_and_join_with_common_friend(self):
cat1 = self.make_cat()
cat2 = self.make_cat()
friend = self.make_cat()
cat1.friend_id = friend.key().id()
cat2.friend_id = friend.key().id()
db.put([cat1, cat2])
pairs = [(cat1, friend), (cat2, friend)]
requesters = [
partial(self.requester_with_join, pair) for pair in pairs
]
fetcher = DataFetcher()
results = fetcher.spawn_and_run_multi(requesters)
for i in xrange(len(pairs)):
self.check_cat_pair_fetched(pairs[i], results[i], fetcher)
self.assert_stats(2, 3, 2)
def _test_already_ran_exception(self):
fetcher = DataFetcher().run()
self.assertRaises(AlreadyRanException, fetcher.run)
def requester(fetcher):
fetcher.run()
fetcher = DataFetcher()
fetcher.spawn(requester)
self.assertRaises(AlreadyRanException, fetcher.run)
def do_fetch():
requesters = [partial(self.request_cat, cat) for cat in cats]
fetcher = DataFetcher()
results = fetcher.spawn_and_run_multi(requesters)
return (fetcher, results)
def do_fetch():
requester = partial(self.request_cat, cat)
fetcher = DataFetcher()
result = fetcher.spawn_and_run(requester)
return (fetcher, result)
def assert_stats(self, num_fetches, num_requests, num_db_fetches):
listener = DataFetcher.get_listener()
self.assertEquals(num_fetches, listener.num_fetches)
self.assertEquals(num_requests, listener.num_requests)
self.assertEquals(num_db_fetches, listener.num_db_fetches)
def setUp(self):
DataFetcher.reset()
DataFetcher.set_listener(DataFetcherTestListener())
def _test_fetch_obj(self):
cat, friend = pair = self.make_cat_pair()
cat1 = DataFetcher.fetch_obj(Cat, cat.key().id())
friend1 = DataFetcher.fetch_obj(Cat, friend.key().id())
self.assert_obj_list_equals(pair, [cat1, friend1])
def _test_not_running_exception(self):
fetcher = DataFetcher()
self.assertRaises(NotRunningException, fetcher.request, "some_key",
db.GqlQuery("select * from Cat"))
self.assertRaises(NotRunningException, fetcher.request_obj, Cat, 1)
def _test_not_run_exception(self):
fetcher = DataFetcher()
self.assertRaises(NotRunException, fetcher.get, "foo")
self.assertRaises(NotRunException, fetcher.get_obj, Cat, 1)
def _test_fetch_objs(self):
pair = self.make_cat_pair()
pair1 = DataFetcher.fetch_objs(Cat, [cat.key().id() for cat in pair])
self.assert_obj_list_equals(pair, pair1)
def _test_fetch_obj(self):
cat, friend = pair = self.make_cat_pair()
cat1 = DataFetcher.fetch_obj(Cat, cat.key().id())
friend1 = DataFetcher.fetch_obj(Cat, friend.key().id())
self.assert_obj_list_equals(pair, [cat1, friend1])
def _test_fetch_objs(self):
pair = self.make_cat_pair()
pair1 = DataFetcher.fetch_objs(Cat, [cat.key().id() for cat in pair])
self.assert_obj_list_equals(pair, pair1)
def setUp(self):
DataFetcher.reset()
DataFetcher.set_listener(DataFetcherTestListener())
def assert_stats(self, num_fetches, num_requests, num_db_fetches):
listener = DataFetcher.get_listener()
self.assertEquals(num_fetches, listener.num_fetches)
self.assertEquals(num_requests, listener.num_requests)
self.assertEquals(num_db_fetches, listener.num_db_fetches)
def do_fetch():
requester = partial(self.request_cat, cat)
fetcher = DataFetcher()
result = fetcher.spawn_and_run(requester)
return (fetcher, result)
def do_fetch():
requesters = [partial(self.request_cat, cat) for cat in cats]
fetcher = DataFetcher()
results = fetcher.spawn_and_run_multi(requesters)
return (fetcher, results)
