def testMemcacheQueue(self): """Tests adding and popping from an in-memory queue with continuation.""" work_index = MEMCACHE_QUEUE.next_index() work_items = [ TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6) ] MEMCACHE_QUEUE.put(work_index, work_items) MEMCACHE_QUEUE.add(work_index, gettime=self.gettime1) testutil.get_tasks("default", expected_count=1) # First pop request. request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 1, result.number) # Continuation task enqueued. next_task = testutil.get_tasks("default", expected_count=2, index=1) self.assertEquals(3, int(next_task["params"]["cursor"])) self.assertTrue(next_task["name"].endswith("-1")) # Second pop request. request = testutil.create_test_request("POST", None, *next_task["params"].items()) os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals(2, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 4, result.number)
def testPopMultiple(self): """Tests popping multiple entities from a queue. Should also cause a continuation task to be inserted to handle entities after the current batch size. """ work_index = TEST_QUEUE.next_index() tasks = [] for i in xrange(6): # Simplify tests by assigning the key names of the TestModel, making it # so the values returned by pop_request() below are predictable. key = db.Key.from_path(TestModel.kind(), i + 1) tasks.append(TestModel(key=key, work_index=work_index, number=i)) db.put(tasks) TEST_QUEUE.add(work_index, gettime=self.gettime1) # First pop request. request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = TEST_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i, result.number) # Continuation one. next_task = testutil.get_tasks('default', expected_count=2, index=1, usec_eta=True) self.assertTrue('cursor' in next_task['params']) self.assertTrue(next_task['name'].endswith('-1')) request = testutil.create_test_request('POST', None, *next_task['params'].items()) os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name'] result_list = TEST_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 3, result.number) # Continuation two. next_task = testutil.get_tasks('default', expected_count=3, index=2, usec_eta=True) next_params = next_task['params'] self.assertTrue('cursor' in next_params) self.assertTrue(next_task['name'].endswith('-2')) request = testutil.create_test_request('POST', None, *next_task['params'].items()) os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name'] result_list = TEST_QUEUE.pop_request(request) self.assertEquals([], result_list) testutil.get_tasks('default', expected_count=3, usec_eta=True)
def testPopMultiple(self): """Tests popping multiple entities from a queue. Should also cause a continuation task to be inserted to handle entities after the current batch size. """ work_index = TEST_QUEUE.next_index() tasks = [] for i in xrange(6): tasks.append(TestModel(work_index=work_index, number=i)) db.put(tasks) TEST_QUEUE.add(work_index, gettime=self.gettime1) # First pop request. request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = TEST_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i, result.number) # Continuation one. next_task = testutil.get_tasks('default', expected_count=2, index=1, usec_eta=True) self.assertTrue('cursor' in next_task['params']) self.assertTrue(next_task['name'].endswith('-1')) request = testutil.create_test_request('POST', None, *next_task['params'].items()) os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name'] result_list = TEST_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 3, result.number) # Continuation two. next_task = testutil.get_tasks('default', expected_count=3, index=2, usec_eta=True) next_params = next_task['params'] self.assertTrue('cursor' in next_params) self.assertTrue(next_task['name'].endswith('-2')) request = testutil.create_test_request('POST', None, *next_task['params'].items()) os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name'] result_list = TEST_QUEUE.pop_request(request) self.assertEquals([], result_list) testutil.get_tasks('default', expected_count=3, usec_eta=True)
def testPopMultiple(self): """Tests popping multiple entities from a queue. Should also cause a continuation task to be inserted to handle entities after the current batch size. """ work_index = TEST_QUEUE.next_index() tasks = [] for i in xrange(6): # Simplify tests by assigning the key names of the TestModel, making it # so the values returned by pop_request() below are predictable. key = db.Key.from_path(TestModel.kind(), i + 1) tasks.append(TestModel(key=key, work_index=work_index, number=i)) db.put(tasks) TEST_QUEUE.add(work_index, gettime=self.gettime1) # First pop request. request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = TEST_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i, result.number) # Continuation one. next_task = testutil.get_tasks("default", expected_count=2, index=1, usec_eta=True) self.assertTrue("cursor" in next_task["params"]) self.assertTrue(next_task["name"].endswith("-1")) request = testutil.create_test_request("POST", None, *next_task["params"].items()) os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"] result_list = TEST_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 3, result.number) # Continuation two. next_task = testutil.get_tasks("default", expected_count=3, index=2, usec_eta=True) next_params = next_task["params"] self.assertTrue("cursor" in next_params) self.assertTrue(next_task["name"].endswith("-2")) request = testutil.create_test_request("POST", None, *next_task["params"].items()) os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"] result_list = TEST_QUEUE.pop_request(request) self.assertEquals([], result_list) testutil.get_tasks("default", expected_count=3, usec_eta=True)
def testShardedQueue(self): """Tests adding and popping from a sharded queue with continuation.""" from google.appengine.api import apiproxy_stub_map stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') old_valid = stub._IsValidQueue stub._IsValidQueue = lambda *a, **k: True try: work_index = SHARDED_QUEUE.next_index() tasks = [] for i in xrange(5): tasks.append(TestModel(work_index=work_index, number=i)) db.put(tasks) SHARDED_QUEUE.add(work_index, gettime=self.gettime1) queue_name = 'default-%d' % (1 + (work_index % 4)) testutil.get_tasks(queue_name, expected_count=1) # First pop request. request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = SHARDED_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i, result.number) # Continuation one. next_task = testutil.get_tasks(queue_name, expected_count=2, index=1) self.assertTrue('cursor' in next_task['params']) self.assertTrue(next_task['name'].endswith('-1')) finally: stub._IsValidQueue = old_valid
def testShardedQueue(self): """Tests adding and popping from a sharded queue with continuation.""" from google.appengine.api import apiproxy_stub_map stub = apiproxy_stub_map.apiproxy.GetStub("taskqueue") stub._queues[None]._all_queues_valid = True try: work_index = SHARDED_QUEUE.next_index() tasks = [] for i in xrange(5): # Simplify tests by assigning the key names of the TestModel, making it # so the values returned by pop_request() below are predictable. key = db.Key.from_path(TestModel.kind(), i + 1) tasks.append(TestModel(key=key, work_index=work_index, number=i)) db.put(tasks) SHARDED_QUEUE.add(work_index, gettime=self.gettime1) queue_name = "default-%d" % (1 + (work_index % 4)) testutil.get_tasks(queue_name, expected_count=1) # First pop request. request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = SHARDED_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i, result.number) # Continuation one. next_task = testutil.get_tasks(queue_name, expected_count=2, index=1) self.assertTrue("cursor" in next_task["params"]) self.assertTrue(next_task["name"].endswith("-1")) finally: stub._queues[None]._all_queues_valid = False
def testMemcacheQueue_PopError(self): """Tests calling pop() when memcache is down.""" work_index = MEMCACHE_QUEUE.next_index() entity = TestModel(work_index=work_index, number=0) MEMCACHE_QUEUE.put(work_index, [entity]) memcache.flush_all() request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals([], result_list)
def testMemcacheQueue_PopDecodeError(self): """Tests when proto decoding fails on the pop() call.""" work_index = MEMCACHE_QUEUE.next_index() work_items = [ TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6) ] MEMCACHE_QUEUE.put(work_index, work_items) memcache.set(MEMCACHE_QUEUE._create_index_key(work_index, 1), "bad data") request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = MEMCACHE_QUEUE.pop_request(request)
def testMemcacheQueue_PopError(self): """Tests calling pop() when memcache is down.""" work_index = MEMCACHE_QUEUE.next_index() entity = TestModel(work_index=work_index, number=0) MEMCACHE_QUEUE.put(work_index, [entity]) memcache.flush_all() request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals([], result_list)
def testMemcacheQueue_PopHoles(self): """Tests when there are holes in the memcache array.""" work_index = MEMCACHE_QUEUE.next_index() work_items = [ TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6) ] MEMCACHE_QUEUE.put(work_index, work_items) memcache.delete(MEMCACHE_QUEUE._create_index_key(work_index, 1)) request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals([1, 3], [r.number for r in result_list])
def testMemcacheQueue(self): """Tests adding and popping from an in-memory queue with continuation.""" work_index = MEMCACHE_QUEUE.next_index() work_items = [ TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6) ] MEMCACHE_QUEUE.put(work_index, work_items) MEMCACHE_QUEUE.add(work_index, gettime=self.gettime1) testutil.get_tasks('default', expected_count=1) # First pop request. request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 1, result.number) # Continuation task enqueued. next_task = testutil.get_tasks('default', expected_count=2, index=1) self.assertEquals(3, int(next_task['params']['cursor'])) self.assertTrue(next_task['name'].endswith('-1')) # Second pop request. request = testutil.create_test_request('POST', None, *next_task['params'].items()) os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name'] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals(2, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i + 4, result.number)
def testMemcacheQueue_PopDecodeError(self): """Tests when proto decoding fails on the pop() call.""" work_index = MEMCACHE_QUEUE.next_index() work_items = [ TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6) ] MEMCACHE_QUEUE.put(work_index, work_items) memcache.set(MEMCACHE_QUEUE._create_index_key(work_index, 1), 'bad data') request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = MEMCACHE_QUEUE.pop_request(request)
def testMemcacheQueue_PopHoles(self): """Tests when there are holes in the memcache array.""" work_index = MEMCACHE_QUEUE.next_index() work_items = [ TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6) ] MEMCACHE_QUEUE.put(work_index, work_items) memcache.delete(MEMCACHE_QUEUE._create_index_key(work_index, 1)) request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = MEMCACHE_QUEUE.pop_request(request) self.assertEquals([1, 3], [r.number for r in result_list])
def testPopOne(self): """Tests popping a single entity from the queue.""" t1 = TestModel(work_index=TEST_QUEUE.next_index(), number=1) # Ensure these other tasks don't interfere. t2 = TestModel(number=2) t3 = TestModel(number=3) db.put([t1, t2, t3]) TEST_QUEUE.add(t1.work_index) request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(t1.work_index)["name"] result = TEST_QUEUE.pop_request(request) self.assertEquals(1, len(result)) self.assertEquals(t1.key(), result[0].key()) self.assertEquals(t1.work_index, result[0].work_index) self.assertEquals(1, result[0].number)
def testPopAlreadyExists(self): """Tests popping work items when the continuation task already exists.""" work_index = TEST_QUEUE.next_index() tasks = [] for i in xrange(6): tasks.append(TestModel(work_index=work_index, number=i)) db.put(tasks) TEST_QUEUE.add(work_index, gettime=self.gettime1) testutil.get_tasks("default", expected_count=1) request = testutil.create_test_request("POST", None) os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"] result_list = TEST_QUEUE.pop_request(request) testutil.get_tasks("default", expected_count=2) result_list = TEST_QUEUE.pop_request(request) testutil.get_tasks("default", expected_count=2)
def testPopAlreadyExists(self): """Tests popping work items when the continuation task already exists.""" work_index = TEST_QUEUE.next_index() tasks = [] for i in xrange(6): tasks.append(TestModel(work_index=work_index, number=i)) db.put(tasks) TEST_QUEUE.add(work_index, gettime=self.gettime1) testutil.get_tasks('default', expected_count=1) request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = TEST_QUEUE.pop_request(request) testutil.get_tasks('default', expected_count=2) result_list = TEST_QUEUE.pop_request(request) testutil.get_tasks('default', expected_count=2)
def testPopOne(self): """Tests popping a single entity from the queue.""" t1 = TestModel(work_index=TEST_QUEUE.next_index(), number=1) # Ensure these other tasks don't interfere. t2 = TestModel(number=2) t3 = TestModel(number=3) db.put([t1, t2, t3]) TEST_QUEUE.add(t1.work_index) request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(t1.work_index)['name'] result = TEST_QUEUE.pop_request(request) self.assertEquals(1, len(result)) self.assertEquals(t1.key(), result[0].key()) self.assertEquals(t1.work_index, result[0].work_index) self.assertEquals(1, result[0].number)
def testShardedQueue(self): """Tests adding and popping from a sharded queue with continuation.""" from google.appengine.api import apiproxy_stub_map stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') stub._queues[None]._all_queues_valid = True try: work_index = SHARDED_QUEUE.next_index() tasks = [] for i in xrange(5): # Simplify tests by assigning the key names of the TestModel, making it # so the values returned by pop_request() below are predictable. key = db.Key.from_path(TestModel.kind(), i + 1) tasks.append( TestModel(key=key, work_index=work_index, number=i)) db.put(tasks) SHARDED_QUEUE.add(work_index, gettime=self.gettime1) queue_name = 'default-%d' % (1 + (work_index % 4)) testutil.get_tasks(queue_name, expected_count=1) # First pop request. request = testutil.create_test_request('POST', None) os.environ['HTTP_X_APPENGINE_TASKNAME'] = \ self.expect_task(work_index)['name'] result_list = SHARDED_QUEUE.pop_request(request) self.assertEquals(3, len(result_list)) for i, result in enumerate(result_list): self.assertEquals(work_index, result.work_index) self.assertEquals(i, result.number) # Continuation one. next_task = testutil.get_tasks(queue_name, expected_count=2, index=1) self.assertTrue('cursor' in next_task['params']) self.assertTrue(next_task['name'].endswith('-1')) finally: stub._queues[None]._all_queues_valid = False