def testMemcacheQueue(self):
"""Tests adding and popping from an in-memory queue with continuation."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
MEMCACHE_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks("default", expected_count=1)
# First pop request.
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 1, result.number)
# Continuation task enqueued.
next_task = testutil.get_tasks("default", expected_count=2, index=1)
self.assertEquals(3, int(next_task["params"]["cursor"]))
self.assertTrue(next_task["name"].endswith("-1"))
# Second pop request.
request = testutil.create_test_request("POST", None, *next_task["params"].items())
os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(2, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 4, result.number)
def testPopMultiple(self):
"""Tests popping multiple entities from a queue.
Should also cause a continuation task to be inserted to handle entities
after the current batch size.
"""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks('default',
expected_count=2,
index=1,
usec_eta=True)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 3, result.number)
# Continuation two.
next_task = testutil.get_tasks('default',
expected_count=3,
index=2,
usec_eta=True)
next_params = next_task['params']
self.assertTrue('cursor' in next_params)
self.assertTrue(next_task['name'].endswith('-2'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals([], result_list)
testutil.get_tasks('default', expected_count=3, usec_eta=True)
def testPopMultiple(self):
"""Tests popping multiple entities from a queue.
Should also cause a continuation task to be inserted to handle entities
after the current batch size.
"""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks('default',
expected_count=2,
index=1,
usec_eta=True)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 3, result.number)
# Continuation two.
next_task = testutil.get_tasks('default',
expected_count=3,
index=2,
usec_eta=True)
next_params = next_task['params']
self.assertTrue('cursor' in next_params)
self.assertTrue(next_task['name'].endswith('-2'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals([], result_list)
testutil.get_tasks('default', expected_count=3, usec_eta=True)
def testPopMultiple(self):
"""Tests popping multiple entities from a queue.
Should also cause a continuation task to be inserted to handle entities
after the current batch size.
"""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
# First pop request.
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks("default", expected_count=2, index=1, usec_eta=True)
self.assertTrue("cursor" in next_task["params"])
self.assertTrue(next_task["name"].endswith("-1"))
request = testutil.create_test_request("POST", None, *next_task["params"].items())
os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"]
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 3, result.number)
# Continuation two.
next_task = testutil.get_tasks("default", expected_count=3, index=2, usec_eta=True)
next_params = next_task["params"]
self.assertTrue("cursor" in next_params)
self.assertTrue(next_task["name"].endswith("-2"))
request = testutil.create_test_request("POST", None, *next_task["params"].items())
os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"]
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals([], result_list)
testutil.get_tasks("default", expected_count=3, usec_eta=True)
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
old_valid = stub._IsValidQueue
stub._IsValidQueue = lambda *a, **k: True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = 'default-%d' % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name,
expected_count=2,
index=1)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
finally:
stub._IsValidQueue = old_valid
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
old_valid = stub._IsValidQueue
stub._IsValidQueue = lambda *a, **k: True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = 'default-%d' % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name,
expected_count=2,
index=1)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
finally:
stub._IsValidQueue = old_valid
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub("taskqueue")
stub._queues[None]._all_queues_valid = True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = "default-%d" % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name, expected_count=2, index=1)
self.assertTrue("cursor" in next_task["params"])
self.assertTrue(next_task["name"].endswith("-1"))
finally:
stub._queues[None]._all_queues_valid = False
def testMemcacheQueue_PopError(self):
"""Tests calling pop() when memcache is down."""
work_index = MEMCACHE_QUEUE.next_index()
entity = TestModel(work_index=work_index, number=0)
MEMCACHE_QUEUE.put(work_index, [entity])
memcache.flush_all()
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals([], result_list)
def testMemcacheQueue_PopDecodeError(self):
"""Tests when proto decoding fails on the pop() call."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
memcache.set(MEMCACHE_QUEUE._create_index_key(work_index, 1), "bad data")
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
def testMemcacheQueue_PopError(self):
"""Tests calling pop() when memcache is down."""
work_index = MEMCACHE_QUEUE.next_index()
entity = TestModel(work_index=work_index, number=0)
MEMCACHE_QUEUE.put(work_index, [entity])
memcache.flush_all()
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals([], result_list)
def testMemcacheQueue_PopHoles(self):
"""Tests when there are holes in the memcache array."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
memcache.delete(MEMCACHE_QUEUE._create_index_key(work_index, 1))
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals([1, 3], [r.number for r in result_list])
def testMemcacheQueue(self):
"""Tests adding and popping from an in-memory queue with continuation."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i),
work_index=work_index,
number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
MEMCACHE_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks('default', expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 1, result.number)
# Continuation task enqueued.
next_task = testutil.get_tasks('default', expected_count=2, index=1)
self.assertEquals(3, int(next_task['params']['cursor']))
self.assertTrue(next_task['name'].endswith('-1'))
# Second pop request.
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(2, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 4, result.number)
def testMemcacheQueue_PopDecodeError(self):
"""Tests when proto decoding fails on the pop() call."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i),
work_index=work_index,
number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
memcache.set(MEMCACHE_QUEUE._create_index_key(work_index, 1),
'bad data')
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
def testMemcacheQueue_PopHoles(self):
"""Tests when there are holes in the memcache array."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i),
work_index=work_index,
number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
memcache.delete(MEMCACHE_QUEUE._create_index_key(work_index, 1))
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals([1, 3], [r.number for r in result_list])
def testPopOne(self):
"""Tests popping a single entity from the queue."""
t1 = TestModel(work_index=TEST_QUEUE.next_index(), number=1)
# Ensure these other tasks don't interfere.
t2 = TestModel(number=2)
t3 = TestModel(number=3)
db.put([t1, t2, t3])
TEST_QUEUE.add(t1.work_index)
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(t1.work_index)["name"]
result = TEST_QUEUE.pop_request(request)
self.assertEquals(1, len(result))
self.assertEquals(t1.key(), result[0].key())
self.assertEquals(t1.work_index, result[0].work_index)
self.assertEquals(1, result[0].number)
def testPopAlreadyExists(self):
"""Tests popping work items when the continuation task already exists."""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks("default", expected_count=1)
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks("default", expected_count=2)
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks("default", expected_count=2)
def testPopAlreadyExists(self):
"""Tests popping work items when the continuation task already exists."""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks('default', expected_count=1)
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks('default', expected_count=2)
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks('default', expected_count=2)
def testPopOne(self):
"""Tests popping a single entity from the queue."""
t1 = TestModel(work_index=TEST_QUEUE.next_index(), number=1)
# Ensure these other tasks don't interfere.
t2 = TestModel(number=2)
t3 = TestModel(number=3)
db.put([t1, t2, t3])
TEST_QUEUE.add(t1.work_index)
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(t1.work_index)['name']
result = TEST_QUEUE.pop_request(request)
self.assertEquals(1, len(result))
self.assertEquals(t1.key(), result[0].key())
self.assertEquals(t1.work_index, result[0].work_index)
self.assertEquals(1, result[0].number)
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
stub._queues[None]._all_queues_valid = True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(
TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = 'default-%d' % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name,
expected_count=2,
index=1)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
finally:
stub._queues[None]._all_queues_valid = False
