def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub("taskqueue")
stub._queues[None]._all_queues_valid = True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = "default-%d" % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name, expected_count=2, index=1)
self.assertTrue("cursor" in next_task["params"])
self.assertTrue(next_task["name"].endswith("-1"))
finally:
stub._queues[None]._all_queues_valid = False
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
old_valid = stub._IsValidQueue
stub._IsValidQueue = lambda *a, **k: True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = 'default-%d' % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name,
expected_count=2,
index=1)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
finally:
stub._IsValidQueue = old_valid
def testMemcacheQueue(self):
"""Tests adding and popping from an in-memory queue with continuation."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i), work_index=work_index, number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
MEMCACHE_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks("default", expected_count=1)
# First pop request.
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 1, result.number)
# Continuation task enqueued.
next_task = testutil.get_tasks("default", expected_count=2, index=1)
self.assertEquals(3, int(next_task["params"]["cursor"]))
self.assertTrue(next_task["name"].endswith("-1"))
# Second pop request.
request = testutil.create_test_request("POST", None, *next_task["params"].items())
os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"]
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(2, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 4, result.number)
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
old_valid = stub._IsValidQueue
stub._IsValidQueue = lambda *a, **k: True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = 'default-%d' % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name,
expected_count=2,
index=1)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
finally:
stub._IsValidQueue = old_valid
def testOneFragment(self):
"""Tests when there is one fragment in the queue."""
virtual_feed.inject_virtual_feed(self.topic, self.format,
self.header_footer, self.entries_map)
task = testutil.get_tasks('default', index=0, expected_count=1)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task['name']
self.handle('post')
event_list = list(main.EventToDeliver.all())
self.assertEquals(1, len(event_list))
event = event_list[0]
# No parent to ensure it's not rate limited by an entity group.
self.assertEquals(None, event.key().parent())
self.assertEquals(self.topic, event.topic)
self.assertEquals(
'<?xml version="1.0" encoding="utf-8"?>\n'
'<feed><id>tag:my-id</id>\n\n'
'<entry>third data</entry>\n'
'<entry>second data</entry>\n'
'<entry>first data</entry>\n'
'</feed>', event.payload)
self.assertEquals('application/atom+xml', event.content_type)
self.assertEquals(1, event.max_failures)
task = testutil.get_tasks('event-delivery', index=0, expected_count=1)
self.assertEquals(str(event.key()), task['params']['event_key'])
def testOneFragment(self):
"""Tests when there is one fragment in the queue."""
virtual_feed.inject_virtual_feed(
self.topic, self.format, self.header_footer, self.entries_map)
task = testutil.get_tasks('default', index=0, expected_count=1)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task['name']
self.handle('post')
event_list = list(main.EventToDeliver.all())
self.assertEquals(1, len(event_list))
event = event_list[0]
# No parent to ensure it's not rate limited by an entity group.
self.assertEquals(None, event.key().parent())
self.assertEquals(self.topic, event.topic)
self.assertEquals(
'<?xml version="1.0" encoding="utf-8"?>\n'
'<feed><id>tag:my-id</id>\n\n'
'<entry>third data</entry>\n'
'<entry>second data</entry>\n'
'<entry>first data</entry>\n'
'</feed>',
event.payload)
self.assertEquals('application/atom+xml', event.content_type)
self.assertEquals(1, event.max_failures)
task = testutil.get_tasks('event-delivery', index=0, expected_count=1)
self.assertEquals(str(event.key()), task['params']['event_key'])
def testPopMultiple(self):
"""Tests popping multiple entities from a queue.
Should also cause a continuation task to be inserted to handle entities
after the current batch size.
"""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks('default',
expected_count=2,
index=1,
usec_eta=True)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 3, result.number)
# Continuation two.
next_task = testutil.get_tasks('default',
expected_count=3,
index=2,
usec_eta=True)
next_params = next_task['params']
self.assertTrue('cursor' in next_params)
self.assertTrue(next_task['name'].endswith('-2'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals([], result_list)
testutil.get_tasks('default', expected_count=3, usec_eta=True)
def testAfterThreshold(self):
"""Tests when a subscription is not yet ready for reconfirmation."""
self.assertTrue(Subscription.insert(
self.callback, self.topic, self.token, self.secret,
now=self.getnow, lease_seconds=self.threshold_seconds))
sub = self.get_subscription()
self.mapper.run(sub)
testutil.get_tasks(main.POLLING_QUEUE, expected_count=0)
def testIgnoreUnverified(self):
"""Tests that unverified subscriptions are skipped."""
self.assertTrue(Subscription.request_insert(
self.callback, self.topic, self.token, self.secret,
now=self.getnow))
sub = self.get_subscription()
self.mapper.run(sub)
testutil.get_tasks(main.POLLING_QUEUE, expected_count=0)
def testPopMultiple(self):
"""Tests popping multiple entities from a queue.
Should also cause a continuation task to be inserted to handle entities
after the current batch size.
"""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks('default',
expected_count=2,
index=1,
usec_eta=True)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 3, result.number)
# Continuation two.
next_task = testutil.get_tasks('default',
expected_count=3,
index=2,
usec_eta=True)
next_params = next_task['params']
self.assertTrue('cursor' in next_params)
self.assertTrue(next_task['name'].endswith('-2'))
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals([], result_list)
testutil.get_tasks('default', expected_count=3, usec_eta=True)
def testPopMultiple(self):
"""Tests popping multiple entities from a queue.
Should also cause a continuation task to be inserted to handle entities
after the current batch size.
"""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
# First pop request.
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks("default", expected_count=2, index=1, usec_eta=True)
self.assertTrue("cursor" in next_task["params"])
self.assertTrue(next_task["name"].endswith("-1"))
request = testutil.create_test_request("POST", None, *next_task["params"].items())
os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"]
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 3, result.number)
# Continuation two.
next_task = testutil.get_tasks("default", expected_count=3, index=2, usec_eta=True)
next_params = next_task["params"]
self.assertTrue("cursor" in next_params)
self.assertTrue(next_task["name"].endswith("-2"))
request = testutil.create_test_request("POST", None, *next_task["params"].items())
os.environ["HTTP_X_APPENGINE_TASKNAME"] = next_task["name"]
result_list = TEST_QUEUE.pop_request(request)
self.assertEquals([], result_list)
testutil.get_tasks("default", expected_count=3, usec_eta=True)
def testMultipleFragments(self):
"""Tests when there is more than one fragment in the queue."""
virtual_feed.inject_virtual_feed(
self.topic, self.format, self.header_footer, self.entries_map)
virtual_feed.inject_virtual_feed(
self.topic, self.format, self.header_footer, self.entries_map)
task = testutil.get_tasks('default', index=0, expected_count=1)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task['name']
self.handle('post')
event_list = list(main.EventToDeliver.all())
self.assertEquals(1, len(event_list))
event = event_list[0]
self.assertEquals(self.topic, event.topic)
self.assertEquals(
'<?xml version="1.0" encoding="utf-8"?>\n'
'<feed><id>tag:my-id</id>\n\n'
'<entry>third data</entry>\n'
'<entry>second data</entry>\n'
'<entry>first data</entry>\n'
'<entry>third data</entry>\n'
'<entry>second data</entry>\n'
'<entry>first data</entry>\n'
'</feed>',
event.payload)
def testAddOne(self):
"""Tests adding a single entity to the queue."""
t = TestModel(number=1)
t.work_index = TEST_QUEUE.next_index()
t.put()
TEST_QUEUE.add(t.work_index, gettime=self.gettime1)
self.assertTasksEqual([self.expect_task(t.work_index)], testutil.get_tasks("default", usec_eta=True))
def testAddMultipleDifferent(self):
"""Tests adding multiple different tasks to different indexes."""
t1 = TestModel(number=1)
t2 = TestModel(number=2)
t3 = TestModel(number=3)
tasks1 = [t1, t2, t3]
work_index = TEST_QUEUE.next_index()
for t in tasks1:
t.work_index = work_index
db.put(tasks1)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
memcache.incr('fjq-TestModel-index')
t4 = TestModel(number=4)
t5 = TestModel(number=5)
t6 = TestModel(number=6)
tasks2 = [t4, t5, t6]
work_index2 = TEST_QUEUE.next_index()
for t in tasks2:
t.work_index = work_index2
db.put(tasks2)
TEST_QUEUE.add(work_index2, gettime=self.gettime2)
self.assertNotEqual(work_index, work_index2)
self.assertTasksEqual(
[self.expect_task(work_index),
self.expect_task(work_index2, now_time=self.now2)],
testutil.get_tasks('default', usec_eta=True))
def testMultipleFragments(self):
"""Tests when there is more than one fragment in the queue."""
virtual_feed.inject_virtual_feed(self.topic, self.format,
self.header_footer, self.entries_map)
virtual_feed.inject_virtual_feed(self.topic, self.format,
self.header_footer, self.entries_map)
task = testutil.get_tasks('default', index=0, expected_count=1)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task['name']
self.handle('post')
event_list = list(main.EventToDeliver.all())
self.assertEquals(1, len(event_list))
event = event_list[0]
self.assertEquals(self.topic, event.topic)
self.assertEquals(
'<?xml version="1.0" encoding="utf-8"?>\n'
'<feed><id>tag:my-id</id>\n\n'
'<entry>third data</entry>\n'
'<entry>second data</entry>\n'
'<entry>first data</entry>\n'
'<entry>third data</entry>\n'
'<entry>second data</entry>\n'
'<entry>first data</entry>\n'
'</feed>', event.payload)
def testAddMultipleDifferent(self):
"""Tests adding multiple different tasks to different indexes."""
t1 = TestModel(number=1)
t2 = TestModel(number=2)
t3 = TestModel(number=3)
tasks1 = [t1, t2, t3]
work_index = TEST_QUEUE.next_index()
for t in tasks1:
t.work_index = work_index
db.put(tasks1)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
memcache.incr('fjq-TestModel-index')
t4 = TestModel(number=4)
t5 = TestModel(number=5)
t6 = TestModel(number=6)
tasks2 = [t4, t5, t6]
work_index2 = TEST_QUEUE.next_index()
for t in tasks2:
t.work_index = work_index2
db.put(tasks2)
TEST_QUEUE.add(work_index2, gettime=self.gettime2)
self.assertNotEqual(work_index, work_index2)
self.assertTasksEqual([
self.expect_task(work_index),
self.expect_task(work_index2, now_time=self.now2)
], testutil.get_tasks('default', usec_eta=True))
def testAddOne(self):
"""Tests adding a single entity to the queue."""
t = TestModel(number=1)
t.work_index = TEST_QUEUE.next_index()
t.put()
TEST_QUEUE.add(t.work_index, gettime=self.gettime1)
self.assertTasksEqual([self.expect_task(t.work_index)],
testutil.get_tasks('default', usec_eta=True))
def testBeforeThreshold(self):
"""Tests when a subscription is ready for reconfirmation."""
self.assertTrue(Subscription.insert(
self.callback, self.topic, self.token, self.secret,
now=self.getnow, lease_seconds=self.threshold_seconds-1))
sub = self.get_subscription()
self.mapper.run(sub)
task = testutil.get_tasks(main.POLLING_QUEUE, index=0, expected_count=1)
self.assertEquals('polling', task['headers']['X-AppEngine-QueueName'])
def testShardedQueue(self):
"""Tests adding and popping from a sharded queue with continuation."""
from google.appengine.api import apiproxy_stub_map
stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue')
stub._queues[None]._all_queues_valid = True
try:
work_index = SHARDED_QUEUE.next_index()
tasks = []
for i in xrange(5):
# Simplify tests by assigning the key names of the TestModel, making it
# so the values returned by pop_request() below are predictable.
key = db.Key.from_path(TestModel.kind(), i + 1)
tasks.append(
TestModel(key=key, work_index=work_index, number=i))
db.put(tasks)
SHARDED_QUEUE.add(work_index, gettime=self.gettime1)
queue_name = 'default-%d' % (1 + (work_index % 4))
testutil.get_tasks(queue_name, expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = SHARDED_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i, result.number)
# Continuation one.
next_task = testutil.get_tasks(queue_name,
expected_count=2,
index=1)
self.assertTrue('cursor' in next_task['params'])
self.assertTrue(next_task['name'].endswith('-1'))
finally:
stub._queues[None]._all_queues_valid = False
def testMemcacheQueue(self):
"""Tests adding and popping from an in-memory queue with continuation."""
work_index = MEMCACHE_QUEUE.next_index()
work_items = [
TestModel(key=db.Key.from_path(TestModel.kind(), i),
work_index=work_index,
number=i) for i in xrange(1, 6)
]
MEMCACHE_QUEUE.put(work_index, work_items)
MEMCACHE_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks('default', expected_count=1)
# First pop request.
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(3, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 1, result.number)
# Continuation task enqueued.
next_task = testutil.get_tasks('default', expected_count=2, index=1)
self.assertEquals(3, int(next_task['params']['cursor']))
self.assertTrue(next_task['name'].endswith('-1'))
# Second pop request.
request = testutil.create_test_request('POST', None,
*next_task['params'].items())
os.environ['HTTP_X_APPENGINE_TASKNAME'] = next_task['name']
result_list = MEMCACHE_QUEUE.pop_request(request)
self.assertEquals(2, len(result_list))
for i, result in enumerate(result_list):
self.assertEquals(work_index, result.work_index)
self.assertEquals(i + 4, result.number)
def testAddMultiple(self):
"""Tests adding multiple tasks on the same index."""
t1 = TestModel(number=1)
t2 = TestModel(number=2)
t3 = TestModel(number=3)
tasks = [t1, t2, t3]
work_index = TEST_QUEUE.next_index()
for t in tasks:
t.work_index = work_index
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
self.assertTasksEqual([self.expect_task(work_index)], testutil.get_tasks("default", usec_eta=True))
def testAddMultiple(self):
"""Tests adding multiple tasks on the same index."""
t1 = TestModel(number=1)
t2 = TestModel(number=2)
t3 = TestModel(number=3)
tasks = [t1, t2, t3]
work_index = TEST_QUEUE.next_index()
for t in tasks:
t.work_index = work_index
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
self.assertTasksEqual([self.expect_task(work_index)],
testutil.get_tasks('default', usec_eta=True))
def testMultipleQueues(self):
"""Tests multiple virtual feeds and queues."""
virtual_feed.inject_virtual_feed(self.topic, self.format,
self.header_footer, self.entries_map)
virtual_feed.inject_virtual_feed(self.topic2, self.format,
self.header_footer, self.entries_map)
task1, task2 = testutil.get_tasks('default', expected_count=2)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task1['name']
self.handle('post')
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task2['name']
self.handle('post')
event_list = list(main.EventToDeliver.all())
self.assertEquals(2, len(event_list))
self.assertEquals(self.topic, event_list[0].topic)
self.assertEquals(self.topic2, event_list[1].topic)
def testZeroBatchTime(self):
"""Tests that zero batch time results in no task ETA."""
work_index = TEST_QUEUE_ZERO_BATCH_TIME.next_index()
task = TestModel(work_index=work_index, number=1)
db.put(task)
before_index = memcache.get(TEST_QUEUE_ZERO_BATCH_TIME.index_name)
self.assertEquals(work_index, fork_join_queue.knuth_hash(before_index))
TEST_QUEUE_ZERO_BATCH_TIME.add(work_index, gettime=self.gettime1)
# This confirms the behavior that batch_period_ms of zero will cause
# immediate increment after adding the tasks.
after_index = memcache.get(TEST_QUEUE_ZERO_BATCH_TIME.index_name)
self.assertEquals(before_index + 1, after_index)
self.assertTasksEqual([self.expect_task(work_index)],
testutil.get_tasks('default', usec_eta=True),
check_eta=False)
def testZeroBatchTime(self):
"""Tests that zero batch time results in no task ETA."""
work_index = TEST_QUEUE_ZERO_BATCH_TIME.next_index()
task = TestModel(work_index=work_index, number=1)
db.put(task)
before_index = memcache.get(TEST_QUEUE_ZERO_BATCH_TIME.index_name)
self.assertEquals(work_index, fork_join_queue.knuth_hash(before_index))
TEST_QUEUE_ZERO_BATCH_TIME.add(work_index, gettime=self.gettime1)
# This confirms the behavior that batch_period_ms of zero will cause
# immediate increment after adding the tasks.
after_index = memcache.get(TEST_QUEUE_ZERO_BATCH_TIME.index_name)
self.assertEquals(before_index + 1, after_index)
self.assertTasksEqual(
[self.expect_task(work_index)], testutil.get_tasks("default", usec_eta=True), check_eta=False
)
def testMultipleQueues(self):
"""Tests multiple virtual feeds and queues."""
virtual_feed.inject_virtual_feed(
self.topic, self.format, self.header_footer, self.entries_map)
virtual_feed.inject_virtual_feed(
self.topic2, self.format, self.header_footer, self.entries_map)
task1, task2 = testutil.get_tasks('default', expected_count=2)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task1['name']
self.handle('post')
os.environ['HTTP_X_APPENGINE_TASKNAME'] = task2['name']
self.handle('post')
event_list = list(main.EventToDeliver.all())
self.assertEquals(2, len(event_list))
self.assertEquals(self.topic, event_list[0].topic)
self.assertEquals(self.topic2, event_list[1].topic)
def testInsertOneFragment(self):
"""Tests inserting one new fragment."""
virtual_feed.inject_virtual_feed(
self.topic, self.format, self.header_footer, self.entries_map)
task = testutil.get_tasks('default', index=0, expected_count=1)
self.assertTrue(task['name'].startswith(
'fjq-FeedFragment-54124f41c1ea6e67e4beacac85b9f015e6830d41--'
'my-version-'))
results = virtual_feed.VIRTUAL_FEED_QUEUE.pop(task['name'])
self.assertEquals(1, len(results))
fragment = results[0]
self.assertEquals(self.topic, fragment.topic)
self.assertEquals(self.header_footer, fragment.header_footer)
self.assertEquals(self.format, fragment.format)
self.assertEquals(
'<entry>third data</entry>\n' # Hash order
'<entry>second data</entry>\n'
'<entry>first data</entry>',
fragment.entries)
def testInsertOneFragment(self):
"""Tests inserting one new fragment."""
virtual_feed.inject_virtual_feed(self.topic, self.format,
self.header_footer, self.entries_map)
task = testutil.get_tasks('default', index=0, expected_count=1)
self.assertTrue(task['name'].startswith(
'fjq-FeedFragment-54124f41c1ea6e67e4beacac85b9f015e6830d41--'
'my-version-'))
results = virtual_feed.VIRTUAL_FEED_QUEUE.pop(task['name'])
self.assertEquals(1, len(results))
fragment = results[0]
self.assertEquals(self.topic, fragment.topic)
self.assertEquals(self.header_footer, fragment.header_footer)
self.assertEquals(self.format, fragment.format)
self.assertEquals(
'<entry>third data</entry>\n' # Hash order
'<entry>second data</entry>\n'
'<entry>first data</entry>',
fragment.entries)
def testPopAlreadyExists(self):
"""Tests popping work items when the continuation task already exists."""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks("default", expected_count=1)
request = testutil.create_test_request("POST", None)
os.environ["HTTP_X_APPENGINE_TASKNAME"] = self.expect_task(work_index)["name"]
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks("default", expected_count=2)
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks("default", expected_count=2)
def testInsertMultipleFragments(self):
"""Tests inserting multiple fragments on different virtual topics."""
virtual_feed.inject_virtual_feed(self.topic, self.format,
self.header_footer, self.entries_map)
virtual_feed.inject_virtual_feed(self.topic2, self.format,
self.header_footer, self.entries_map)
task1, task2 = testutil.get_tasks('default', expected_count=2)
self.assertTrue(task1['name'].startswith(
'fjq-FeedFragment-54124f41c1ea6e67e4beacac85b9f015e6830d41--'
'my-version-'))
self.assertTrue(task2['name'].startswith(
'fjq-FeedFragment-0449375bf584a7a5d3a09b344a726dead30c3927--'
'my-version-'))
virtual_feed.VIRTUAL_FEED_QUEUE.name = \
'fjq-FeedFragment-54124f41c1ea6e67e4beacac85b9f015e6830d41-'
fragment1 = virtual_feed.VIRTUAL_FEED_QUEUE.pop(task1['name'])[0]
self.assertEquals(self.topic, fragment1.topic)
virtual_feed.VIRTUAL_FEED_QUEUE.name = \
'fjq-FeedFragment-0449375bf584a7a5d3a09b344a726dead30c3927-'
fragment2 = virtual_feed.VIRTUAL_FEED_QUEUE.pop(task2['name'])[0]
self.assertEquals(self.topic2, fragment2.topic)
def testInsertMultipleFragments(self):
"""Tests inserting multiple fragments on different virtual topics."""
virtual_feed.inject_virtual_feed(
self.topic, self.format, self.header_footer, self.entries_map)
virtual_feed.inject_virtual_feed(
self.topic2, self.format, self.header_footer, self.entries_map)
task1, task2 = testutil.get_tasks('default', expected_count=2)
self.assertTrue(task1['name'].startswith(
'fjq-FeedFragment-54124f41c1ea6e67e4beacac85b9f015e6830d41--'
'my-version-'))
self.assertTrue(task2['name'].startswith(
'fjq-FeedFragment-0449375bf584a7a5d3a09b344a726dead30c3927--'
'my-version-'))
virtual_feed.VIRTUAL_FEED_QUEUE.name = \
'fjq-FeedFragment-54124f41c1ea6e67e4beacac85b9f015e6830d41-'
fragment1 = virtual_feed.VIRTUAL_FEED_QUEUE.pop(task1['name'])[0]
self.assertEquals(self.topic, fragment1.topic)
virtual_feed.VIRTUAL_FEED_QUEUE.name = \
'fjq-FeedFragment-0449375bf584a7a5d3a09b344a726dead30c3927-'
fragment2 = virtual_feed.VIRTUAL_FEED_QUEUE.pop(task2['name'])[0]
self.assertEquals(self.topic2, fragment2.topic)
def testPopAlreadyExists(self):
"""Tests popping work items when the continuation task already exists."""
work_index = TEST_QUEUE.next_index()
tasks = []
for i in xrange(6):
tasks.append(TestModel(work_index=work_index, number=i))
db.put(tasks)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
testutil.get_tasks('default', expected_count=1)
request = testutil.create_test_request('POST', None)
os.environ['HTTP_X_APPENGINE_TASKNAME'] = \
self.expect_task(work_index)['name']
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks('default', expected_count=2)
result_list = TEST_QUEUE.pop_request(request)
testutil.get_tasks('default', expected_count=2)
def testStart(self):
"""Tests starting the mapreduce job."""
job_id = offline_jobs.start_count_subscriptions()
self.assertTrue(job_id is not None)
task = testutil.get_tasks('default', expected_count=1, index=0)
self.assertEquals('/mapreduce/pipeline/run', task['url'])
def testStart(self):
"""Tests starting the mapreduce job."""
job_id = offline_jobs.start_count_subscriptions()
self.assertTrue(job_id is not None)
task = testutil.get_tasks('default', expected_count=1, index=0)
self.assertEquals('/mapreduce/pipeline/run', task['url'])
