def _continue(self, start_key, batch_size):
q = self.get_query()
# If we're resuming, pick up where we left off last time.
if start_key:
q.filter("__key__ >", start_key)
# Keep updating records until we run out of time.
try:
# Steps over the results, returning each entity and its index.
for i, entity in enumerate(q):
result = self.map(entity)
# allow .map() to return either ([updates], [deletes]) or None
map_updates, map_deletes = result or ([], [])
self.to_put.extend(map_updates)
self.to_delete.extend(map_deletes)
# Do updates and deletes in batches.
if (i + 1) >= batch_size:
self._batch_write()
# Record the last entity we processed.
start_key = entity.key()
# Continue deferred...
raise DeadlineExceededError()
self._batch_write()
except DeadlineExceededError:
# Write any unfinished updates to the datastore.
# There is not enough time for this - Google pls.
# When we get DeadlineExceededError, we are already
# down to less than a second, which is too short
# to do a batch write.
#self._batch_write()
# Queue a new task to pick up where we left off.
deferred.defer(self._continue, start_key, batch_size)
return
deferred.defer(self._do_finish)
def test_timer(testbed):
app = webtest.TestApp(main.app)
with mock.patch('main.time.sleep') as sleep_mock:
sleep_mock.side_effect = DeadlineExceededError()
app.get('/timer', status=500)
assert sleep_mock.called
def test_cq_status_fetch_stores_intermediate_status(self):
urlfetch_mock = mock.Mock()
urlfetch_mock.side_effect = [
self._mock_response({
'more': True,
'cursor': 'abcd',
'results': []
}),
self._mock_response({
'more': False,
'cursor': None,
'results': []
}),
]
with mock.patch('google.appengine.api.urlfetch.fetch', urlfetch_mock):
with mock.patch(
'status.cq_status.parse_cq_data') as parse_cq_data_mock:
parse_cq_data_mock.side_effect = [
None, DeadlineExceededError()
]
cq_status.fetch_cq_status()
fetch_status = FetchStatus.query().get()
self.assertEqual(fetch_status.cursor, 'abcd')
self.assertEqual(fetch_status.begin, '1445609862.0')
self.assertEqual(fetch_status.end, '1446214662.0')
self.assertEqual(fetch_status.done, False)
def tickle_entity_volitle(entity):
""" Like `tickle_entity`, but raises DeadlineExceededError every 3rd call. """
call_count = getattr(tickle_entity_volitle, "call_count", 1)
tickle_entity_volitle.call_count = call_count + 1
if call_count % 3 == 0:
raise DeadlineExceededError()
else:
tickle_entity(entity)
def test_cq_status_fetch_captures_deadline_exceeded_errors(self):
urlfetch_mock = mock.Mock()
urlfetch_mock.return_value.content = json.dumps({
'more': False, 'cursor': None, 'results': []})
with mock.patch('google.appengine.api.urlfetch.fetch', urlfetch_mock):
with mock.patch('status.cq_status.parse_cq_data') as parse_cq_data_mock:
parse_cq_data_mock.side_effect = [DeadlineExceededError()]
cq_status.fetch_cq_status()
def ping(self, currtime=None):
"""Enforce the deadline, return time remaining"""
# Allow override for testing
if not currtime:
currtime = datetime.datetime.now()
# Raise artifical deadline
if currtime >= self.deadline:
raise DeadlineExceededError()
# Return time remaining
return self.deadline - currtime
def test_atomic_context_manager_catches_deadlineexceedederror(self):
""" Make sure that DeadlineExceededError causes the transaction to be rolled back when
using atomic() as a context manager.
"""
from .test_connector import TestUser
with self.assertRaises(DeadlineExceededError):
with transaction.atomic():
TestUser.objects.create(username="******", field2="bar")
raise DeadlineExceededError()
self.assertEqual(0, TestUser.objects.count())
def test_send_fail_deadline_error(self):
message = WebhookRequest(
MockNotification(webhook_message_data={'data': 'value'}),
'https://www.thebluealliance.com', 'secret')
error_mock = Mock()
error_mock.side_effect = DeadlineExceededError('testing')
with patch.object(urllib2, 'urlopen',
error_mock) as mock_urlopen, patch.object(
message,
'defer_track_notification') as mock_track:
success = message.send()
mock_urlopen.assert_called_once()
mock_track.assert_not_called()
self.assertTrue(success)
# -*- coding: utf-8 -*- """ A urls module that raises a DeadlineExceededError :Copyright: (c) 2010 Ian Lewis <*****@*****.**> All rights reserved. :license: BSD, see LICENSE for more details. """ from google.appengine.runtime import DeadlineExceededError raise DeadlineExceededError()
def txn():
TestUser.objects.create(username="******", field2="bar")
self.assertTrue(transaction.in_atomic_block())
raise DeadlineExceededError()