def testLargePermanentFailure(self):
"""Test when DynamoDB limits the size of the traceback of a failing op."""
def _BuggyOpMethod(client, callback, blob):
raise Exception('some permanent failure')
op = self._CreateTestOp(user_id=1, handler=_BuggyOpMethod, blob='A' * (UserOpManager._MAX_OPERATION_SIZE - 200))
user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_BuggyOpMethod], callback=self.stop)
user_op_mgr.Execute()
self.wait()
Operation.Query(self._client, 1, op.operation_id, None, lambda op: self.stop(op))
op = self.wait()
self.assertIsNotNone(op.first_failure)
self.assertIsNotNone(op.last_failure)
self.assertLessEqual(len(op.first_failure), 100)
self.assertLessEqual(len(op.last_failure), 100)
def testAbortOfPermissionError(self):
"""Test op that hits an abortable error."""
def _BuggyOpMethod(client, callback):
self._method_count += 1
# PermissionError is one of the exceptions which qualifies as abortable.
raise PermissionError('Not Authorized')
op = self._CreateTestOp(user_id=1, handler=_BuggyOpMethod)
user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_BuggyOpMethod], callback=self.stop)
user_op_mgr.Execute()
self.wait()
self.assertEqual(self._method_count, 1)
# Ensure that operation does not exist in the db.
Operation.Query(self._client, 1, op.operation_id, None, lambda op: self.stop(op), must_exist=False)
op = self.wait()
self.assertTrue(op is None)
def testPermanentFailure(self):
"""Test op that continually fails."""
def _BuggyOpMethod(client, callback):
self._method_count += 1
raise Exception('some permanent failure')
op = self._CreateTestOp(user_id=1, handler=_BuggyOpMethod)
user_op_mgr = self._CreateUserOpManager(user_id=1, handlers=[_BuggyOpMethod], callback=self.stop)
user_op_mgr.Execute()
self.wait()
self.assertEqual(self._method_count, 3)
# Ensure that operation still exists in db with right values.
Operation.Query(self._client, 1, op.operation_id, None, lambda op: self.stop(op))
op = self.wait()
self.assertIsNotNone(op.first_failure)
self.assertIsNotNone(op.last_failure)
self.assertEqual(op.attempts, 3)
self.assertEqual(op.quarantine, 1)
self.assertGreater(op.backoff, 0)