def raise_error(self):
raise HttpError('blam')
def fake_post(envelope):
if len(json.dumps(envelope)) > MAX_BYTES_PER_POST:
daemon_log.info("fake_post(): rejecting oversized message")
raise HttpError()
def raise_error(self):
raise HttpError("blam")
def test_session_backoff(self):
"""Test that when messages to the manager are being dropped due to POST failure,
sending SESSION_CREATE_REQUEST messages has a power-of-two backoff wait"""
client = mock.Mock()
client._fqdn = "test_server"
client.boot_time = IMLDateTime.utcnow()
client.start_time = IMLDateTime.utcnow()
writer = client.writer = HttpWriter(client)
reader = client.reader = HttpReader(client)
daemon_log.setLevel(logging.DEBUG)
TestPlugin = mock.Mock()
mock_plugin_instance = mock.Mock()
mock_plugin_instance.start_session = mock.Mock(
return_value={'foo': 'bar'})
client.device_plugins.get = mock.Mock(
return_value=lambda (plugin_name): mock_plugin_instance)
client.device_plugins.get_plugins = mock.Mock(
return_value={'test_plugin': TestPlugin})
client.sessions = SessionTable(client)
client.post = mock.Mock(side_effect=HttpError())
# Pick an arbitrary time to use as a base for simulated waits
t_0 = datetime.datetime.now()
old_datetime = datetime.datetime
try:
def expect_message_at(t):
datetime.datetime.now = mock.Mock(
return_value=t - datetime.timedelta(seconds=0.1))
writer.poll('test_plugin')
self.assertEqual(writer._messages.qsize(), 0)
datetime.datetime.now = mock.Mock(
return_value=t + datetime.timedelta(seconds=0.1))
writer.poll('test_plugin')
self.assertEqual(writer._messages.qsize(), 1)
datetime.datetime = mock.Mock()
datetime.datetime.now = mock.Mock(return_value=t_0)
# Stage 1: failing to POST, backing off
# =====================================
# Poll should put some session creation messages
writer.poll('test_plugin')
self.assertEqual(writer._messages.qsize(), 1)
# Another poll immediately after shouldn't add any messages (MIN_SESSION_BACKOFF hasn't passed)
writer.poll('test_plugin')
self.assertEqual(writer._messages.qsize(), 1)
# Send should consume the messages, and they go to nowhere because the POST fails
writer.send()
client.post.assert_called_once()
self.assertEqual(len(client.post.call_args[0][0]['messages']), 1)
# First time boundary: where the first repeat should happen
from chroma_agent.agent_client import MIN_SESSION_BACKOFF
t_1 = t_0 + MIN_SESSION_BACKOFF
expect_message_at(t_1)
# Have another crack at sending, it should fail and empty the queue
writer.send()
self.assertTrue(writer._messages.empty())
# Second time boundary: where the second repeat should happen
t_2 = t_1 + MIN_SESSION_BACKOFF * 2
expect_message_at(t_2)
# Stage 2: success in POST, session creation
# ==========================================
# This time we'll let the message go through, and a session to begin.
client.post = mock.Mock()
writer.send()
# HttpReader receives a response from the manager, and should reset the backoff counters.
reader._handle_messages([{
'type': 'SESSION_CREATE_RESPONSE',
'plugin': 'test_plugin',
'session_id': 'id_foo',
'session_seq': 0,
'body': {}
}])
self.assertEqual(len(client.sessions._sessions), 1)
session = client.sessions.get('test_plugin')
# State 3: POSTs start failing again, see delay again
# ===================================================
# Break the POST link again
client.post = mock.Mock(side_effect=HttpError())
# Poll will get a DATA message from initial_scan
session.initial_scan = mock.Mock(return_value={'foo': 'bar'})
writer.poll('test_plugin')
session.initial_scan.assert_called_once()
self.assertFalse(writer._messages.empty())
# Send will fail to send it, and as a result destroy the session
writer.send()
self.assertTrue(writer._messages.empty())
self.assertEqual(len(client.sessions._sessions), 0)
# Move to some point beyond the first backoff cycle
t_3 = t_0 + datetime.timedelta(seconds=60)
datetime.datetime.now = mock.Mock(return_value=t_3)
writer.poll('test_plugin')
self.assertEqual(writer._messages.qsize(), 1)
writer.poll('test_plugin')
self.assertEqual(writer._messages.qsize(), 1)
writer.send()
self.assertEqual(writer._messages.qsize(), 0)
# Check the backoff time has gone back to MIN_SESSION_BACKOFF
t_4 = t_3 + MIN_SESSION_BACKOFF
expect_message_at(t_4)
finally:
datetime.datetime = old_datetime