def init_protocols(self):
# The initialization of ProtocolUtil for the Monitor thread should be done within the thread itself rather
# than initializing it in the ExtHandler thread. This is done to avoid any concurrency issues as each
# thread would now have its own ProtocolUtil object as per the SingletonPerThread model.
self.protocol_util = get_protocol_util()
self.protocol = self.protocol_util.get_protocol()
self.health_service = HealthService(self.protocol.get_endpoint())
def test_it_should_not_send_a_health_signal_when_the_hearbeat_fails(self):
with _mock_wire_protocol() as protocol:
with patch('azurelinuxagent.common.event.EventLogger.add_event') as add_event_patcher:
health_service_post_requests = []
def http_get_handler(url, *_, **__):
if self.is_host_plugin_health_request(url):
del health_service_post_requests[:] # clear the requests; after this error there should be no more POSTs
raise IOError('A CLIENT ERROR')
return None
def http_post_handler(url, _, **__):
if self.is_health_service_request(url):
health_service_post_requests.append(url)
return MockHttpResponse(status=200)
return None
protocol.set_http_handlers(http_get_handler=http_get_handler, http_post_handler=http_post_handler)
health_service = HealthService(protocol.get_endpoint())
SendHostPluginHeartbeat(protocol, health_service).run()
self.assertEqual(0, len(health_service_post_requests), "No health signals should have been posted: {0}".format(health_service_post_requests))
heartbeat_events = [kwargs for _, kwargs in add_event_patcher.call_args_list if kwargs['op'] == 'HostPluginHeartbeat']
self.assertTrue(len(heartbeat_events) == 1, "The monitor thread should have reported exactly 1 telemetry event for an unhealthy host ga plugin")
self.assertFalse(heartbeat_events[0]['is_success'], 'The reported event should indicate failure')
self.assertIn('A CLIENT ERROR', heartbeat_events[0]['message'], 'The failure does not include the expected message')
def daemon(self):
try:
# The protocol needs to be instantiated in the monitor thread itself (to avoid concurrency issues with the protocol object each
# thread uses a different instance as per the SingletonPerThread model.
protocol_util = get_protocol_util()
protocol = protocol_util.get_protocol()
health_service = HealthService(protocol.get_endpoint())
periodic_operations = [
ResetPeriodicLogMessages(),
ReportNetworkErrors(),
PollResourceUsage(),
SendHostPluginHeartbeat(protocol, health_service),
SendImdsHeartbeat(protocol_util, health_service)
]
report_network_configuration_changes = ReportNetworkConfigurationChanges()
if conf.get_monitor_network_configuration_changes():
periodic_operations.append(report_network_configuration_changes)
else:
logger.info("Monitor.NetworkConfigurationChanges is disabled.")
report_network_configuration_changes.log_network_configuration()
while not self.stopped():
try:
for op in periodic_operations:
op.run()
finally:
PeriodicOperation.sleep_until_next_operation(periodic_operations)
except Exception as e:
logger.error("An error occurred in the monitor thread; will exit the thread.\n{0}", ustr(e))
def test_it_should_report_a_telemetry_event_when_host_plugin_is_not_healthy(
self):
with _mock_wire_protocol() as protocol:
# the error triggers only after ERROR_STATE_DELTA_DEFAULT
with patch(
'azurelinuxagent.common.errorstate.ErrorState.is_triggered',
return_value=True):
with patch('azurelinuxagent.common.event.EventLogger.add_event'
) as add_event_patcher:
def http_get_handler(url, *_, **__):
if self.is_host_plugin_health_request(url):
return MockHttpResponse(status=503)
return None
protocol.set_http_handlers(
http_get_handler=http_get_handler)
health_service = HealthService(protocol.get_endpoint())
SendHostPluginHeartbeat(protocol, health_service).run()
heartbeat_events = [
kwargs
for _, kwargs in add_event_patcher.call_args_list
if kwargs['op'] == 'HostPluginHeartbeatExtended'
]
self.assertTrue(
len(heartbeat_events) == 1,
"The monitor thread should have reported exactly 1 telemetry event for an unhealthy host ga plugin"
)
self.assertFalse(
heartbeat_events[0]['is_success'],
'The reported event should indicate failure')
def test_observation_length(self):
health_service = HealthService('endpoint')
# make 100 observations
for i in range(0, 100):
health_service._observe(is_healthy=True, name='{0}'.format(i))
# ensure we keep only 10
self.assertEqual(10, len(health_service.observations))
# ensure we keep the most recent 10
self.assertEqual('90', health_service.observations[0].name)
self.assertEqual('99', health_service.observations[9].name)
def test_it_should_report_host_ga_health(self):
with _mock_wire_protocol() as protocol:
def http_post_handler(url, _, **__):
if self.is_health_service_request(url):
http_post_handler.health_service_posted = True
return MockHttpResponse(status=200)
return None
http_post_handler.health_service_posted = False
protocol.set_http_handlers(http_post_handler=http_post_handler)
health_service = HealthService(protocol.get_endpoint())
SendHostPluginHeartbeat(protocol, health_service).run()
self.assertTrue(http_post_handler.health_service_posted, "The monitor thread did not report host ga plugin health")
def __init__(self, endpoint, container_id, role_config_name):
if endpoint is None:
raise ProtocolError("HostGAPlugin: Endpoint not provided")
self.is_initialized = False
self.is_available = False
self.api_versions = None
self.endpoint = endpoint
self.container_id = container_id
self.deployment_id = None
self.role_config_name = role_config_name
self.manifest_uri = None
self.health_service = HealthService(endpoint)
self.fetch_error_state = ErrorState(min_timedelta=ERROR_STATE_HOST_PLUGIN_FAILURE)
self.status_error_state = ErrorState(min_timedelta=ERROR_STATE_HOST_PLUGIN_FAILURE)
self.fetch_last_timestamp = None
self.status_last_timestamp = None
def test_observation_json(self):
health_service = HealthService('endpoint')
health_service.observations.append(Observation(name='name',
is_healthy=True,
value='value',
description='description'))
expected_json = '{"Source": "WALinuxAgent", ' \
'"Api": "reporttargethealth", ' \
'"Version": "1.0", ' \
'"Observations": [{' \
'"Value": "value", ' \
'"ObservationName": "name", ' \
'"Description": "description", ' \
'"IsHealthy": true' \
'}]}'
expected = sorted(json.loads(expected_json).items())
actual = sorted(json.loads(health_service.as_json).items())
self.assertEqual(expected, actual)
def init_protocols(self):
self.protocol = self.protocol_util.get_protocol()
self.health_service = HealthService(self.protocol.endpoint)
def test_reporting(self, patch_post, patch_add_event):
health_service = HealthService('endpoint')
health_service.report_host_plugin_status(is_healthy=True, response='response')
self.assertEqual(1, patch_post.call_count)
self.assertEqual(0, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_STATUS_OBSERVATION_NAME,
is_healthy=True,
value='response',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_status(is_healthy=False, response='error')
self.assertEqual(2, patch_post.call_count)
self.assertEqual(1, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args, response='error')
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_STATUS_OBSERVATION_NAME,
is_healthy=False,
value='error',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_extension_artifact(is_healthy=True, source='source', response='response')
self.assertEqual(3, patch_post.call_count)
self.assertEqual(1, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_ARTIFACT_OBSERVATION_NAME,
is_healthy=True,
value='response',
description='source')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_extension_artifact(is_healthy=False, source='source', response='response')
self.assertEqual(4, patch_post.call_count)
self.assertEqual(2, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args, response='response')
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_ARTIFACT_OBSERVATION_NAME,
is_healthy=False,
value='response',
description='source')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_heartbeat(is_healthy=True)
self.assertEqual(5, patch_post.call_count)
self.assertEqual(2, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_HEARTBEAT_OBSERVATION_NAME,
is_healthy=True,
value='',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_heartbeat(is_healthy=False)
self.assertEqual(3, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args)
self.assertEqual(6, patch_post.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_HEARTBEAT_OBSERVATION_NAME,
is_healthy=False,
value='',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_versions(is_healthy=True, response='response')
self.assertEqual(7, patch_post.call_count)
self.assertEqual(3, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_VERSIONS_OBSERVATION_NAME,
is_healthy=True,
value='response',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_versions(is_healthy=False, response='response')
self.assertEqual(8, patch_post.call_count)
self.assertEqual(4, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args, response='response')
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_VERSIONS_OBSERVATION_NAME,
is_healthy=False,
value='response',
description='')
self.assertEqual(0, len(health_service.observations))
patch_post.side_effect = HttpError()
health_service.report_host_plugin_versions(is_healthy=True, response='')
self.assertEqual(9, patch_post.call_count)
self.assertEqual(4, patch_add_event.call_count)
self.assertEqual(0, len(health_service.observations))