def test_reporting(self, patch_post, patch_add_event):
health_service = HealthService('endpoint')
health_service.report_host_plugin_status(is_healthy=True, response='response')
self.assertEqual(1, patch_post.call_count)
self.assertEqual(0, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_STATUS_OBSERVATION_NAME,
is_healthy=True,
value='response',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_status(is_healthy=False, response='error')
self.assertEqual(2, patch_post.call_count)
self.assertEqual(1, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args, response='error')
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_STATUS_OBSERVATION_NAME,
is_healthy=False,
value='error',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_extension_artifact(is_healthy=True, source='source', response='response')
self.assertEqual(3, patch_post.call_count)
self.assertEqual(1, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_ARTIFACT_OBSERVATION_NAME,
is_healthy=True,
value='response',
description='source')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_extension_artifact(is_healthy=False, source='source', response='response')
self.assertEqual(4, patch_post.call_count)
self.assertEqual(2, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args, response='response')
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_ARTIFACT_OBSERVATION_NAME,
is_healthy=False,
value='response',
description='source')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_heartbeat(is_healthy=True)
self.assertEqual(5, patch_post.call_count)
self.assertEqual(2, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_HEARTBEAT_OBSERVATION_NAME,
is_healthy=True,
value='',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_heartbeat(is_healthy=False)
self.assertEqual(3, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args)
self.assertEqual(6, patch_post.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_HEARTBEAT_OBSERVATION_NAME,
is_healthy=False,
value='',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_versions(is_healthy=True, response='response')
self.assertEqual(7, patch_post.call_count)
self.assertEqual(3, patch_add_event.call_count)
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_VERSIONS_OBSERVATION_NAME,
is_healthy=True,
value='response',
description='')
self.assertEqual(0, len(health_service.observations))
health_service.report_host_plugin_versions(is_healthy=False, response='response')
self.assertEqual(8, patch_post.call_count)
self.assertEqual(4, patch_add_event.call_count)
self.assert_telemetry(call_args=patch_add_event.call_args, response='response')
self.assert_observation(call_args=patch_post.call_args,
name=HealthService.HOST_PLUGIN_VERSIONS_OBSERVATION_NAME,
is_healthy=False,
value='response',
description='')
self.assertEqual(0, len(health_service.observations))
patch_post.side_effect = HttpError()
health_service.report_host_plugin_versions(is_healthy=True, response='')
self.assertEqual(9, patch_post.call_count)
self.assertEqual(4, patch_add_event.call_count)
self.assertEqual(0, len(health_service.observations))
class HostPluginProtocol(object):
_is_default_channel = False
FETCH_REPORTING_PERIOD = datetime.timedelta(minutes=1)
STATUS_REPORTING_PERIOD = datetime.timedelta(minutes=1)
def __init__(self, endpoint, container_id, role_config_name):
if endpoint is None:
raise ProtocolError("HostGAPlugin: Endpoint not provided")
self.is_initialized = False
self.is_available = False
self.api_versions = None
self.endpoint = endpoint
self.container_id = container_id
self.deployment_id = None
self.role_config_name = role_config_name
self.manifest_uri = None
self.health_service = HealthService(endpoint)
self.fetch_error_state = ErrorState(
min_timedelta=ERROR_STATE_HOST_PLUGIN_FAILURE)
self.status_error_state = ErrorState(
min_timedelta=ERROR_STATE_HOST_PLUGIN_FAILURE)
self.fetch_last_timestamp = None
self.status_last_timestamp = None
@staticmethod
def is_default_channel():
return HostPluginProtocol._is_default_channel
@staticmethod
def set_default_channel(is_default):
HostPluginProtocol._is_default_channel = is_default
def ensure_initialized(self):
if not self.is_initialized:
self.api_versions = self.get_api_versions()
self.is_available = API_VERSION in self.api_versions
self.is_initialized = self.is_available
from azurelinuxagent.common.event import WALAEventOperation, report_event
report_event(WALAEventOperation.InitializeHostPlugin,
is_success=self.is_available)
return self.is_available
def get_health(self):
"""
Call the /health endpoint
:return: True if 200 received, False otherwise
"""
url = URI_FORMAT_HEALTH.format(self.endpoint, HOST_PLUGIN_PORT)
logger.verbose("HostGAPlugin: Getting health from [{0}]", url)
status_ok = False
try:
response = restutil.http_get(url, max_retry=1)
status_ok = restutil.request_succeeded(response)
except HttpError as e:
logger.verbose("HostGAPlugin: Exception getting health", ustr(e))
return status_ok
def get_api_versions(self):
url = URI_FORMAT_GET_API_VERSIONS.format(self.endpoint,
HOST_PLUGIN_PORT)
logger.verbose(
"HostGAPlugin: Getting API versions at [{0}]".format(url))
return_val = []
error_response = ''
is_healthy = False
try:
headers = {HEADER_CONTAINER_ID: self.container_id}
response = restutil.http_get(url, headers)
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
logger.error(
"HostGAPlugin: Failed Get API versions: {0}".format(
error_response))
else:
return_val = ustr(remove_bom(response.read()),
encoding='utf-8')
is_healthy = True
except HttpError as e:
logger.error(
"HostGAPlugin: Exception Get API versions: {0}".format(e))
self.health_service.report_host_plugin_versions(
is_healthy=is_healthy, response=error_response)
return return_val
def get_artifact_request(self, artifact_url, artifact_manifest_url=None):
if not self.ensure_initialized():
raise ProtocolError(
"HostGAPlugin: Host plugin channel is not available")
if textutil.is_str_none_or_whitespace(artifact_url):
raise ProtocolError(
"HostGAPlugin: No extension artifact url was provided")
url = URI_FORMAT_GET_EXTENSION_ARTIFACT.format(self.endpoint,
HOST_PLUGIN_PORT)
headers = {
HEADER_VERSION: API_VERSION,
HEADER_CONTAINER_ID: self.container_id,
HEADER_HOST_CONFIG_NAME: self.role_config_name,
HEADER_ARTIFACT_LOCATION: artifact_url
}
if artifact_manifest_url is not None:
headers[HEADER_ARTIFACT_MANIFEST_LOCATION] = artifact_manifest_url
return url, headers
def report_fetch_health(self,
uri,
is_healthy=True,
source='',
response=''):
if uri != URI_FORMAT_GET_EXTENSION_ARTIFACT.format(
self.endpoint, HOST_PLUGIN_PORT):
return
if self.should_report(is_healthy, self.fetch_error_state,
self.fetch_last_timestamp,
HostPluginProtocol.FETCH_REPORTING_PERIOD):
self.fetch_last_timestamp = datetime.datetime.utcnow()
health_signal = self.fetch_error_state.is_triggered() is False
self.health_service.report_host_plugin_extension_artifact(
is_healthy=health_signal, source=source, response=response)
def report_status_health(self, is_healthy, response=''):
if self.should_report(is_healthy, self.status_error_state,
self.status_last_timestamp,
HostPluginProtocol.STATUS_REPORTING_PERIOD):
self.status_last_timestamp = datetime.datetime.utcnow()
health_signal = self.status_error_state.is_triggered() is False
self.health_service.report_host_plugin_status(
is_healthy=health_signal, response=response)
@staticmethod
def should_report(is_healthy, error_state, last_timestamp, period):
"""
Determine whether a health signal should be reported
:param is_healthy: whether the current measurement is healthy
:param error_state: the error state which is tracking time since failure
:param last_timestamp: the last measurement time stamp
:param period: the reporting period
:return: True if the signal should be reported, False otherwise
"""
if is_healthy:
# we only reset the error state upon success, since we want to keep
# reporting the failure; this is different to other uses of error states
# which do not have a separate periodicity
error_state.reset()
else:
error_state.incr()
if last_timestamp is None:
last_timestamp = datetime.datetime.utcnow() - period
return datetime.datetime.utcnow() >= (last_timestamp + period)
def put_vm_log(self, content):
raise NotImplementedError("Unimplemented")
def put_vm_status(self, status_blob, sas_url, config_blob_type=None):
"""
Try to upload the VM status via the host plugin /status channel
:param sas_url: the blob SAS url to pass to the host plugin
:param config_blob_type: the blob type from the extension config
:type status_blob: StatusBlob
"""
if not self.ensure_initialized():
raise ProtocolError("HostGAPlugin: HostGAPlugin is not available")
if status_blob is None or status_blob.vm_status is None:
raise ProtocolError("HostGAPlugin: Status blob was not provided")
logger.verbose("HostGAPlugin: Posting VM status")
blob_type = status_blob.type if status_blob.type else config_blob_type
if blob_type == "BlockBlob":
self._put_block_blob_status(sas_url, status_blob)
else:
self._put_page_blob_status(sas_url, status_blob)
def _put_block_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_block_blob_headers(
len(status_blob.data)),
bytearray(status_blob.data,
encoding='utf-8')),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy,
response=error_response)
raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}".format(
error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
def _put_page_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
# Convert the status into a blank-padded string whose length is modulo 512
status = bytearray(status_blob.data, encoding='utf-8')
status_size = int((len(status) + 511) / 512) * 512
status = bytearray(status_blob.data.ljust(status_size),
encoding='utf-8')
# First, initialize an empty blob
response = restutil.http_put(
url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_create_headers(status_size)),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy,
response=error_response)
raise HttpError(
"HostGAPlugin: Failed PageBlob clean-up: {0}".format(
error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: PageBlob clean-up succeeded")
# Then, upload the blob in pages
if sas_url.count("?") <= 0:
sas_url = "{0}?comp=page".format(sas_url)
else:
sas_url = "{0}&comp=page".format(sas_url)
start = 0
end = 0
while start < len(status):
# Create the next page
end = start + min(len(status) - start, MAXIMUM_PAGEBLOB_PAGE_SIZE)
page_size = int((end - start + 511) / 512) * 512
buf = bytearray(page_size)
buf[0:end - start] = status[start:end]
# Send the page
response = restutil.http_put(
url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_page_headers(start, end), buf),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(
response)
self.report_status_health(is_healthy=is_healthy,
response=error_response)
raise HttpError("HostGAPlugin Error: Put PageBlob bytes "
"[{0},{1}]: {2}".format(
start, end, error_response))
# Advance to the next page (if any)
start = end
def _build_status_data(self, sas_url, blob_headers, content=None):
headers = []
for name in iter(blob_headers.keys()):
headers.append({
'headerName': name,
'headerValue': blob_headers[name]
})
data = {'requestUri': sas_url, 'headers': headers}
if not content is None:
data['content'] = self._base64_encode(content)
return json.dumps(data, sort_keys=True)
def _build_status_headers(self):
return {
HEADER_VERSION: API_VERSION,
"Content-type": "application/json",
HEADER_CONTAINER_ID: self.container_id,
HEADER_HOST_CONFIG_NAME: self.role_config_name
}
def _base64_encode(self, data):
s = base64.b64encode(bytes(data))
if PY_VERSION_MAJOR > 2:
return s.decode('utf-8')
return s
class HostPluginProtocol(object):
_is_default_channel = False
FETCH_REPORTING_PERIOD = datetime.timedelta(minutes=1)
STATUS_REPORTING_PERIOD = datetime.timedelta(minutes=1)
def __init__(self, endpoint, container_id, role_config_name):
if endpoint is None:
raise ProtocolError("HostGAPlugin: Endpoint not provided")
self.is_initialized = False
self.is_available = False
self.api_versions = None
self.endpoint = endpoint
self.container_id = container_id
self.deployment_id = None
self.role_config_name = role_config_name
self.manifest_uri = None
self.health_service = HealthService(endpoint)
self.fetch_error_state = ErrorState(min_timedelta=ERROR_STATE_HOST_PLUGIN_FAILURE)
self.status_error_state = ErrorState(min_timedelta=ERROR_STATE_HOST_PLUGIN_FAILURE)
self.fetch_last_timestamp = None
self.status_last_timestamp = None
@staticmethod
def is_default_channel():
return HostPluginProtocol._is_default_channel
@staticmethod
def set_default_channel(is_default):
HostPluginProtocol._is_default_channel = is_default
def ensure_initialized(self):
if not self.is_initialized:
self.api_versions = self.get_api_versions()
self.is_available = API_VERSION in self.api_versions
self.is_initialized = self.is_available
from azurelinuxagent.common.event import WALAEventOperation, report_event
report_event(WALAEventOperation.InitializeHostPlugin,
is_success=self.is_available)
return self.is_available
def get_health(self):
"""
Call the /health endpoint
:return: True if 200 received, False otherwise
"""
url = URI_FORMAT_HEALTH.format(self.endpoint,
HOST_PLUGIN_PORT)
logger.verbose("HostGAPlugin: Getting health from [{0}]", url)
response = restutil.http_get(url, max_retry=1)
return restutil.request_succeeded(response)
def get_api_versions(self):
url = URI_FORMAT_GET_API_VERSIONS.format(self.endpoint,
HOST_PLUGIN_PORT)
logger.verbose("HostGAPlugin: Getting API versions at [{0}]"
.format(url))
return_val = []
error_response = ''
is_healthy = False
try:
headers = {HEADER_CONTAINER_ID: self.container_id}
response = restutil.http_get(url, headers)
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
logger.error("HostGAPlugin: Failed Get API versions: {0}".format(error_response))
is_healthy = not restutil.request_failed_at_hostplugin(response)
else:
return_val = ustr(remove_bom(response.read()), encoding='utf-8')
is_healthy = True
except HttpError as e:
logger.error("HostGAPlugin: Exception Get API versions: {0}".format(e))
self.health_service.report_host_plugin_versions(is_healthy=is_healthy, response=error_response)
return return_val
def get_artifact_request(self, artifact_url, artifact_manifest_url=None):
if not self.ensure_initialized():
raise ProtocolError("HostGAPlugin: Host plugin channel is not available")
if textutil.is_str_none_or_whitespace(artifact_url):
raise ProtocolError("HostGAPlugin: No extension artifact url was provided")
url = URI_FORMAT_GET_EXTENSION_ARTIFACT.format(self.endpoint,
HOST_PLUGIN_PORT)
headers = {HEADER_VERSION: API_VERSION,
HEADER_CONTAINER_ID: self.container_id,
HEADER_HOST_CONFIG_NAME: self.role_config_name,
HEADER_ARTIFACT_LOCATION: artifact_url}
if artifact_manifest_url is not None:
headers[HEADER_ARTIFACT_MANIFEST_LOCATION] = artifact_manifest_url
return url, headers
def report_fetch_health(self, uri, is_healthy=True, source='', response=''):
if uri != URI_FORMAT_GET_EXTENSION_ARTIFACT.format(self.endpoint, HOST_PLUGIN_PORT):
return
if self.should_report(is_healthy,
self.fetch_error_state,
self.fetch_last_timestamp,
HostPluginProtocol.FETCH_REPORTING_PERIOD):
self.fetch_last_timestamp = datetime.datetime.utcnow()
health_signal = self.fetch_error_state.is_triggered() is False
self.health_service.report_host_plugin_extension_artifact(is_healthy=health_signal,
source=source,
response=response)
def report_status_health(self, is_healthy, response=''):
if self.should_report(is_healthy,
self.status_error_state,
self.status_last_timestamp,
HostPluginProtocol.STATUS_REPORTING_PERIOD):
self.status_last_timestamp = datetime.datetime.utcnow()
health_signal = self.status_error_state.is_triggered() is False
self.health_service.report_host_plugin_status(is_healthy=health_signal,
response=response)
@staticmethod
def should_report(is_healthy, error_state, last_timestamp, period):
"""
Determine whether a health signal should be reported
:param is_healthy: whether the current measurement is healthy
:param error_state: the error state which is tracking time since failure
:param last_timestamp: the last measurement time stamp
:param period: the reporting period
:return: True if the signal should be reported, False otherwise
"""
if is_healthy:
# we only reset the error state upon success, since we want to keep
# reporting the failure; this is different to other uses of error states
# which do not have a separate periodicity
error_state.reset()
else:
error_state.incr()
if last_timestamp is None:
last_timestamp = datetime.datetime.utcnow() - period
return datetime.datetime.utcnow() >= (last_timestamp + period)
def put_vm_log(self, content):
raise NotImplementedError("Unimplemented")
def put_vm_status(self, status_blob, sas_url, config_blob_type=None):
"""
Try to upload the VM status via the host plugin /status channel
:param sas_url: the blob SAS url to pass to the host plugin
:param config_blob_type: the blob type from the extension config
:type status_blob: StatusBlob
"""
if not self.ensure_initialized():
raise ProtocolError("HostGAPlugin: HostGAPlugin is not available")
if status_blob is None or status_blob.vm_status is None:
raise ProtocolError("HostGAPlugin: Status blob was not provided")
logger.verbose("HostGAPlugin: Posting VM status")
blob_type = status_blob.type if status_blob.type else config_blob_type
if blob_type == "BlockBlob":
self._put_block_blob_status(sas_url, status_blob)
else:
self._put_page_blob_status(sas_url, status_blob)
def _put_block_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_block_blob_headers(len(status_blob.data)),
bytearray(status_blob.data, encoding='utf-8')),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}"
.format(error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
def _put_page_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
# Convert the status into a blank-padded string whose length is modulo 512
status = bytearray(status_blob.data, encoding='utf-8')
status_size = int((len(status) + 511) / 512) * 512
status = bytearray(status_blob.data.ljust(status_size), encoding='utf-8')
# First, initialize an empty blob
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_create_headers(status_size)),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError("HostGAPlugin: Failed PageBlob clean-up: {0}"
.format(error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: PageBlob clean-up succeeded")
# Then, upload the blob in pages
if sas_url.count("?") <= 0:
sas_url = "{0}?comp=page".format(sas_url)
else:
sas_url = "{0}&comp=page".format(sas_url)
start = 0
end = 0
while start < len(status):
# Create the next page
end = start + min(len(status) - start, MAXIMUM_PAGEBLOB_PAGE_SIZE)
page_size = int((end - start + 511) / 512) * 512
buf = bytearray(page_size)
buf[0: end - start] = status[start: end]
# Send the page
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_page_headers(start, end),
buf),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError(
"HostGAPlugin Error: Put PageBlob bytes "
"[{0},{1}]: {2}".format(start, end, error_response))
# Advance to the next page (if any)
start = end
def _build_status_data(self, sas_url, blob_headers, content=None):
headers = []
for name in iter(blob_headers.keys()):
headers.append({
'headerName': name,
'headerValue': blob_headers[name]
})
data = {
'requestUri': sas_url,
'headers': headers
}
if not content is None:
data['content'] = self._base64_encode(content)
return json.dumps(data, sort_keys=True)
def _build_status_headers(self):
return {
HEADER_VERSION: API_VERSION,
"Content-type": "application/json",
HEADER_CONTAINER_ID: self.container_id,
HEADER_HOST_CONFIG_NAME: self.role_config_name
}
def _base64_encode(self, data):
s = base64.b64encode(bytes(data))
if PY_VERSION_MAJOR > 2:
return s.decode('utf-8')
return s