def _put_page_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
# Convert the status into a blank-padded string whose length is modulo 512
status = bytearray(status_blob.data, encoding='utf-8')
status_size = int((len(status) + 511) / 512) * 512
status = bytearray(status_blob.data.ljust(status_size), encoding='utf-8')
# First, initialize an empty blob
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_create_headers(status_size)),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError("HostGAPlugin: Failed PageBlob clean-up: {0}"
.format(error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: PageBlob clean-up succeeded")
# Then, upload the blob in pages
if sas_url.count("?") <= 0:
sas_url = "{0}?comp=page".format(sas_url)
else:
sas_url = "{0}&comp=page".format(sas_url)
start = 0
end = 0
while start < len(status):
# Create the next page
end = start + min(len(status) - start, MAXIMUM_PAGEBLOB_PAGE_SIZE)
page_size = int((end - start + 511) / 512) * 512
buf = bytearray(page_size)
buf[0: end - start] = status[start: end]
# Send the page
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_page_headers(start, end),
buf),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError(
"HostGAPlugin Error: Put PageBlob bytes "
"[{0},{1}]: {2}".format(start, end, error_response))
# Advance to the next page (if any)
start = end
def _put_page_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
# Convert the status into a blank-padded string whose length is modulo 512
status = bytearray(status_blob.data, encoding='utf-8')
status_size = int((len(status) + 511) / 512) * 512
status = bytearray(status_blob.data.ljust(status_size), encoding='utf-8')
# First, initialize an empty blob
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_create_headers(status_size)),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError("HostGAPlugin: Failed PageBlob clean-up: {0}"
.format(error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: PageBlob clean-up succeeded")
# Then, upload the blob in pages
if sas_url.count("?") <= 0:
sas_url = "{0}?comp=page".format(sas_url)
else:
sas_url = "{0}&comp=page".format(sas_url)
start = 0
end = 0
while start < len(status):
# Create the next page
end = start + min(len(status) - start, MAXIMUM_PAGEBLOB_PAGE_SIZE)
page_size = int((end - start + 511) / 512) * 512
buf = bytearray(page_size)
buf[0: end - start] = status[start: end]
# Send the page
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_page_blob_page_headers(start, end),
buf),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError(
"HostGAPlugin Error: Put PageBlob bytes "
"[{0},{1}]: {2}".format(start, end, error_response))
# Advance to the next page (if any)
start = end
def test_request_failed(self):
self.assertTrue(restutil.request_failed(None))
resp = Mock()
for status in restutil.OK_CODES:
resp.status = status
self.assertFalse(restutil.request_failed(resp))
self.assertFalse(httpclient.BAD_REQUEST in restutil.OK_CODES)
resp.status = httpclient.BAD_REQUEST
self.assertTrue(restutil.request_failed(resp))
self.assertFalse(
restutil.request_failed(resp, ok_codes=[httpclient.BAD_REQUEST]))
def test_request_failed(self):
self.assertTrue(restutil.request_failed(None))
resp = Mock()
for status in restutil.OK_CODES:
resp.status = status
self.assertFalse(restutil.request_failed(resp))
self.assertFalse(httpclient.BAD_REQUEST in restutil.OK_CODES)
resp.status = httpclient.BAD_REQUEST
self.assertTrue(restutil.request_failed(resp))
self.assertFalse(
restutil.request_failed(
resp, ok_codes=[httpclient.BAD_REQUEST]))
def _get_metadata_from_endpoint(self, endpoint, resource_path, headers):
"""
Get metadata from one of the IMDS endpoints.
:param str endpoint: IMDS endpoint to call
:param str resource_path: path of IMDS resource
:param bool headers: headers to send in the request
:return: Tuple<status:int, response:str>
status: one of the following response status codes: IMDS_RESPONSE_SUCCESS, IMDS_RESPONSE_ERROR,
IMDS_CONNECTION_ERROR, IMDS_INTERNAL_SERVER_ERROR
response: IMDS response on IMDS_RESPONSE_SUCCESS, failure message otherwise
"""
try:
resp = self._http_get(endpoint=endpoint, resource_path=resource_path, headers=headers)
except ResourceGoneError:
return IMDS_INTERNAL_SERVER_ERROR, "IMDS error in /metadata/{0}: HTTP Failed with Status Code 410: Gone".format(resource_path)
except HttpError as e:
msg = str(e)
if self._regex_throttled.match(msg):
return IMDS_RESPONSE_ERROR, "IMDS error in /metadata/{0}: Throttled".format(resource_path)
if self._regex_ioerror.match(msg):
logger.periodic_warn(logger.EVERY_FIFTEEN_MINUTES,
"[PERIODIC] [IMDS_CONNECTION_ERROR] Unable to connect to IMDS endpoint {0}".format(endpoint))
return IMDS_CONNECTION_ERROR, "IMDS error in /metadata/{0}: Unable to connect to endpoint".format(resource_path)
return IMDS_INTERNAL_SERVER_ERROR, "IMDS error in /metadata/{0}: {1}".format(resource_path, msg)
if resp.status >= 500:
return IMDS_INTERNAL_SERVER_ERROR, "IMDS error in /metadata/{0}: {1}".format(
resource_path, restutil.read_response_error(resp))
if restutil.request_failed(resp):
return IMDS_RESPONSE_ERROR, "IMDS error in /metadata/{0}: {1}".format(
resource_path, restutil.read_response_error(resp))
return IMDS_RESPONSE_SUCCESS, resp.read()
def put_vm_log(self, content):
"""
Try to upload VM logs, a compressed zip file, via the host plugin /vmAgentLog channel.
:param content: the binary content of the zip file to upload
"""
if not self.ensure_initialized():
raise ProtocolError("HostGAPlugin: HostGAPlugin is not available")
if content is None:
raise ProtocolError(
"HostGAPlugin: Invalid argument passed to upload VM logs. Content was not provided."
)
url = URI_FORMAT_PUT_LOG.format(self.endpoint, HOST_PLUGIN_PORT)
response = restutil.http_put(url,
data=content,
headers=self._build_log_headers(),
redact_data=True)
if restutil.request_failed(response): # pylint: disable=R1720
error_response = restutil.read_response_error(response)
raise HttpError("HostGAPlugin: Upload VM logs failed: {0}".format(
error_response))
return response
def get_api_versions(self):
url = URI_FORMAT_GET_API_VERSIONS.format(self.endpoint,
HOST_PLUGIN_PORT)
logger.verbose(
"HostGAPlugin: Getting API versions at [{0}]".format(url))
return_val = []
error_response = ''
is_healthy = False
try:
headers = {HEADER_CONTAINER_ID: self.container_id}
response = restutil.http_get(url, headers)
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
logger.error(
"HostGAPlugin: Failed Get API versions: {0}".format(
error_response))
else:
return_val = ustr(remove_bom(response.read()),
encoding='utf-8')
is_healthy = True
except HttpError as e:
logger.error(
"HostGAPlugin: Exception Get API versions: {0}".format(e))
self.health_service.report_host_plugin_versions(
is_healthy=is_healthy, response=error_response)
return return_val
def _put_data(self, url, data, headers=None):
headers = _add_content_type(headers)
try:
resp = restutil.http_put(url, json.dumps(data), headers=headers)
except HttpError as e:
raise ProtocolError(ustr(e))
if restutil.request_failed(resp):
raise ProtocolError("{0} - PUT: {1}".format(resp.status, url))
def _put_block_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_block_blob_headers(len(status_blob.data)),
bytearray(status_blob.data, encoding='utf-8')),
headers=self._build_status_headers())
if restutil.request_failed(response):
raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}".format(
restutil.read_response_error(response)))
else:
logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
def validate(self):
"""
Determines whether the metadata instance api returns 200, and the response
is valid: compute should contain location, name, subscription id, and vm size
and network should contain mac address and private ip address.
:return: Tuple<is_healthy:bool, error_response:str>
is_healthy: True when validation succeeds, False otherwise
error_response: validation failure details to assist with debugging
"""
# ensure we get a 200
resp = restutil.http_get(self.instance_url,
headers=self._health_headers)
if restutil.request_failed(resp):
return False, "{0}".format(restutil.read_response_error(resp))
# ensure the response is valid json
data = resp.read()
try:
json_data = json.loads(ustr(data, encoding="utf-8"))
except Exception as e:
return False, "JSON parsing failed: {0}".format(ustr(e))
# ensure all expected fields are present and have a value
try:
self.check_field(json_data, 'compute')
self.check_field(json_data['compute'], 'location')
self.check_field(json_data['compute'], 'name')
self.check_field(json_data['compute'], 'subscriptionId')
self.check_field(json_data['compute'], 'vmSize')
self.check_field(json_data, 'network')
self.check_field(json_data['network'], 'interface')
self.check_field(json_data['network']['interface'][0],
'macAddress')
self.check_field(json_data['network']['interface'][0], 'ipv4')
self.check_field(json_data['network']['interface'][0]['ipv4'],
'ipAddress')
self.check_field(
json_data['network']['interface'][0]['ipv4']['ipAddress'][0],
'privateIpAddress')
except ValueError as v:
return False, ustr(v)
return True, ''
def _put_block_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_block_blob_headers(len(status_blob.data)),
bytearray(status_blob.data, encoding='utf-8')),
headers=self._build_status_headers())
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy, response=error_response)
raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}"
.format(error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
def get_compute(self):
"""
Fetch compute information.
:return: instance of a ComputeInfo
:rtype: ComputeInfo
"""
resp = restutil.http_get(self.compute_url, headers=self._headers)
if restutil.request_failed(resp):
raise HttpError("{0} - GET: {1}".format(resp.status, self.compute_url))
data = resp.read()
data = json.loads(ustr(data, encoding="utf-8"))
compute_info = ComputeInfo()
set_properties('compute', compute_info, data)
return compute_info
def get_compute(self):
"""
Fetch compute information.
:return: instance of a ComputeInfo
:rtype: ComputeInfo
"""
resp = restutil.http_get(self.compute_url, headers=self._headers)
if restutil.request_failed(resp):
raise HttpError("{0} - GET: {1}".format(resp.status, self.compute_url))
data = resp.read()
data = json.loads(ustr(data, encoding="utf-8"))
compute_info = ComputeInfo()
set_properties('compute', compute_info, data)
return compute_info
def validate(self):
"""
Determines whether the metadata instance api returns 200, and the response
is valid: compute should contain location, name, subscription id, and vm size
and network should contain mac address and private ip address.
:return: Tuple<is_healthy:bool, error_response:str>
is_healthy: True when validation succeeds, False otherwise
error_response: validation failure details to assist with debugging
"""
# ensure we get a 200
resp = restutil.http_get(self.instance_url, headers=self._health_headers)
if restutil.request_failed(resp):
return False, "{0}".format(restutil.read_response_error(resp))
# ensure the response is valid json
data = resp.read()
try:
json_data = json.loads(ustr(data, encoding="utf-8"))
except Exception as e:
return False, "JSON parsing failed: {0}".format(ustr(e))
# ensure all expected fields are present and have a value
try:
self.check_field(json_data, 'compute')
self.check_field(json_data['compute'], 'location')
self.check_field(json_data['compute'], 'name')
self.check_field(json_data['compute'], 'subscriptionId')
self.check_field(json_data['compute'], 'vmSize')
self.check_field(json_data, 'network')
self.check_field(json_data['network'], 'interface')
self.check_field(json_data['network']['interface'][0], 'macAddress')
self.check_field(json_data['network']['interface'][0], 'ipv4')
self.check_field(json_data['network']['interface'][0]['ipv4'], 'ipAddress')
self.check_field(json_data['network']['interface'][0]['ipv4']['ipAddress'][0], 'privateIpAddress')
except ValueError as v:
return False, ustr(v)
return True, ''
def _put_block_blob_status(self, sas_url, status_blob):
url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
response = restutil.http_put(url,
data=self._build_status_data(
sas_url,
status_blob.get_block_blob_headers(
len(status_blob.data)),
bytearray(status_blob.data,
encoding='utf-8')),
headers=self._build_status_headers())
if restutil.request_failed(response): # pylint: disable=R1720
error_response = restutil.read_response_error(response)
is_healthy = not restutil.request_failed_at_hostplugin(response)
self.report_status_health(is_healthy=is_healthy,
response=error_response)
raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}".format(
error_response))
else:
self.report_status_health(is_healthy=True)
logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
def _get_data(self, url, headers=None):
try:
resp = restutil.http_get(url, headers=headers)
except HttpError as e:
raise ProtocolError(ustr(e))
# NOT_MODIFIED (304) response means the call was successful, so allow that to proceed.
is_not_modified = restutil.request_not_modified(resp)
if restutil.request_failed(resp) and not is_not_modified:
raise ProtocolError("{0} - GET: {1}".format(resp.status, url))
data = resp.read()
etag = resp.getheader('ETag')
# If the response was 304, then explicilty set data to None
if is_not_modified:
data = None
if data is not None:
data = json.loads(ustr(data, encoding="utf-8"))
return data, etag
def get_api_versions(self):
url = URI_FORMAT_GET_API_VERSIONS.format(self.endpoint,
HOST_PLUGIN_PORT)
logger.verbose("HostGAPlugin: Getting API versions at [{0}]"
.format(url))
return_val = []
error_response = ''
is_healthy = False
try:
headers = {HEADER_CONTAINER_ID: self.container_id}
response = restutil.http_get(url, headers)
if restutil.request_failed(response):
error_response = restutil.read_response_error(response)
logger.error("HostGAPlugin: Failed Get API versions: {0}".format(error_response))
is_healthy = not restutil.request_failed_at_hostplugin(response)
else:
return_val = ustr(remove_bom(response.read()), encoding='utf-8')
is_healthy = True
except HttpError as e:
logger.error("HostGAPlugin: Exception Get API versions: {0}".format(e))
self.health_service.report_host_plugin_versions(is_healthy=is_healthy, response=error_response)
return return_val