def test_get_as_string(self):
self.assertIsNone(encoding.get_as_string(None))
content_map = [("data", "data"), (b"data", "data"),
("data".encode(), "data"),
("data".encode("utf-16"), None)]
with testutils.LogSnatcher("cloudbaseinit.utils.encoding") as snatch:
for content, expect in content_map:
self.assertEqual(expect, encoding.get_as_string(content))
self.assertIn("couldn't decode", snatch.output[0].lower())
def _get_ipv4_routing_table(self):
routing_table = []
heap = kernel32.GetProcessHeap()
size = wintypes.ULONG(ctypes.sizeof(Win32_MIB_IPFORWARDTABLE))
p = kernel32.HeapAlloc(heap, 0, ctypes.c_size_t(size.value))
if not p:
raise exception.CloudbaseInitException(
'Unable to allocate memory for the IP forward table')
p_forward_table = ctypes.cast(
p, ctypes.POINTER(Win32_MIB_IPFORWARDTABLE))
try:
err = iphlpapi.GetIpForwardTable(p_forward_table,
ctypes.byref(size), 0)
if err == self.ERROR_INSUFFICIENT_BUFFER:
kernel32.HeapFree(heap, 0, p_forward_table)
p = kernel32.HeapAlloc(heap, 0, ctypes.c_size_t(size.value))
if not p:
raise exception.CloudbaseInitException(
'Unable to allocate memory for the IP forward table')
p_forward_table = ctypes.cast(
p, ctypes.POINTER(Win32_MIB_IPFORWARDTABLE))
err = iphlpapi.GetIpForwardTable(p_forward_table,
ctypes.byref(size), 0)
if err != self.ERROR_NO_DATA:
if err:
raise exception.CloudbaseInitException(
'Unable to get IP forward table. Error: %s' % err)
forward_table = p_forward_table.contents
table = ctypes.cast(
ctypes.addressof(forward_table.table),
ctypes.POINTER(Win32_MIB_IPFORWARDROW *
forward_table.dwNumEntries)).contents
i = 0
while i < forward_table.dwNumEntries:
row = table[i]
routing_table.append((
encoding.get_as_string(Ws2_32.inet_ntoa(
row.dwForwardDest)),
encoding.get_as_string(Ws2_32.inet_ntoa(
row.dwForwardMask)),
encoding.get_as_string(Ws2_32.inet_ntoa(
row.dwForwardNextHop)),
row.dwForwardIfIndex,
row.dwForwardMetric1))
i += 1
return routing_table
finally:
kernel32.HeapFree(heap, 0, p_forward_table)
def _get_ipv4_routing_table(self):
routing_table = []
heap = kernel32.GetProcessHeap()
size = wintypes.ULONG(ctypes.sizeof(Win32_MIB_IPFORWARDTABLE))
p = kernel32.HeapAlloc(heap, 0, ctypes.c_size_t(size.value))
if not p:
raise exception.CloudbaseInitException(
'Unable to allocate memory for the IP forward table')
p_forward_table = ctypes.cast(p,
ctypes.POINTER(Win32_MIB_IPFORWARDTABLE))
try:
err = iphlpapi.GetIpForwardTable(p_forward_table,
ctypes.byref(size), 0)
if err == self.ERROR_INSUFFICIENT_BUFFER:
kernel32.HeapFree(heap, 0, p_forward_table)
p = kernel32.HeapAlloc(heap, 0, ctypes.c_size_t(size.value))
if not p:
raise exception.CloudbaseInitException(
'Unable to allocate memory for the IP forward table')
p_forward_table = ctypes.cast(
p, ctypes.POINTER(Win32_MIB_IPFORWARDTABLE))
err = iphlpapi.GetIpForwardTable(p_forward_table,
ctypes.byref(size), 0)
if err != self.ERROR_NO_DATA:
if err:
raise exception.CloudbaseInitException(
'Unable to get IP forward table. Error: %s' % err)
forward_table = p_forward_table.contents
table = ctypes.cast(
ctypes.addressof(forward_table.table),
ctypes.POINTER(Win32_MIB_IPFORWARDROW *
forward_table.dwNumEntries)).contents
i = 0
while i < forward_table.dwNumEntries:
row = table[i]
routing_table.append(
(encoding.get_as_string(
Ws2_32.inet_ntoa(row.dwForwardDest)),
encoding.get_as_string(
Ws2_32.inet_ntoa(row.dwForwardMask)),
encoding.get_as_string(
Ws2_32.inet_ntoa(row.dwForwardNextHop)),
row.dwForwardIfIndex, row.dwForwardMetric1))
i += 1
return routing_table
finally:
kernel32.HeapFree(heap, 0, p_forward_table)
def test_get_as_string(self):
self.assertIsNone(encoding.get_as_string(None))
content_map = [
("data", "data"),
(b"data", "data"),
("data".encode(), "data"),
("data".encode("utf-16"), None)
]
with testutils.LogSnatcher("cloudbaseinit.utils.encoding") as snatch:
for content, expect in content_map:
self.assertEqual(expect, encoding.get_as_string(content))
self.assertIn("couldn't decode", snatch.output[0].lower())
def test__get_health_report_xml(self, mock_get_incarnation,
mock_get_container_id,
mock_get_role_instance_id):
mock_state = 'FakeState'
mock_substatus = 'FakeStatus'
mock_description = 'FakeDescription'
mock_get_incarnation.return_value = "fake"
mock_get_container_id.return_value = "fakeid"
mock_get_role_instance_id.return_value = "fakeroleid"
res = self._azureservice._get_health_report_xml(mock_state,
mock_substatus,
mock_description)
expected_result = "<?xml version='1.0' encoding='utf-8'?>\n<Health>" \
"<GoalStateIncarnation>{}</GoalStateIncarnation>" \
"<Container><ContainerId>{}</ContainerId>" \
"<RoleInstanceList><Role><InstanceId>{}</InstanceId>" \
"<Health><State>{}</State><Details><SubStatus>{}</SubStatus>" \
"<Description>{}</Description></Details></Health>" \
"</Role></RoleInstanceList></Container></Health>"
self.assertEqual(encoding.get_as_string(res),
expected_result.format(
mock_get_incarnation.return_value,
mock_get_container_id.return_value,
mock_get_role_instance_id.return_value,
mock_state,
mock_substatus,
mock_description))
def _test_get_role_properties_xml(self, mock_get_container_id,
mock_get_role_instance_id,
properties):
mock_get_container_id.return_value = "fake container id"
mock_get_role_instance_id.return_value = "fake instance id"
res = self._azureservice._get_role_properties_xml(properties)
expected_properties = ""
property_template = '<Property name=\"{property_name}"'\
' value="{value}" />'
result_template = ("<?xml version=\'1.0\' encoding=\'utf-8\'?>\n"
"<RoleProperties><Container><ContainerId>"
"{container_id}</ContainerId><RoleInstances>"
"<RoleInstance><Id>{instance_id}</Id>"
"{properties}</RoleInstance>"
"</RoleInstances></Container></RoleProperties>")
if properties:
expected_properties = "<Properties>"
for name, value in properties.items():
expected_properties += property_template.format(
property_name=name,
value=value)
expected_properties += "</Properties>"
else:
expected_properties = "<Properties />"
expected_result = result_template.format(
container_id=mock_get_container_id.return_value,
instance_id=mock_get_role_instance_id.return_value,
properties=expected_properties)
self.assertEqual(encoding.get_as_string(res), expected_result)
def _password_client(self, body=None, headers=None, decode=True):
"""Client for the Password Server."""
port = CONF.cloudstack.password_server_port
with contextlib.closing(
http_client.HTTPConnection(self._metadata_host,
port,
timeout=TIMEOUT)) as connection:
try:
connection.request("GET", "/", body=body, headers=headers)
response = connection.getresponse()
except http_client.HTTPException as exc:
LOG.error("Request failed: %s", exc)
raise
content = response.read()
if decode:
content = encoding.get_as_string(content)
if response.status != 200:
raise http_client.HTTPException(
"%(status)s %(reason)s - %(message)r", {
"status": response.status,
"reason": response.reason,
"message": content
})
return content
def _parse_shell_variables(content):
"""Returns a dictionary with variables and their values.
This is a dummy approach, because it works only with simple literals.
"""
# preprocess the content
lines = []
for line in content.splitlines():
if not line or line.startswith(b"#"):
continue
lines.append(line)
# for cleaner pattern matching
lines.append(b"__REGEX_DUMMY__='__regex_dummy__'")
sep = b"\r\n" if b"\r\n" in content else b"\n"
new_content = sep.join(lines)
# get pairs
pairs = {}
pattern = (br"(?P<key>\w+)=((?P<int_value>\d+)|"
br"['\"](?P<str_value>[\s\S]*?)['\"])(?=\s+\w+=)")
for match in re.finditer(pattern, new_content):
key = encoding.get_as_string(match.group("key"))
val = match.group("str_value")
if match.group("int_value"):
val = int(match.group("int_value"))
pairs[key] = val
return pairs
def _test_get_role_properties_xml(self, mock_get_container_id,
mock_get_role_instance_id, properties):
mock_get_container_id.return_value = "fake container id"
mock_get_role_instance_id.return_value = "fake instance id"
res = self._azureservice._get_role_properties_xml(properties)
expected_properties = ""
property_template = '<Property name=\"{property_name}"'\
' value="{value}" />'
result_template = ("<?xml version=\'1.0\' encoding=\'utf-8\'?>\n"
"<RoleProperties><Container><ContainerId>"
"{container_id}</ContainerId><RoleInstances>"
"<RoleInstance><Id>{instance_id}</Id>"
"{properties}</RoleInstance>"
"</RoleInstances></Container></RoleProperties>")
if properties:
expected_properties = "<Properties>"
for name, value in properties.items():
expected_properties += property_template.format(
property_name=name, value=value)
expected_properties += "</Properties>"
else:
expected_properties = "<Properties />"
expected_result = result_template.format(
container_id=mock_get_container_id.return_value,
instance_id=mock_get_role_instance_id.return_value,
properties=expected_properties)
self.assertEqual(encoding.get_as_string(res), expected_result)
def test__get_health_report_xml(self, mock_get_incarnation,
mock_get_container_id,
mock_get_role_instance_id):
mock_state = 'FakeState'
mock_substatus = 'FakeStatus'
mock_description = 'FakeDescription'
mock_get_incarnation.return_value = "fake"
mock_get_container_id.return_value = "fakeid"
mock_get_role_instance_id.return_value = "fakeroleid"
res = self._azureservice._get_health_report_xml(
mock_state, mock_substatus, mock_description)
expected_result = "<?xml version='1.0' encoding='utf-8'?>\n<Health>" \
"<GoalStateIncarnation>{}</GoalStateIncarnation>" \
"<Container><ContainerId>{}</ContainerId>" \
"<RoleInstanceList><Role><InstanceId>{}</InstanceId>" \
"<Health><State>{}</State><Details><SubStatus>{}</SubStatus>" \
"<Description>{}</Description></Details></Health>" \
"</Role></RoleInstanceList></Container></Health>"
self.assertEqual(
encoding.get_as_string(res),
expected_result.format(mock_get_incarnation.return_value,
mock_get_container_id.return_value,
mock_get_role_instance_id.return_value,
mock_state, mock_substatus,
mock_description))
def _get_password(self):
"""Get the password from the Password Server.
The Password Server can be found on the DHCP_SERVER on the port 8080.
.. note:
The Password Server can return the following values:
* `bad_request`: the Password Server did not recognise
the request
* `saved_password`: the password was already deleted from
the Password Server
* ``: the Password Server did not have any
password for this instance
* the password
"""
LOG.debug("Try to get password from the Password Server.")
headers = {"DomU_Request": "send_my_password"}
password = None
with contextlib.closing(http_client.HTTPConnection(
self._router_ip, 8080, timeout=TIMEOUT)) as connection:
for _ in range(CONF.retry_count):
try:
connection.request("GET", "/", headers=headers)
response = connection.getresponse()
except http_client.HTTPException as exc:
LOG.exception(exc)
continue
if response.status != 200:
LOG.warning("Getting password failed: %(status)s "
"%(reason)s - %(message)r",
{"status": response.status,
"reason": response.reason,
"message": response.read()})
continue
content = response.read().strip()
if not content:
LOG.warning("The Password Server did not have any "
"password for the current instance.")
continue
if content == BAD_REQUEST:
LOG.error("The Password Server did not recognise the "
"request.")
break
if content == SAVED_PASSWORD:
LOG.warning("For this instance the password was already "
"taken from the Password Server.")
break
LOG.info("The password server return a valid password "
"for the current instance.")
password = encoding.get_as_string(content)
break
return password
def _get_cert_base64(self, cert_data):
base64_cert_data = encoding.get_as_string(cert_data)
if base64_cert_data.startswith(x509constants.PEM_HEADER):
base64_cert_data = base64_cert_data[len(x509constants.PEM_HEADER):]
if base64_cert_data.endswith(x509constants.PEM_FOOTER):
base64_cert_data = base64_cert_data[:len(base64_cert_data) -
len(x509constants.PEM_FOOTER)]
return base64_cert_data.replace("\n", "")
def _get_cert_base64(self, cert_data):
base64_cert_data = encoding.get_as_string(cert_data)
if base64_cert_data.startswith(x509constants.PEM_HEADER):
base64_cert_data = base64_cert_data[len(x509constants.PEM_HEADER):]
if base64_cert_data.endswith(x509constants.PEM_FOOTER):
base64_cert_data = base64_cert_data[:len(base64_cert_data) -
len(x509constants.PEM_FOOTER)]
return base64_cert_data.replace("\n", "")
def _get_cache_data(self, path, decode=False):
"""Get meta data with caching and decoding support."""
key = (path, decode)
if key in self._cache:
LOG.debug("Using cached copy of metadata: '%s'" % path)
return self._cache[key]
else:
data = self._exec_with_retry(lambda: self._get_data(path))
if decode:
data = encoding.get_as_string(data)
self._cache[key] = data
return data
def _get_cache_data(self, path, decode=False):
"""Get meta data with caching and decoding support."""
key = (path, decode)
if key in self._cache:
LOG.debug("Using cached copy of metadata: '%s'" % path)
return self._cache[key]
else:
data = self._exec_with_retry(lambda: self._get_data(path))
if decode:
data = encoding.get_as_string(data)
self._cache[key] = data
return data
def get_network_details(self):
network_config = self._get_meta_data().get('network_config')
if not network_config:
return None
key = "content_path"
if key not in network_config:
return None
content_name = network_config[key].rsplit("/", 1)[-1]
content = self.get_content(content_name)
content = encoding.get_as_string(content)
return debiface.parse(content)
def get_network_details(self):
network_config = self._get_meta_data().get('network_config')
if not network_config:
return None
key = "content_path"
if key not in network_config:
return None
content_name = network_config[key].rsplit("/", 1)[-1]
content = self.get_content(content_name)
content = encoding.get_as_string(content)
return debiface.parse(content)
def _get_headers(user_data):
"""Returns the header of the given user data.
:param user_data: Represents the content of the user data.
:rtype: A string chunk containing the header or None.
.. note :: In case the content type is not valid,
None will be returned.
"""
content = encoding.get_as_string(user_data)
if content:
return content.split("\n\n")[0]
else:
raise exception.CloudbaseInitException("No header could be found."
"The user data content is "
"either invalid or empty.")
def _get_headers(user_data):
"""Returns the header of the given user data.
:param user_data: Represents the content of the user data.
:rtype: A string chunk containing the header or None.
.. note :: In case the content type is not valid,
None will be returned.
"""
content = encoding.get_as_string(user_data)
if content:
return content.split("\n\n")[0]
else:
raise exception.CloudbaseInitException("No header could be found."
"The user data content is "
"either invalid or empty.")
def get_client_auth_certs(self):
"""Gather all unique certificates found among the metadata.
If there are no certificates under "meta" or "keys" field,
then try looking into user-data for this kind of information.
"""
certs = []
meta_data = self._get_meta_data()
meta = meta_data.get("meta")
if meta:
cert_data_list = []
idx = 0
while True:
# Chunking is necessary as metadata items can be
# max. 255 chars long.
cert_chunk = meta.get("admin_cert%d" % idx)
if not cert_chunk:
break
cert_data_list.append(cert_chunk)
idx += 1
if cert_data_list:
# It's a list of strings for sure.
certs.append("".join(cert_data_list))
keys = meta_data.get("keys")
if keys:
for key_dict in keys:
if key_dict["type"] == "x509":
certs.append(key_dict["data"])
if not certs:
# Look if the user_data contains a PEM certificate
try:
user_data = self.get_user_data().strip()
if user_data.startswith(
x509constants.PEM_HEADER.encode()):
certs.append(encoding.get_as_string(user_data))
except base.NotExistingMetadataException:
LOG.debug("user_data metadata not present")
return list(set((cert.strip() for cert in certs)))
def get_client_auth_certs(self):
"""Gather all unique certificates found among the metadata.
If there are no certificates under "meta" or "keys" field,
then try looking into user-data for this kind of information.
"""
certs = []
meta_data = self._get_meta_data()
meta = meta_data.get("meta")
if meta:
cert_data_list = []
idx = 0
while True:
# Chunking is necessary as metadata items can be
# max. 255 chars long.
cert_chunk = meta.get("admin_cert%d" % idx)
if not cert_chunk:
break
cert_data_list.append(cert_chunk)
idx += 1
if cert_data_list:
# It's a list of strings for sure.
certs.append("".join(cert_data_list))
keys = meta_data.get("keys")
if keys:
for key_dict in keys:
if key_dict["type"] == "x509":
certs.append(key_dict["data"])
if not certs:
# Look if the user_data contains a PEM certificate
try:
user_data = self.get_user_data().strip()
if user_data.startswith(
x509constants.PEM_HEADER.encode()):
certs.append(encoding.get_as_string(user_data))
except base.NotExistingMetadataException:
LOG.debug("user_data metadata not present")
return list(set((cert.strip() for cert in certs)))
def _wire_server_request(self, path, data_xml=None, headers=None,
parse_xml=True):
if not self._base_url:
raise exception.CloudbaseInitException(
"Azure WireServer base url not set")
all_headers = self._headers.copy()
if data_xml:
all_headers["Content-Type"] = "text/xml; charset=utf-8"
if headers:
all_headers.update(headers)
data = self._exec_with_retry(
lambda: super(AzureService, self)._http_request(
path, data_xml, headers=all_headers))
if parse_xml:
return untangle.parse(six.StringIO(encoding.get_as_string(data)))
else:
return data
def _wire_server_request(self, path, data_xml=None, headers=None,
parse_xml=True):
if not self._base_url:
raise exception.CloudbaseInitException(
"Azure WireServer base url not set")
all_headers = self._headers.copy()
if data_xml:
all_headers["Content-Type"] = "text/xml; charset=utf-8"
if headers:
all_headers.update(headers)
data = self._exec_with_retry(
lambda: super(AzureService, self)._http_request(
path, data_xml, headers=all_headers))
if parse_xml:
return untangle.parse(six.StringIO(encoding.get_as_string(data)))
else:
return data
def load(self):
super(BSIService, self).load()
tries = 10
while tries > 0:
try:
f = open('C:/URL.tmp', 'r')
break
except FileNotFoundError:
tries = tries - 1
LOG.debug('Could not find file, %i retries remaining', tries)
time.sleep(5)
if tries == 0:
raise base.NotExistingMetadataException()
self.URL = f.read()
f.close()
LOG.debug('%s', self.URL)
raw_information = encoding.get_as_string(self._get_data(self.URL))
self.decoded_information = json.loads(raw_information)
LOG.debug('%s', self.decoded_information)
return True
def _password_client(self, body=None, headers=None, decode=True):
"""Client for the Password Server."""
port = CONF.cloudstack.password_server_port
with contextlib.closing(http_client.HTTPConnection(
self._metadata_host, port, timeout=TIMEOUT)) as connection:
try:
connection.request("GET", "/", body=body, headers=headers)
response = connection.getresponse()
except http_client.HTTPException as exc:
LOG.error("Request failed: %s", exc)
raise
content = response.read()
if decode:
content = encoding.get_as_string(content)
if response.status != 200:
raise http_client.HTTPException(
"%(status)s %(reason)s - %(message)r",
{"status": response.status, "reason": response.reason,
"message": content})
return content
def _parse_shell_variables(content):
"""Returns a dictionary with variables and their values.
This is a dummy approach, because it works only with simple literals.
"""
# preprocess the content
lines = []
for line in content.splitlines():
if not line or line.startswith(b"#"):
continue
lines.append(line)
# for cleaner pattern matching
lines.append(b"__REGEX_DUMMY__='__regex_dummy__'")
sep = b"\r\n" if b"\r\n" in content else b"\n"
new_content = sep.join(lines)
# get pairs
pairs = {}
pattern = (br"(?P<key>\w+)=(['\"](?P<str_value>[\s\S]+?)['\"]|"
br"(?P<int_value>\d+))(?=\s+\w+=)")
for match in re.finditer(pattern, new_content):
key = encoding.get_as_string(match.group("key"))
pairs[key] = (match.group("str_value") or
int(match.group("int_value")))
return pairs
def _parse_mime(self, user_data):
user_data_str = encoding.get_as_string(user_data)
LOG.debug('User data content:\n%s' % user_data_str)
return email.message_from_string(user_data_str).walk()
def _parse_mime(self, user_data):
user_data_str = encoding.get_as_string(user_data)
LOG.debug('User data content:\n%s' % user_data_str)
return email.message_from_string(user_data_str).walk()
def _parse_mime(self, user_data):
user_data_str = encoding.get_as_string(user_data)
return email.message_from_string(user_data_str).walk()
def _get_meta_data(self, version='latest'):
path = posixpath.normpath(
posixpath.join('openstack', version, 'meta_data.json'))
data = self._get_cache_data(path)
if data:
return json.loads(encoding.get_as_string(data))
def _get_network_data(self):
"""Grabs network_data to be parsed."""
data = self._get_data(self._NETWORK_DATA_JSON)
network_data = json.loads(encoding.get_as_string(data))
return network_data
def _get_password(self):
"""Get the password from the Password Server.
The Password Server can be found on the DHCP_SERVER on the port 8080.
.. note:
The Password Server can return the following values:
* `bad_request`: the Password Server did not recognise
the request
* `saved_password`: the password was already deleted from
the Password Server
* ``: the Password Server did not have any
password for this instance
* the password
"""
LOG.debug("Try to get password from the Password Server.")
headers = {"DomU_Request": "send_my_password"}
password = None
with contextlib.closing(
http_client.HTTPConnection(self._router_ip,
8080,
timeout=TIMEOUT)) as connection:
for _ in range(CONF.retry_count):
try:
connection.request("GET", "/", headers=headers)
response = connection.getresponse()
except http_client.HTTPException as exc:
LOG.exception(exc)
continue
if response.status != 200:
LOG.warning(
"Getting password failed: %(status)s "
"%(reason)s - %(message)r", {
"status": response.status,
"reason": response.reason,
"message": response.read()
})
continue
content = response.read().strip()
if not content:
LOG.warning("The Password Server did not have any "
"password for the current instance.")
continue
if content == BAD_REQUEST:
LOG.error("The Password Server did not recognise the "
"request.")
break
if content == SAVED_PASSWORD:
LOG.warning("For this instance the password was already "
"taken from the Password Server.")
break
LOG.info("The password server return a valid password "
"for the current instance.")
password = encoding.get_as_string(content)
break
return password
def _get_meta_data(self, version='latest'):
path = posixpath.normpath(
posixpath.join('openstack', version, 'meta_data.json'))
data = self._get_cache_data(path)
if data:
return json.loads(encoding.get_as_string(data))