def get_password(metadata_url=METADATA_URL,
api_version=API_VERSION,
password_server_port=PASSWORD_SERVER_PORT,
url_timeout=URL_TIMEOUT,
url_retries=URL_RETRIES):
"""Obtain the VM's password if set.
Once fetched the password is marked saved. Future calls to this method may
return empty string or 'saved_password'."""
password_url = "{}:{}/{}/".format(metadata_url, password_server_port,
api_version)
response = url_helper.read_file_or_url(
password_url,
ssl_details=None,
headers={"DomU_Request": "send_my_password"},
timeout=url_timeout,
retries=url_retries)
password = response.contents.decode('utf-8')
# the password is empty or already saved
# Note: the original metadata server would answer an additional
# 'bad_request' status, but the Exoscale implementation does not.
if password in ['', 'saved_password']:
return None
# save the password
url_helper.read_file_or_url(password_url,
ssl_details=None,
headers={"DomU_Request": "saved_password"},
timeout=url_timeout,
retries=url_retries)
return password
def test_read_file_or_url_str_from_url_redacts_noheaders(self):
"""When no headers_redact, header values are in logs and requests."""
url = 'http://hostname/path'
headers = {'sensitive': 'sekret', 'server': 'blah'}
httpretty.register_uri(httpretty.GET, url)
read_file_or_url(url, headers=headers)
for k in headers.keys():
self.assertEqual(headers[k], httpretty.last_request().headers[k])
logs = self.logs.getvalue()
self.assertNotIn(REDACTED, logs)
self.assertIn('sekret', logs)
def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self):
"""Headers are redacted from logs but unredacted in requests."""
url = 'http://hostname/path'
headers = {'sensitive': 'sekret', 'server': 'blah'}
httpretty.register_uri(httpretty.GET, url)
read_file_or_url(url, headers=headers, headers_redact=['sensitive'])
logs = self.logs.getvalue()
for k in headers.keys():
self.assertEqual(headers[k], httpretty.last_request().headers[k])
self.assertIn(REDACTED, logs)
self.assertNotIn('sekret', logs)
def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self):
"""Headers are redacted from logs but unredacted in requests."""
url = 'http://hostname/path'
headers = {'sensitive': 'sekret', 'server': 'blah'}
httpretty.register_uri(httpretty.GET, url)
# By default, httpretty will log our request along with the header,
# so if we don't change this the secret will show up in the logs
logging.getLogger('httpretty.core').setLevel(logging.CRITICAL)
read_file_or_url(url, headers=headers, headers_redact=['sensitive'])
logs = self.logs.getvalue()
for k in headers.keys():
self.assertEqual(headers[k], httpretty.last_request().headers[k])
self.assertIn(REDACTED, logs)
self.assertNotIn('sekret', logs)
def post(self, url, data=None, extra_headers=None):
headers = self.headers
if extra_headers is not None:
headers = self.headers.copy()
headers.update(extra_headers)
return url_helper.read_file_or_url(url, data=data, headers=headers,
timeout=5, retries=10)
def get(self, url, secure=False):
headers = self.headers
if secure:
headers = self.headers.copy()
headers.update(self.extra_secure_headers)
return url_helper.read_file_or_url(url, headers=headers, timeout=5,
retries=10)
def get_instance_userdata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None,
timeout=5,
retries=5,
headers_cb=None,
headers_redact=None,
exception_cb=None):
ud_url = url_helper.combine_url(metadata_address, api_version)
ud_url = url_helper.combine_url(ud_url, 'user-data')
user_data = ''
try:
if not exception_cb:
# It is ok for userdata to not exist (thats why we are stopping if
# NOT_FOUND occurs) and just in that case returning an empty
# string.
exception_cb = functools.partial(skip_retry_on_codes,
SKIP_USERDATA_CODES)
response = url_helper.read_file_or_url(ud_url,
ssl_details=ssl_details,
timeout=timeout,
retries=retries,
exception_cb=exception_cb,
headers_cb=headers_cb,
headers_redact=headers_redact)
user_data = response.contents
except url_helper.UrlError as e:
if e.code not in SKIP_USERDATA_CODES:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
except Exception:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
return user_data
def _do_include(self, content, append_msg):
# Include a list of urls, one per line
# also support '#include <url here>'
# or #include-once '<url here>'
include_once_on = False
for line in content.splitlines():
lc_line = line.lower()
if lc_line.startswith("#include-once"):
line = line[len("#include-once"):].lstrip()
# Every following include will now
# not be refetched.... but will be
# re-read from a local urlcache (if it worked)
include_once_on = True
elif lc_line.startswith("#include"):
line = line[len("#include"):].lstrip()
# Disable the include once if it was on
# if it wasn't, then this has no effect.
include_once_on = False
if line.startswith("#"):
continue
include_url = line.strip()
if not include_url:
continue
include_once_fn = None
content = None
if include_once_on:
include_once_fn = self._get_include_once_filename(include_url)
if include_once_on and os.path.isfile(include_once_fn):
content = util.load_file(include_once_fn)
else:
try:
resp = read_file_or_url(include_url,
timeout=5,
retries=10,
ssl_details=self.ssl_details)
if include_once_on and resp.ok():
util.write_file(include_once_fn,
resp.contents,
mode=0o600)
if resp.ok():
content = resp.contents
else:
LOG.warning(("Fetching from %s resulted in"
" a invalid http code of %s"),
include_url, resp.code)
except UrlError as urle:
message = str(urle)
# Older versions of requests.exceptions.HTTPError may not
# include the errant url. Append it for clarity in logs.
if include_url not in message:
message += ' for url: {0}'.format(include_url)
LOG.warning(message)
except IOError as ioe:
LOG.warning("Fetching from %s resulted in %s", include_url,
ioe)
if content is not None:
new_msg = convert_string(content)
self._process_msg(new_msg, append_msg)
def test_readurl_timeout(self, readurl_timeout, request_timeout):
url = "http://hostname/path"
m_response = mock.MagicMock()
class FakeSession(requests.Session):
@classmethod
def request(cls, **kwargs):
expected_kwargs = {
"url": url,
"allow_redirects": True,
"method": "GET",
"headers": {
"User-Agent": "Cloud-Init/%s"
% (version.version_string())
},
"timeout": request_timeout,
}
if request_timeout is None:
expected_kwargs.pop("timeout")
assert kwargs == expected_kwargs
return m_response
with mock.patch(
M_PATH + "requests.Session", side_effect=[FakeSession()]
):
response = read_file_or_url(url, timeout=readurl_timeout)
assert response._response == m_response
def test_read_file_or_url_passes_params_to_readurl(self, m_readurl,
timeout):
"""read_file_or_url passes all params through to readurl."""
url = "http://hostname/path"
response = "This is my url content\n"
m_readurl.return_value = response
params = {
"url": url,
"timeout": timeout,
"retries": 2,
"headers": {
"somehdr": "val"
},
"data": "data",
"sec_between": 1,
"ssl_details": {
"cert_file": "/path/cert.pem"
},
"headers_cb": "headers_cb",
"exception_cb": "exception_cb",
}
assert response == read_file_or_url(**params)
params.pop("url") # url is passed in as a positional arg
assert m_readurl.call_args_list == [mock.call(url, **params)]
def test_read_file_or_url_str_from_file(self):
"""Test that str(result.contents) on file is text version of contents.
It should not be "b'data'", but just "'data'" """
tmpf = self.tmp_path("myfile1")
data = b'This is my file content\n'
util.write_file(tmpf, data, omode="wb")
result = read_file_or_url("file://%s" % tmpf)
self.assertEqual(result.contents, data)
self.assertEqual(str(result), data.decode('utf-8'))
def test_read_file_or_url_str_from_url(self):
"""Test that str(result.contents) on url is text version of contents.
It should not be "b'data'", but just "'data'" """
url = 'http://hostname/path'
data = b'This is my url content\n'
httpretty.register_uri(httpretty.GET, url, data)
result = read_file_or_url(url)
self.assertEqual(result.contents, data)
self.assertEqual(str(result), data.decode('utf-8'))
def test_read_file_or_url_str_from_file(self):
"""Test that str(result.contents) on file is text version of contents.
It should not be "b'data'", but just "'data'" """
tmpf = self.tmp_path("myfile1")
data = b'This is my file content\n'
util.write_file(tmpf, data, omode="wb")
result = read_file_or_url("file://%s" % tmpf)
self.assertEqual(result.contents, data)
self.assertEqual(str(result), data.decode('utf-8'))
def test_read_file_or_url_str_from_url(self):
"""Test that str(result.contents) on url is text version of contents.
It should not be "b'data'", but just "'data'" """
url = 'http://hostname/path'
data = b'This is my url content\n'
httpretty.register_uri(httpretty.GET, url, data)
result = read_file_or_url(url)
self.assertEqual(result.contents, data)
self.assertEqual(str(result), data.decode('utf-8'))
def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self):
"""Readurl param defaults used when unspecified by read_file_or_url
Param defaults tested are as follows:
retries: 0, additional headers None beyond default, method: GET,
data: None, check_status: True and allow_redirects: True
"""
url = "http://hostname/path"
m_response = mock.MagicMock()
class FakeSession(requests.Session):
@classmethod
def request(cls, **kwargs):
self.assertEqual(
{
"url": url,
"allow_redirects": True,
"method": "GET",
"headers": {
"User-Agent": "Cloud-Init/%s"
% (version.version_string())
},
},
kwargs,
)
return m_response
with mock.patch(M_PATH + "requests.Session") as m_session:
error = requests.exceptions.HTTPError("broke")
m_session.side_effect = [error, FakeSession()]
# assert no retries and check_status == True
with self.assertRaises(UrlError) as context_manager:
response = read_file_or_url(url)
self.assertEqual("broke", str(context_manager.exception))
# assert default headers, method, url and allow_redirects True
# Success on 2nd call with FakeSession
response = read_file_or_url(url)
self.assertEqual(m_response, response._response)
def get_instance_userdata(api_version='latest',
metadata_address='http://169.254.169.254',
ssl_details=None, timeout=5, retries=5):
ud_url = url_helper.combine_url(metadata_address, api_version)
ud_url = url_helper.combine_url(ud_url, 'user-data')
user_data = ''
try:
# It is ok for userdata to not exist (thats why we are stopping if
# NOT_FOUND occurs) and just in that case returning an empty string.
exception_cb = functools.partial(_skip_retry_on_codes,
SKIP_USERDATA_CODES)
response = url_helper.read_file_or_url(
ud_url, ssl_details=ssl_details, timeout=timeout,
retries=retries, exception_cb=exception_cb)
user_data = response.contents
except url_helper.UrlError as e:
if e.code not in SKIP_USERDATA_CODES:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
except Exception:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
return user_data
def test_read_file_or_url_passes_params_to_readurl(self, m_readurl):
"""read_file_or_url passes all params through to readurl."""
url = 'http://hostname/path'
response = 'This is my url content\n'
m_readurl.return_value = response
params = {
'url': url,
'timeout': 1,
'retries': 2,
'headers': {
'somehdr': 'val'
},
'data': 'data',
'sec_between': 1,
'ssl_details': {
'cert_file': '/path/cert.pem'
},
'headers_cb': 'headers_cb',
'exception_cb': 'exception_cb'
}
self.assertEqual(response, read_file_or_url(**params))
params.pop('url') # url is passed in as a positional arg
self.assertEqual([mock.call(url, **params)], m_readurl.call_args_list)
def attempt_cmdline_url(path, network=True, cmdline=None):
"""Write data from url referenced in command line to path.
path: a file to write content to if downloaded.
network: should network access be assumed.
cmdline: the cmdline to parse for cloud-config-url.
This is used in MAAS datasource, in "ephemeral" (read-only root)
environment where the instance netboots to iscsi ro root.
and the entity that controls the pxe config has to configure
the maas datasource.
An attempt is made on network urls even in local datasource
for case of network set up in initramfs.
Return value is a tuple of a logger function (logging.DEBUG)
and a message indicating what happened.
"""
if cmdline is None:
cmdline = util.get_cmdline()
try:
cmdline_name, url = parse_cmdline_url(cmdline)
except KeyError:
return (logging.DEBUG, "No kernel command line url found.")
path_is_local = url.startswith("file://") or url.startswith("/")
if path_is_local and os.path.exists(path):
if network:
m = ("file '%s' existed, possibly from local stage download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.INFO
if path_is_local:
level = logging.DEBUG
else:
m = ("file '%s' existed, possibly from previous boot download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.WARN
return (level, m)
kwargs = {'url': url, 'timeout': 10, 'retries': 2}
if network or path_is_local:
level = logging.WARN
kwargs['sec_between'] = 1
else:
level = logging.DEBUG
kwargs['sec_between'] = .1
data = None
header = b'#cloud-config'
try:
resp = url_helper.read_file_or_url(**kwargs)
if resp.ok():
data = resp.contents
if not resp.contents.startswith(header):
if cmdline_name == 'cloud-config-url':
level = logging.WARN
else:
level = logging.INFO
return (level, "contents of '%s' did not start with %s" %
(url, header))
else:
return (level,
"url '%s' returned code %s. Ignoring." % (url, resp.code))
except url_helper.UrlError as e:
return (level, "retrieving url '%s' failed: %s" % (url, e))
util.write_file(path, data, mode=0o600)
return (logging.INFO, "wrote cloud-config data from %s='%s' to %s" %
(cmdline_name, url, path))
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if "phone_home" not in cfg:
log.debug(
"Skipping module named %s, "
"no 'phone_home' configuration found",
name,
)
return
ph_cfg = cfg["phone_home"]
if "url" not in ph_cfg:
log.warning(
"Skipping module named %s, "
"no 'url' found in 'phone_home' configuration",
name,
)
return
url = ph_cfg["url"]
post_list = ph_cfg.get("post", "all")
tries = ph_cfg.get("tries")
try:
tries = int(tries) # type: ignore
except ValueError:
tries = 10
util.logexc(
log,
"Configuration entry 'tries' is not an integer, using %s instead",
tries,
)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys["instance_id"] = cloud.get_instance_id()
all_keys["hostname"] = cloud.get_hostname()
all_keys["fqdn"] = cloud.get_hostname(fqdn=True)
pubkeys = {
"pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub",
"pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub",
"pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub",
"pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub",
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log,
"%s: failed to open, can not phone home that data!",
path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warning(
"Requested key %s from 'post'"
" configuration list not available",
k,
)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = "N/A"
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
"INSTANCE_ID": all_keys["instance_id"],
}
url = templater.render_string(url, url_params)
try:
url_helper.read_file_or_url(
url,
data=real_submit_keys,
retries=tries,
sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths),
)
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def get(self, url, secure=False):
headers = self.headers
if secure:
headers = self.headers.copy()
headers.update(self.extra_secure_headers)
return url_helper.read_file_or_url(url, headers=headers)
def post(self, url, data=None, extra_headers=None):
headers = self.headers
if extra_headers is not None:
headers = self.headers.copy()
headers.update(extra_headers)
return url_helper.read_file_or_url(url, data=data, headers=headers)
def attempt_cmdline_url(path, network=True, cmdline=None):
"""Write data from url referenced in command line to path.
path: a file to write content to if downloaded.
network: should network access be assumed.
cmdline: the cmdline to parse for cloud-config-url.
This is used in MAAS datasource, in "ephemeral" (read-only root)
environment where the instance netboots to iscsi ro root.
and the entity that controls the pxe config has to configure
the maas datasource.
An attempt is made on network urls even in local datasource
for case of network set up in initramfs.
Return value is a tuple of a logger function (logging.DEBUG)
and a message indicating what happened.
"""
if cmdline is None:
cmdline = util.get_cmdline()
try:
cmdline_name, url = parse_cmdline_url(cmdline)
except KeyError:
return (logging.DEBUG, "No kernel command line url found.")
path_is_local = url.startswith("file://") or url.startswith("/")
if path_is_local and os.path.exists(path):
if network:
m = ("file '%s' existed, possibly from local stage download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.INFO
if path_is_local:
level = logging.DEBUG
else:
m = ("file '%s' existed, possibly from previous boot download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.WARN
return (level, m)
kwargs = {'url': url, 'timeout': 10, 'retries': 2}
if network or path_is_local:
level = logging.WARN
kwargs['sec_between'] = 1
else:
level = logging.DEBUG
kwargs['sec_between'] = .1
data = None
header = b'#cloud-config'
try:
resp = url_helper.read_file_or_url(**kwargs)
if resp.ok():
data = resp.contents
if not resp.contents.startswith(header):
if cmdline_name == 'cloud-config-url':
level = logging.WARN
else:
level = logging.INFO
return (
level,
"contents of '%s' did not start with %s" % (url, header))
else:
return (level,
"url '%s' returned code %s. Ignoring." % (url, resp.code))
except url_helper.UrlError as e:
return (level, "retrieving url '%s' failed: %s" % (url, e))
util.write_file(path, data, mode=0o600)
return (logging.INFO,
"wrote cloud-config data from %s='%s' to %s" %
(cmdline_name, url, path))
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if 'phone_home' not in cfg:
log.debug(("Skipping module named %s, "
"no 'phone_home' configuration found"), name)
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warning(("Skipping module named %s, "
"no 'url' found in 'phone_home' configuration"), name)
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries')
try:
tries = int(tries)
except Exception:
tries = 10
util.logexc(
log, "Configuration entry 'tries' is not an integer, "
"using %s instead", tries)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
all_keys['fqdn'] = cloud.get_hostname(fqdn=True)
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log, "%s: failed to open, can not phone home that "
"data!", path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warning(("Requested key %s from 'post'"
" configuration list not available"), k)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = 'N/A'
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
'INSTANCE_ID': all_keys['instance_id'],
}
url = templater.render_string(url, url_params)
try:
url_helper.read_file_or_url(url,
data=real_submit_keys,
retries=tries,
sec_between=3,
ssl_details=util.fetch_ssl_details(
cloud.paths))
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
def handle(name, cfg, cloud, log, args):
if len(args) != 0:
ph_cfg = util.read_conf(args[0])
else:
if 'phone_home' not in cfg:
log.debug(("Skipping module named %s, "
"no 'phone_home' configuration found"), name)
return
ph_cfg = cfg['phone_home']
if 'url' not in ph_cfg:
log.warn(("Skipping module named %s, "
"no 'url' found in 'phone_home' configuration"), name)
return
url = ph_cfg['url']
post_list = ph_cfg.get('post', 'all')
tries = ph_cfg.get('tries')
try:
tries = int(tries)
except Exception:
tries = 10
util.logexc(log, "Configuration entry 'tries' is not an integer, "
"using %s instead", tries)
if post_list == "all":
post_list = POST_LIST_ALL
all_keys = {}
all_keys['instance_id'] = cloud.get_instance_id()
all_keys['hostname'] = cloud.get_hostname()
all_keys['fqdn'] = cloud.get_hostname(fqdn=True)
pubkeys = {
'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub',
'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
}
for (n, path) in pubkeys.items():
try:
all_keys[n] = util.load_file(path)
except Exception:
util.logexc(log, "%s: failed to open, can not phone home that "
"data!", path)
submit_keys = {}
for k in post_list:
if k in all_keys:
submit_keys[k] = all_keys[k]
else:
submit_keys[k] = None
log.warn(("Requested key %s from 'post'"
" configuration list not available"), k)
# Get them read to be posted
real_submit_keys = {}
for (k, v) in submit_keys.items():
if v is None:
real_submit_keys[k] = 'N/A'
else:
real_submit_keys[k] = str(v)
# Incase the url is parameterized
url_params = {
'INSTANCE_ID': all_keys['instance_id'],
}
url = templater.render_string(url, url_params)
try:
url_helper.read_file_or_url(
url, data=real_submit_keys, retries=tries, sec_between=3,
ssl_details=util.fetch_ssl_details(cloud.paths))
except Exception:
util.logexc(log, "Failed to post phone home data to %s in %s tries",
url, tries)
