def get_password(metadata_url=METADATA_URL, api_version=API_VERSION, password_server_port=PASSWORD_SERVER_PORT, url_timeout=URL_TIMEOUT, url_retries=URL_RETRIES): """Obtain the VM's password if set. Once fetched the password is marked saved. Future calls to this method may return empty string or 'saved_password'.""" password_url = "{}:{}/{}/".format(metadata_url, password_server_port, api_version) response = url_helper.read_file_or_url( password_url, ssl_details=None, headers={"DomU_Request": "send_my_password"}, timeout=url_timeout, retries=url_retries) password = response.contents.decode('utf-8') # the password is empty or already saved # Note: the original metadata server would answer an additional # 'bad_request' status, but the Exoscale implementation does not. if password in ['', 'saved_password']: return None # save the password url_helper.read_file_or_url(password_url, ssl_details=None, headers={"DomU_Request": "saved_password"}, timeout=url_timeout, retries=url_retries) return password
def test_read_file_or_url_str_from_url_redacts_noheaders(self): """When no headers_redact, header values are in logs and requests.""" url = 'http://hostname/path' headers = {'sensitive': 'sekret', 'server': 'blah'} httpretty.register_uri(httpretty.GET, url) read_file_or_url(url, headers=headers) for k in headers.keys(): self.assertEqual(headers[k], httpretty.last_request().headers[k]) logs = self.logs.getvalue() self.assertNotIn(REDACTED, logs) self.assertIn('sekret', logs)
def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): """Headers are redacted from logs but unredacted in requests.""" url = 'http://hostname/path' headers = {'sensitive': 'sekret', 'server': 'blah'} httpretty.register_uri(httpretty.GET, url) read_file_or_url(url, headers=headers, headers_redact=['sensitive']) logs = self.logs.getvalue() for k in headers.keys(): self.assertEqual(headers[k], httpretty.last_request().headers[k]) self.assertIn(REDACTED, logs) self.assertNotIn('sekret', logs)
def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self): """Headers are redacted from logs but unredacted in requests.""" url = 'http://hostname/path' headers = {'sensitive': 'sekret', 'server': 'blah'} httpretty.register_uri(httpretty.GET, url) # By default, httpretty will log our request along with the header, # so if we don't change this the secret will show up in the logs logging.getLogger('httpretty.core').setLevel(logging.CRITICAL) read_file_or_url(url, headers=headers, headers_redact=['sensitive']) logs = self.logs.getvalue() for k in headers.keys(): self.assertEqual(headers[k], httpretty.last_request().headers[k]) self.assertIn(REDACTED, logs) self.assertNotIn('sekret', logs)
def post(self, url, data=None, extra_headers=None): headers = self.headers if extra_headers is not None: headers = self.headers.copy() headers.update(extra_headers) return url_helper.read_file_or_url(url, data=data, headers=headers, timeout=5, retries=10)
def get(self, url, secure=False): headers = self.headers if secure: headers = self.headers.copy() headers.update(self.extra_secure_headers) return url_helper.read_file_or_url(url, headers=headers, timeout=5, retries=10)
def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5, headers_cb=None, headers_redact=None, exception_cb=None): ud_url = url_helper.combine_url(metadata_address, api_version) ud_url = url_helper.combine_url(ud_url, 'user-data') user_data = '' try: if not exception_cb: # It is ok for userdata to not exist (thats why we are stopping if # NOT_FOUND occurs) and just in that case returning an empty # string. exception_cb = functools.partial(skip_retry_on_codes, SKIP_USERDATA_CODES) response = url_helper.read_file_or_url(ud_url, ssl_details=ssl_details, timeout=timeout, retries=retries, exception_cb=exception_cb, headers_cb=headers_cb, headers_redact=headers_redact) user_data = response.contents except url_helper.UrlError as e: if e.code not in SKIP_USERDATA_CODES: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return user_data
def _do_include(self, content, append_msg): # Include a list of urls, one per line # also support '#include <url here>' # or #include-once '<url here>' include_once_on = False for line in content.splitlines(): lc_line = line.lower() if lc_line.startswith("#include-once"): line = line[len("#include-once"):].lstrip() # Every following include will now # not be refetched.... but will be # re-read from a local urlcache (if it worked) include_once_on = True elif lc_line.startswith("#include"): line = line[len("#include"):].lstrip() # Disable the include once if it was on # if it wasn't, then this has no effect. include_once_on = False if line.startswith("#"): continue include_url = line.strip() if not include_url: continue include_once_fn = None content = None if include_once_on: include_once_fn = self._get_include_once_filename(include_url) if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: try: resp = read_file_or_url(include_url, timeout=5, retries=10, ssl_details=self.ssl_details) if include_once_on and resp.ok(): util.write_file(include_once_fn, resp.contents, mode=0o600) if resp.ok(): content = resp.contents else: LOG.warning(("Fetching from %s resulted in" " a invalid http code of %s"), include_url, resp.code) except UrlError as urle: message = str(urle) # Older versions of requests.exceptions.HTTPError may not # include the errant url. Append it for clarity in logs. if include_url not in message: message += ' for url: {0}'.format(include_url) LOG.warning(message) except IOError as ioe: LOG.warning("Fetching from %s resulted in %s", include_url, ioe) if content is not None: new_msg = convert_string(content) self._process_msg(new_msg, append_msg)
def test_readurl_timeout(self, readurl_timeout, request_timeout): url = "http://hostname/path" m_response = mock.MagicMock() class FakeSession(requests.Session): @classmethod def request(cls, **kwargs): expected_kwargs = { "url": url, "allow_redirects": True, "method": "GET", "headers": { "User-Agent": "Cloud-Init/%s" % (version.version_string()) }, "timeout": request_timeout, } if request_timeout is None: expected_kwargs.pop("timeout") assert kwargs == expected_kwargs return m_response with mock.patch( M_PATH + "requests.Session", side_effect=[FakeSession()] ): response = read_file_or_url(url, timeout=readurl_timeout) assert response._response == m_response
def test_read_file_or_url_passes_params_to_readurl(self, m_readurl, timeout): """read_file_or_url passes all params through to readurl.""" url = "http://hostname/path" response = "This is my url content\n" m_readurl.return_value = response params = { "url": url, "timeout": timeout, "retries": 2, "headers": { "somehdr": "val" }, "data": "data", "sec_between": 1, "ssl_details": { "cert_file": "/path/cert.pem" }, "headers_cb": "headers_cb", "exception_cb": "exception_cb", } assert response == read_file_or_url(**params) params.pop("url") # url is passed in as a positional arg assert m_readurl.call_args_list == [mock.call(url, **params)]
def test_read_file_or_url_str_from_file(self): """Test that str(result.contents) on file is text version of contents. It should not be "b'data'", but just "'data'" """ tmpf = self.tmp_path("myfile1") data = b'This is my file content\n' util.write_file(tmpf, data, omode="wb") result = read_file_or_url("file://%s" % tmpf) self.assertEqual(result.contents, data) self.assertEqual(str(result), data.decode('utf-8'))
def test_read_file_or_url_str_from_url(self): """Test that str(result.contents) on url is text version of contents. It should not be "b'data'", but just "'data'" """ url = 'http://hostname/path' data = b'This is my url content\n' httpretty.register_uri(httpretty.GET, url, data) result = read_file_or_url(url) self.assertEqual(result.contents, data) self.assertEqual(str(result), data.decode('utf-8'))
def test_read_file_or_url_str_from_file(self): """Test that str(result.contents) on file is text version of contents. It should not be "b'data'", but just "'data'" """ tmpf = self.tmp_path("myfile1") data = b'This is my file content\n' util.write_file(tmpf, data, omode="wb") result = read_file_or_url("file://%s" % tmpf) self.assertEqual(result.contents, data) self.assertEqual(str(result), data.decode('utf-8'))
def test_read_file_or_url_str_from_url(self): """Test that str(result.contents) on url is text version of contents. It should not be "b'data'", but just "'data'" """ url = 'http://hostname/path' data = b'This is my url content\n' httpretty.register_uri(httpretty.GET, url, data) result = read_file_or_url(url) self.assertEqual(result.contents, data) self.assertEqual(str(result), data.decode('utf-8'))
def test_wb_read_url_defaults_honored_by_read_file_or_url_callers(self): """Readurl param defaults used when unspecified by read_file_or_url Param defaults tested are as follows: retries: 0, additional headers None beyond default, method: GET, data: None, check_status: True and allow_redirects: True """ url = "http://hostname/path" m_response = mock.MagicMock() class FakeSession(requests.Session): @classmethod def request(cls, **kwargs): self.assertEqual( { "url": url, "allow_redirects": True, "method": "GET", "headers": { "User-Agent": "Cloud-Init/%s" % (version.version_string()) }, }, kwargs, ) return m_response with mock.patch(M_PATH + "requests.Session") as m_session: error = requests.exceptions.HTTPError("broke") m_session.side_effect = [error, FakeSession()] # assert no retries and check_status == True with self.assertRaises(UrlError) as context_manager: response = read_file_or_url(url) self.assertEqual("broke", str(context_manager.exception)) # assert default headers, method, url and allow_redirects True # Success on 2nd call with FakeSession response = read_file_or_url(url) self.assertEqual(m_response, response._response)
def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): ud_url = url_helper.combine_url(metadata_address, api_version) ud_url = url_helper.combine_url(ud_url, 'user-data') user_data = '' try: # It is ok for userdata to not exist (thats why we are stopping if # NOT_FOUND occurs) and just in that case returning an empty string. exception_cb = functools.partial(_skip_retry_on_codes, SKIP_USERDATA_CODES) response = url_helper.read_file_or_url( ud_url, ssl_details=ssl_details, timeout=timeout, retries=retries, exception_cb=exception_cb) user_data = response.contents except url_helper.UrlError as e: if e.code not in SKIP_USERDATA_CODES: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return user_data
def test_read_file_or_url_passes_params_to_readurl(self, m_readurl): """read_file_or_url passes all params through to readurl.""" url = 'http://hostname/path' response = 'This is my url content\n' m_readurl.return_value = response params = { 'url': url, 'timeout': 1, 'retries': 2, 'headers': { 'somehdr': 'val' }, 'data': 'data', 'sec_between': 1, 'ssl_details': { 'cert_file': '/path/cert.pem' }, 'headers_cb': 'headers_cb', 'exception_cb': 'exception_cb' } self.assertEqual(response, read_file_or_url(**params)) params.pop('url') # url is passed in as a positional arg self.assertEqual([mock.call(url, **params)], m_readurl.call_args_list)
def attempt_cmdline_url(path, network=True, cmdline=None): """Write data from url referenced in command line to path. path: a file to write content to if downloaded. network: should network access be assumed. cmdline: the cmdline to parse for cloud-config-url. This is used in MAAS datasource, in "ephemeral" (read-only root) environment where the instance netboots to iscsi ro root. and the entity that controls the pxe config has to configure the maas datasource. An attempt is made on network urls even in local datasource for case of network set up in initramfs. Return value is a tuple of a logger function (logging.DEBUG) and a message indicating what happened. """ if cmdline is None: cmdline = util.get_cmdline() try: cmdline_name, url = parse_cmdline_url(cmdline) except KeyError: return (logging.DEBUG, "No kernel command line url found.") path_is_local = url.startswith("file://") or url.startswith("/") if path_is_local and os.path.exists(path): if network: m = ("file '%s' existed, possibly from local stage download" " of command line url '%s'. Not re-writing." % (path, url)) level = logging.INFO if path_is_local: level = logging.DEBUG else: m = ("file '%s' existed, possibly from previous boot download" " of command line url '%s'. Not re-writing." % (path, url)) level = logging.WARN return (level, m) kwargs = {'url': url, 'timeout': 10, 'retries': 2} if network or path_is_local: level = logging.WARN kwargs['sec_between'] = 1 else: level = logging.DEBUG kwargs['sec_between'] = .1 data = None header = b'#cloud-config' try: resp = url_helper.read_file_or_url(**kwargs) if resp.ok(): data = resp.contents if not resp.contents.startswith(header): if cmdline_name == 'cloud-config-url': level = logging.WARN else: level = logging.INFO return (level, "contents of '%s' did not start with %s" % (url, header)) else: return (level, "url '%s' returned code %s. Ignoring." % (url, resp.code)) except url_helper.UrlError as e: return (level, "retrieving url '%s' failed: %s" % (url, e)) util.write_file(path, data, mode=0o600) return (logging.INFO, "wrote cloud-config data from %s='%s' to %s" % (cmdline_name, url, path))
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if "phone_home" not in cfg: log.debug( "Skipping module named %s, " "no 'phone_home' configuration found", name, ) return ph_cfg = cfg["phone_home"] if "url" not in ph_cfg: log.warning( "Skipping module named %s, " "no 'url' found in 'phone_home' configuration", name, ) return url = ph_cfg["url"] post_list = ph_cfg.get("post", "all") tries = ph_cfg.get("tries") try: tries = int(tries) # type: ignore except ValueError: tries = 10 util.logexc( log, "Configuration entry 'tries' is not an integer, using %s instead", tries, ) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys["instance_id"] = cloud.get_instance_id() all_keys["hostname"] = cloud.get_hostname() all_keys["fqdn"] = cloud.get_hostname(fqdn=True) pubkeys = { "pub_key_dsa": "/etc/ssh/ssh_host_dsa_key.pub", "pub_key_rsa": "/etc/ssh/ssh_host_rsa_key.pub", "pub_key_ecdsa": "/etc/ssh/ssh_host_ecdsa_key.pub", "pub_key_ed25519": "/etc/ssh/ssh_host_ed25519_key.pub", } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warning( "Requested key %s from 'post'" " configuration list not available", k, ) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = "N/A" else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { "INSTANCE_ID": all_keys["instance_id"], } url = templater.render_string(url, url_params) try: url_helper.read_file_or_url( url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths), ) except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
def get(self, url, secure=False): headers = self.headers if secure: headers = self.headers.copy() headers.update(self.extra_secure_headers) return url_helper.read_file_or_url(url, headers=headers)
def post(self, url, data=None, extra_headers=None): headers = self.headers if extra_headers is not None: headers = self.headers.copy() headers.update(extra_headers) return url_helper.read_file_or_url(url, data=data, headers=headers)
def attempt_cmdline_url(path, network=True, cmdline=None): """Write data from url referenced in command line to path. path: a file to write content to if downloaded. network: should network access be assumed. cmdline: the cmdline to parse for cloud-config-url. This is used in MAAS datasource, in "ephemeral" (read-only root) environment where the instance netboots to iscsi ro root. and the entity that controls the pxe config has to configure the maas datasource. An attempt is made on network urls even in local datasource for case of network set up in initramfs. Return value is a tuple of a logger function (logging.DEBUG) and a message indicating what happened. """ if cmdline is None: cmdline = util.get_cmdline() try: cmdline_name, url = parse_cmdline_url(cmdline) except KeyError: return (logging.DEBUG, "No kernel command line url found.") path_is_local = url.startswith("file://") or url.startswith("/") if path_is_local and os.path.exists(path): if network: m = ("file '%s' existed, possibly from local stage download" " of command line url '%s'. Not re-writing." % (path, url)) level = logging.INFO if path_is_local: level = logging.DEBUG else: m = ("file '%s' existed, possibly from previous boot download" " of command line url '%s'. Not re-writing." % (path, url)) level = logging.WARN return (level, m) kwargs = {'url': url, 'timeout': 10, 'retries': 2} if network or path_is_local: level = logging.WARN kwargs['sec_between'] = 1 else: level = logging.DEBUG kwargs['sec_between'] = .1 data = None header = b'#cloud-config' try: resp = url_helper.read_file_or_url(**kwargs) if resp.ok(): data = resp.contents if not resp.contents.startswith(header): if cmdline_name == 'cloud-config-url': level = logging.WARN else: level = logging.INFO return ( level, "contents of '%s' did not start with %s" % (url, header)) else: return (level, "url '%s' returned code %s. Ignoring." % (url, resp.code)) except url_helper.UrlError as e: return (level, "retrieving url '%s' failed: %s" % (url, e)) util.write_file(path, data, mode=0o600) return (logging.INFO, "wrote cloud-config data from %s='%s' to %s" % (cmdline_name, url, path))
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if 'phone_home' not in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: log.warning(("Skipping module named %s, " "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] post_list = ph_cfg.get('post', 'all') tries = ph_cfg.get('tries') try: tries = int(tries) except Exception: tries = 10 util.logexc( log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() all_keys['hostname'] = cloud.get_hostname() all_keys['fqdn'] = cloud.get_hostname(fqdn=True) pubkeys = { 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warning(("Requested key %s from 'post'" " configuration list not available"), k) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = 'N/A' else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { 'INSTANCE_ID': all_keys['instance_id'], } url = templater.render_string(url, url_params) try: url_helper.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details( cloud.paths)) except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: if 'phone_home' not in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: log.warn(("Skipping module named %s, " "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] post_list = ph_cfg.get('post', 'all') tries = ph_cfg.get('tries') try: tries = int(tries) except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() all_keys['hostname'] = cloud.get_hostname() all_keys['fqdn'] = cloud.get_hostname(fqdn=True) pubkeys = { 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: submit_keys[k] = None log.warn(("Requested key %s from 'post'" " configuration list not available"), k) # Get them read to be posted real_submit_keys = {} for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = 'N/A' else: real_submit_keys[k] = str(v) # Incase the url is parameterized url_params = { 'INSTANCE_ID': all_keys['instance_id'], } url = templater.render_string(url, url_params) try: url_helper.read_file_or_url( url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)