def _maybe_fetch_api_token(self, mdurls, timeout=None, max_wait=None):
if self.cloud_name != CloudNames.AWS:
return
urls = []
url2base = {}
url_path = API_TOKEN_ROUTE
request_method = 'PUT'
for url in mdurls:
cur = '{0}/{1}'.format(url, url_path)
urls.append(cur)
url2base[cur] = url
# use the self._status_cb to check for Read errors, which means
# we can't reach the API token URL, so we should disable IMDSv2
LOG.debug('Fetching Ec2 IMDSv2 API Token')
url, response = uhelp.wait_for_url(urls=urls,
max_wait=1,
timeout=1,
status_cb=self._status_cb,
headers_cb=self._get_headers,
request_method=request_method,
headers_redact=AWS_TOKEN_REDACT)
if url and response:
self._api_token = response
return url2base[url]
def wait_for_metadata_service(self):
urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
filtered = [x for x in urls if util.is_resolvable_url(x)]
if set(filtered) != set(urls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(urls) - set(filtered))))
if len(filtered):
urls = filtered
else:
LOG.warn("Empty metadata url list! using default list")
urls = [DEF_MD_URL]
md_urls = []
url2base = {}
for url in urls:
md_url = url_helper.combine_url(url, 'openstack')
md_urls.append(md_url)
url2base[md_url] = url
(max_wait, timeout) = self._get_url_settings()
start_time = time.time()
avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
timeout=timeout)
if avail_url:
LOG.debug("Using metadata source: '%s'", url2base[avail_url])
else:
LOG.debug("Giving up on OpenStack md from %s after %s seconds",
md_urls, int(time.time() - start_time))
self.metadata_address = url2base.get(avail_url)
return bool(avail_url)
def wait_for_metadata_service(self, url):
mcfg = self.ds_cfg
max_wait = 120
try:
max_wait = int(mcfg.get("max_wait", max_wait))
except Exception:
util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
if max_wait == 0:
return False
timeout = 50
try:
if timeout in mcfg:
timeout = int(mcfg.get("timeout", timeout))
except Exception:
LOG.warn("Failed to get timeout, using %s" % timeout)
starttime = time.time()
check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
urls = [check_url]
url = url_helper.wait_for_url(urls=urls, max_wait=max_wait,
timeout=timeout,
exception_cb=self._except_cb,
headers_cb=self._md_headers)
if url:
LOG.debug("Using metadata source: '%s'", url)
else:
LOG.critical("Giving up on md from %s after %i seconds",
urls, int(time.time() - starttime))
return bool(url)
def wait_for_metadata_service(self):
url_params = self.get_url_params()
if url_params.max_wait_seconds <= 0:
return False
urls = [
uhelp.combine_url(self.metadata_address,
'latest/meta-data/instance-id')
]
start_time = time.time()
url, _response = uhelp.wait_for_url(
urls=urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
status_cb=LOG.warning)
if url:
LOG.debug("Using metadata source: '%s'", url)
else:
LOG.critical(("Giving up on waiting for the metadata from %s"
" after %s seconds"), urls,
int(time.time() - start_time))
return bool(url)
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
url_params = self.get_url_params()
if url_params.max_wait_seconds <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", self.metadata_urls)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
LOG.warning("Empty metadata url list! using default list")
mdurls = self.metadata_urls
# try the api token path first
metadata_address = self._maybe_fetch_api_token(mdurls)
# When running on EC2, we always access IMDS with an API token.
# If we could not get an API token, then we assume the IMDS
# endpoint was disabled and we move on without a data source.
# Fallback to IMDSv1 if not running on EC2
if not metadata_address and self.cloud_name != CloudNames.AWS:
# if we can't get a token, use instance-id path
urls = []
url2base = {}
url_path = '{ver}/meta-data/instance-id'.format(
ver=self.min_metadata_version)
request_method = 'GET'
for url in mdurls:
cur = '{0}/{1}'.format(url, url_path)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
url, _ = uhelp.wait_for_url(
urls=urls, max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds, status_cb=LOG.warning,
headers_redact=AWS_TOKEN_REDACT, headers_cb=self._get_headers,
request_method=request_method)
if url:
metadata_address = url2base[url]
if metadata_address:
self.metadata_address = metadata_address
LOG.debug("Using metadata source: '%s'", self.metadata_address)
elif self.cloud_name == CloudNames.AWS:
LOG.warning("IMDS's HTTP endpoint is probably disabled")
else:
LOG.critical("Giving up on md from %s after %s seconds",
urls, int(time.time() - start_time))
return bool(metadata_address)
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
url_params = self.get_url_params()
if url_params.max_wait_seconds <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", self.metadata_urls)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
LOG.warning("Empty metadata url list! using default list")
mdurls = self.metadata_urls
# try the api token path first
metadata_address = self._maybe_fetch_api_token(mdurls)
if not metadata_address:
if self._api_token == API_TOKEN_DISABLED:
LOG.warning('Retrying with IMDSv1')
# if we can't get a token, use instance-id path
urls = []
url2base = {}
url_path = '{ver}/meta-data/instance-id'.format(
ver=self.min_metadata_version)
request_method = 'GET'
for url in mdurls:
cur = '{0}/{1}'.format(url, url_path)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
url, _ = uhelp.wait_for_url(urls=urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
status_cb=LOG.warning,
headers_redact=AWS_TOKEN_REDACT,
headers_cb=self._get_headers,
request_method=request_method)
if url:
metadata_address = url2base[url]
if metadata_address:
self.metadata_address = metadata_address
LOG.debug("Using metadata source: '%s'", self.metadata_address)
else:
LOG.critical("Giving up on md from %s after %s seconds", urls,
int(time.time() - start_time))
return bool(metadata_address)
def wait_for_metadata_service(self):
"""Wait for the metadata service to be reachable."""
metadata_url = "{}/{}/meta-data/instance-id".format(
self.metadata_url, self.api_version)
url = url_helper.wait_for_url(urls=[metadata_url],
max_wait=self.url_max_wait,
timeout=self.url_timeout,
status_cb=LOG.critical)
return bool(url)
def _poll_imds(self, report_ready=True):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "?api-version=2017-04-02"
headers = {"Metadata": "true"}
LOG.debug("Start polling IMDS")
def sleep_cb(response, loop_n):
return 1
def exception_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return
LOG.warning("Exception during polling. Will try DHCP.",
exc_info=True)
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
raise exception
need_report = report_ready
for i in range(IMDS_RETRIES):
try:
with EphemeralDHCPv4() as lease:
if need_report:
self._report_ready(lease=lease)
need_report = False
wait_for_url([url],
max_wait=None,
timeout=60,
status_cb=LOG.info,
headers_cb=lambda url: headers,
sleep_time=1,
exception_cb=exception_cb,
sleep_time_cb=sleep_cb)
return str(readurl(url, headers=headers))
except Exception:
LOG.debug("Exception during polling-retrying dhcp" +
" %d more time(s).", (IMDS_RETRIES - i),
exc_info=True)
def _maybe_fetch_api_token(self, mdurls, timeout=None, max_wait=None):
"""Get an API token for EC2 Instance Metadata Service.
On EC2. IMDS will always answer an API token, unless
the instance owner has disabled the IMDS HTTP endpoint or
the network topology conflicts with the configured hop-limit.
"""
if self.cloud_name != CloudNames.AWS:
return
urls = []
url2base = {}
url_path = API_TOKEN_ROUTE
request_method = "PUT"
for url in mdurls:
cur = "{0}/{1}".format(url, url_path)
urls.append(cur)
url2base[cur] = url
# use the self._imds_exception_cb to check for Read errors
LOG.debug("Fetching Ec2 IMDSv2 API Token")
response = None
url = None
url_params = self.get_url_params()
try:
url, response = uhelp.wait_for_url(
urls=urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
status_cb=LOG.warning,
headers_cb=self._get_headers,
exception_cb=self._imds_exception_cb,
request_method=request_method,
headers_redact=AWS_TOKEN_REDACT,
connect_synchronously=False,
)
except uhelp.UrlError:
# We use the raised exception to interupt the retry loop.
# Nothing else to do here.
pass
if url and response:
self._api_token = response
return url2base[url]
# If we get here, then wait_for_url timed out, waiting for IMDS
# or the IMDS HTTP endpoint is disabled
return None
def wait_for_metadata_service(self):
(max_wait, timeout) = self._get_url_settings()
urls = [uhelp.combine_url(self.metadata_address,
'latest/meta-data/instance-id')]
start_time = time.time()
url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
timeout=timeout, status_cb=LOG.warn)
if url:
LOG.debug("Using metadata source: '%s'", url)
else:
LOG.critical(("Giving up on waiting for the metadata from %s"
" after %s seconds"),
urls, int(time.time() - start_time))
return bool(url)
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
if not mcfg:
mcfg = {}
(max_wait, timeout) = self._get_url_settings()
if max_wait <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", DEF_MD_URLS)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
LOG.warn("Empty metadata url list! using default list")
mdurls = DEF_MD_URLS
urls = []
url2base = {}
for url in mdurls:
cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
url = uhelp.wait_for_url(urls=urls,
max_wait=max_wait,
timeout=timeout,
status_cb=LOG.warn)
if url:
LOG.debug("Using metadata source: '%s'", url2base[url])
else:
LOG.critical("Giving up on md from %s after %s seconds", urls,
int(time.time() - start_time))
self.metadata_address = url2base.get(url)
return bool(url)
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
url_params = self.get_url_params()
if url_params.max_wait_seconds <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", self.metadata_urls)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
LOG.warning("Empty metadata url list! using default list")
mdurls = self.metadata_urls
urls = []
url2base = {}
for url in mdurls:
cur = '{0}/{1}/meta-data/instance-id'.format(
url, self.min_metadata_version)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
url = uhelp.wait_for_url(urls=urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
status_cb=LOG.warn)
if url:
self.metadata_address = url2base[url]
LOG.debug("Using metadata source: '%s'", self.metadata_address)
else:
LOG.critical("Giving up on md from %s after %s seconds", urls,
int(time.time() - start_time))
return bool(url)
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
if not mcfg:
mcfg = {}
(max_wait, timeout) = self._get_url_settings()
if max_wait <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", DEF_MD_URLS)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
LOG.warn("Empty metadata url list! using default list")
mdurls = DEF_MD_URLS
urls = []
url2base = {}
for url in mdurls:
cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
timeout=timeout, status_cb=LOG.warn)
if url:
LOG.debug("Using metadata source: '%s'", url2base[url])
else:
LOG.critical("Giving up on md from %s after %s seconds",
urls, int(time.time() - start_time))
self.metadata_address = url2base.get(url)
return bool(url)
def wait_for_metadata_service(self):
mcfg = self.ds_cfg
url_params = self.get_url_params()
if url_params.max_wait_seconds <= 0:
return False
# Remove addresses from the list that wont resolve.
mdurls = mcfg.get("metadata_urls", self.metadata_urls)
filtered = [x for x in mdurls if util.is_resolvable_url(x)]
if set(filtered) != set(mdurls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(mdurls) - set(filtered))))
if len(filtered):
mdurls = filtered
else:
LOG.warning("Empty metadata url list! using default list")
mdurls = self.metadata_urls
urls = []
url2base = {}
for url in mdurls:
cur = '{0}/{1}/meta-data/instance-id'.format(
url, self.min_metadata_version)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
url = uhelp.wait_for_url(
urls=urls, max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds, status_cb=LOG.warning)
if url:
self.metadata_address = url2base[url]
LOG.debug("Using metadata source: '%s'", self.metadata_address)
else:
LOG.critical("Giving up on md from %s after %s seconds",
urls, int(time.time() - start_time))
return bool(url)
def wait_for_metadata_service(self):
urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
filtered = [x for x in urls if util.is_resolvable_url(x)]
if set(filtered) != set(urls):
LOG.debug(
"Removed the following from metadata urls: %s",
list((set(urls) - set(filtered))),
)
if len(filtered):
urls = filtered
else:
LOG.warning("Empty metadata url list! using default list")
urls = [DEF_MD_URL]
md_urls = []
url2base = {}
for url in urls:
md_url = url_helper.combine_url(url, "openstack")
md_urls.append(md_url)
url2base[md_url] = url
url_params = self.get_url_params()
start_time = time.time()
avail_url, _response = url_helper.wait_for_url(
urls=md_urls,
max_wait=url_params.max_wait_seconds,
timeout=url_params.timeout_seconds,
)
if avail_url:
LOG.debug("Using metadata source: '%s'", url2base[avail_url])
else:
LOG.debug(
"Giving up on OpenStack md from %s after %s seconds",
md_urls,
int(time.time() - start_time),
)
self.metadata_address = url2base.get(avail_url)
return bool(avail_url)
def test_order(self, addresses, expected_address_index, response):
"""Check that the first response gets returned. Simulate a
non-responding endpoint with a response that has a one second wait.
If this test proves flaky, increase wait time. Since it is async,
increasing wait time for the non-responding endpoint should not
increase total test time, assuming async_delay=0 is used and at least
one non-waiting endpoint is registered with httpretty.
Subsequent tests will continue execution after the first response is
received.
"""
self.event.clear()
for address in set(addresses):
responses.add_callback(
responses.GET,
address,
callback=(
self.response_wait
if "sleep" in address
else self.response_nowait
),
content_type="application/json",
)
# Use async_delay=0.0 to avoid adding unnecessary time to tests
# In practice a value such as 0.150 is used
url, response_contents = wait_for_url(
urls=addresses,
max_wait=1,
timeout=1,
connect_synchronously=False,
async_delay=0.0,
)
self.event.set()
# Test for timeout (no responding endpoint)
assert addresses[expected_address_index] == url
assert response.encode() == response_contents
