def get_metadata_from_imds(fallback_nic, retries):
"""Query Azure's network metadata service, returning a dictionary.
If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
IMDS. For more info on IMDS:
https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service
@param fallback_nic: String. The name of the nic which requires active
network in order to query IMDS.
@param retries: The number of retries of the IMDS_URL.
@return: A dict of instance metadata containing compute and network
info.
"""
kwargs = {
'logfunc': LOG.debug,
'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
'func': _get_metadata_from_imds,
'args': (retries, )
}
if net.is_up(fallback_nic):
return util.log_time(**kwargs)
else:
with EphemeralDHCPv4(fallback_nic):
return util.log_time(**kwargs)
def _poll_imds(self, report_ready=True):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "?api-version=2017-04-02"
headers = {"Metadata": "true"}
LOG.debug("Start polling IMDS")
def exc_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return True
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
return False
need_report = report_ready
while True:
try:
with EphemeralDHCPv4() as lease:
if need_report:
self._report_ready(lease=lease)
need_report = False
return readurl(url,
timeout=1,
headers=headers,
exception_cb=exc_cb,
infinite=True).contents
except UrlError:
pass
def get_metadata(url, timeout, retries, sec_between, agent):
# Bring up interface (and try untill one works)
exception = RuntimeError("Failed to DHCP")
# Seek iface with DHCP
for iface in get_interface_list():
try:
with EphemeralDHCPv4(
iface=iface, connectivity_url_data={"url": url}
):
# Check for the metadata route, skip if not there
if not check_route(url):
continue
# Fetch the metadata
v1 = read_metadata(url, timeout, retries, sec_between, agent)
return json.loads(v1)
except (
NoDHCPLeaseError,
subp.ProcessExecutionError,
RuntimeError,
exceptions.RequestException,
) as exc:
LOG.error("DHCP Exception: %s", exc)
exception = exc
raise exception
def _poll_imds(self):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
headers = {"Metadata": "true"}
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
LOG.debug("Start polling IMDS")
while True:
try:
# Save our EphemeralDHCPv4 context so we avoid repeated dhcp
self._ephemeral_dhcp_ctx = EphemeralDHCPv4()
lease = self._ephemeral_dhcp_ctx.obtain_lease()
if report_ready:
path = REPORTED_READY_MARKER_FILE
LOG.info("Creating a marker file to report ready: %s",
path)
util.write_file(
path, "{pid}: {time}\n".format(pid=os.getpid(),
time=time()))
self._report_ready(lease=lease)
report_ready = False
return readurl(url,
timeout=1,
headers=headers,
exception_cb=retry_on_url_exc,
infinite=True,
log_req_resp=False).contents
except UrlError:
# Teardown our EphemeralDHCPv4 context on failure as we retry
self._ephemeral_dhcp_ctx.clean_network()
pass
def _get_data(self):
url_params = self.get_url_params()
network_context = noop()
if self.perform_dhcp_setup:
network_context = EphemeralDHCPv4(self.fallback_interface)
with network_context:
ret = util.log_time(
LOG.debug,
"Crawl of GCE metadata service",
read_md,
kwargs={
"address": self.metadata_address,
"url_params": url_params,
},
)
if not ret["success"]:
if ret["platform_reports_gce"]:
LOG.warning(ret["reason"])
else:
LOG.debug(ret["reason"])
return False
self.metadata = ret["meta-data"]
self.userdata_raw = ret["user-data"]
return True
def _get_data(self):
strict_mode, _sleep = read_strict_mode(
util.get_cfg_by_path(self.sys_cfg, STRICT_ID_PATH,
STRICT_ID_DEFAULT), ("warn", None))
LOG.debug("strict_mode: %s, cloud_name=%s cloud_platform=%s",
strict_mode, self.cloud_name, self.platform)
if strict_mode == "true" and self.cloud_name == CloudNames.UNKNOWN:
return False
elif self.cloud_name == CloudNames.NO_EC2_METADATA:
return False
if self.perform_dhcp_setup: # Setup networking in init-local stage.
if util.is_FreeBSD():
LOG.debug("FreeBSD doesn't support running dhclient with -sf")
return False
try:
with EphemeralDHCPv4(self.fallback_interface):
self._crawled_metadata = util.log_time(
logfunc=LOG.debug, msg='Crawl of metadata service',
func=self.crawl_metadata)
except NoDHCPLeaseError:
return False
else:
self._crawled_metadata = util.log_time(
logfunc=LOG.debug, msg='Crawl of metadata service',
func=self.crawl_metadata)
if not self._crawled_metadata:
return False
self.metadata = self._crawled_metadata.get('meta-data', None)
self.userdata_raw = self._crawled_metadata.get('user-data', None)
self.identity = self._crawled_metadata.get(
'dynamic', {}).get('instance-identity', {}).get('document', {})
return True
def _poll_imds(self):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
headers = {"Metadata": "true"}
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
LOG.debug("Start polling IMDS")
def exc_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return True
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
return False
while True:
try:
with EphemeralDHCPv4() as lease:
if report_ready:
path = REPORTED_READY_MARKER_FILE
LOG.info("Creating a marker file to report ready: %s",
path)
util.write_file(
path, "{pid}: {time}\n".format(pid=os.getpid(),
time=time()))
self._report_ready(lease=lease)
report_ready = False
return readurl(url,
timeout=1,
headers=headers,
exception_cb=exc_cb,
infinite=True).contents
except UrlError:
pass
def _get_data(self):
seed_ret = {}
if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
self.userdata_raw = seed_ret['user-data']
self.metadata = seed_ret['meta-data']
LOG.debug("Using seeded ec2 data from %s", self.seed_dir)
self._cloud_platform = Platforms.SEEDED
return True
strict_mode, _sleep = read_strict_mode(
util.get_cfg_by_path(self.sys_cfg, STRICT_ID_PATH,
STRICT_ID_DEFAULT), ("warn", None))
LOG.debug("strict_mode: %s, cloud_platform=%s",
strict_mode, self.cloud_platform)
if strict_mode == "true" and self.cloud_platform == Platforms.UNKNOWN:
return False
elif self.cloud_platform == Platforms.NO_EC2_METADATA:
return False
if self.perform_dhcp_setup: # Setup networking in init-local stage.
if util.is_FreeBSD():
LOG.debug("FreeBSD doesn't support running dhclient with -sf")
return False
try:
with EphemeralDHCPv4(self.fallback_interface):
return util.log_time(
logfunc=LOG.debug, msg='Crawl of metadata service',
func=self._crawl_metadata)
except NoDHCPLeaseError:
return False
else:
return self._crawl_metadata()
def _get_data(self):
"""Crawl metadata, parse and persist that data for this instance.
@return: True when metadata discovered indicates OpenStack datasource.
False when unable to contact metadata service or when metadata
format is invalid or disabled.
"""
oracle_considered = "Oracle" in self.sys_cfg.get("datasource_list")
if not detect_openstack(accept_oracle=not oracle_considered):
return False
if self.perform_dhcp_setup: # Setup networking in init-local stage.
try:
with EphemeralDHCPv4(self.fallback_interface):
results = util.log_time(
logfunc=LOG.debug,
msg="Crawl of metadata service",
func=self._crawl_metadata,
)
except (NoDHCPLeaseError, sources.InvalidMetaDataException) as e:
util.logexc(LOG, str(e))
return False
else:
try:
results = self._crawl_metadata()
except sources.InvalidMetaDataException as e:
util.logexc(LOG, str(e))
return False
self.dsmode = self._determine_dsmode([results.get("dsmode")])
if self.dsmode == sources.DSMODE_DISABLED:
return False
md = results.get("metadata", {})
md = util.mergemanydict([md, DEFAULT_METADATA])
self.metadata = md
self.ec2_metadata = results.get("ec2-metadata")
self.network_json = results.get("networkdata")
self.userdata_raw = results.get("userdata")
self.version = results["version"]
self.files.update(results.get("files", {}))
vd = results.get("vendordata")
self.vendordata_pure = vd
try:
self.vendordata_raw = sources.convert_vendordata(vd)
except ValueError as e:
LOG.warning("Invalid content in vendor-data: %s", e)
self.vendordata_raw = None
vd2 = results.get("vendordata2")
self.vendordata2_pure = vd2
try:
self.vendordata2_raw = sources.convert_vendordata(vd2)
except ValueError as e:
LOG.warning("Invalid content in vendor-data2: %s", e)
self.vendordata2_raw = None
return True
def _poll_imds(self):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
headers = {"Metadata": "true"}
nl_sock = None
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
def exc_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return True
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
return False
LOG.debug("Wait for vnetswitch to happen")
while True:
try:
# Save our EphemeralDHCPv4 context so we avoid repeated dhcp
self._ephemeral_dhcp_ctx = EphemeralDHCPv4()
lease = self._ephemeral_dhcp_ctx.obtain_lease()
if report_ready:
try:
nl_sock = netlink.create_bound_netlink_socket()
except netlink.NetlinkCreateSocketError as e:
LOG.warning(e)
self._ephemeral_dhcp_ctx.clean_network()
return
path = REPORTED_READY_MARKER_FILE
LOG.info("Creating a marker file to report ready: %s",
path)
util.write_file(
path, "{pid}: {time}\n".format(pid=os.getpid(),
time=time()))
self._report_ready(lease=lease)
report_ready = False
try:
netlink.wait_for_media_disconnect_connect(
nl_sock, lease['interface'])
except AssertionError as error:
LOG.error(error)
return
self._ephemeral_dhcp_ctx.clean_network()
else:
return readurl(url,
timeout=1,
headers=headers,
exception_cb=exc_cb,
infinite=True,
log_req_resp=False).contents
except UrlError:
# Teardown our EphemeralDHCPv4 context on failure as we retry
self._ephemeral_dhcp_ctx.clean_network()
pass
finally:
if nl_sock:
nl_sock.close()
def get_metadata(url, timeout, retries, sec_between, agent):
# Bring up interface
try:
with EphemeralDHCPv4(connectivity_url_data={"url": url}):
# Fetch the metadata
v1 = read_metadata(url, timeout, retries, sec_between, agent)
except (NoDHCPLeaseError) as exc:
LOG.error("Bailing, DHCP Exception: %s", exc)
raise
return json.loads(v1)
def _get_data(self):
(on_hetzner, serial) = get_hcloud_data()
if not on_hetzner:
return False
try:
with EphemeralDHCPv4(
iface=net.find_fallback_nic(),
connectivity_url_data={
"url": BASE_URL_V1 + "/metadata/instance-id",
},
):
md = hc_helper.read_metadata(
self.metadata_address,
timeout=self.timeout,
sec_between=self.wait_retry,
retries=self.retries,
)
ud = hc_helper.read_userdata(
self.userdata_address,
timeout=self.timeout,
sec_between=self.wait_retry,
retries=self.retries,
)
except (NoDHCPLeaseError) as e:
LOG.error("Bailing, DHCP Exception: %s", e)
raise
# Hetzner cloud does not support binary user-data. So here, do a
# base64 decode of the data if we can. The end result being that a
# user can provide base64 encoded (possibly gzipped) data as user-data.
#
# The fallout is that in the event of b64 encoded user-data,
# /var/lib/cloud-init/cloud-config.txt will not be identical to the
# user-data provided. It will be decoded.
self.userdata_raw = hc_helper.maybe_b64decode(ud)
self.metadata_full = md
# hostname is name provided by user at launch. The API enforces it is
# a valid hostname, but it is not guaranteed to be resolvable in dns or
# fully qualified.
self.metadata["instance-id"] = md["instance-id"]
self.metadata["local-hostname"] = md["hostname"]
self.metadata["network-config"] = md.get("network-config", None)
self.metadata["public-keys"] = md.get("public-keys", None)
self.vendordata_raw = md.get("vendor_data", None)
# instance-id and serial from SMBIOS should be identical
if self.get_instance_id() != serial:
raise RuntimeError(
"SMBIOS serial does not match instance ID from metadata")
return True
def _get_data(self):
(is_upcloud, server_uuid) = self._get_sysinfo()
# only proceed if we know we are on UpCloud
if not is_upcloud:
return False
LOG.info("Running on UpCloud. server_uuid=%s", server_uuid)
if self.perform_dhcp_setup: # Setup networking in init-local stage.
try:
LOG.debug("Finding a fallback NIC")
nic = cloudnet.find_fallback_nic()
LOG.debug("Discovering metadata via DHCP interface %s", nic)
with EphemeralDHCPv4(nic):
md = util.log_time(
logfunc=LOG.debug,
msg="Reading from metadata service",
func=self._read_metadata,
)
except (NoDHCPLeaseError, sources.InvalidMetaDataException) as e:
util.logexc(LOG, str(e))
return False
else:
try:
LOG.debug(
"Discovering metadata without DHCP-configured networking")
md = util.log_time(
logfunc=LOG.debug,
msg="Reading from metadata service",
func=self._read_metadata,
)
except sources.InvalidMetaDataException as e:
util.logexc(LOG, str(e))
LOG.info(
"No DHCP-enabled interfaces available, "
"unable to fetch metadata for %s",
server_uuid,
)
return False
self.metadata_full = md
self.metadata["instance-id"] = md.get("instance_id", server_uuid)
self.metadata["local-hostname"] = md.get("hostname")
self.metadata["network"] = md.get("network")
self.metadata["public-keys"] = md.get("public_keys")
self.metadata["availability_zone"] = md.get("region", "default")
self.vendordata_raw = md.get("vendor_data", None)
self.userdata_raw = md.get("user_data", None)
return True
def get_metadata(url, timeout, retries, sec_between):
# Bring up interface
try:
with EphemeralDHCPv4(connectivity_url=url):
# Fetch the metadata
v1 = read_metadata(url, timeout, retries, sec_between)
except (NoDHCPLeaseError) as exc:
LOG.error("Bailing, DHCP Exception: %s", exc)
raise
v1_json = json.loads(v1)
metadata = v1_json
return metadata
def _get_data(self):
if not on_scaleway():
return False
if self._fallback_interface is None:
self._fallback_interface = net.find_fallback_nic()
try:
with EphemeralDHCPv4(self._fallback_interface):
util.log_time(logfunc=LOG.debug,
msg='Crawl of metadata service',
func=self._crawl_metadata)
except (NoDHCPLeaseError) as e:
util.logexc(LOG, str(e))
return False
return True
def _get_data(self):
"""Crawl metadata, parse and persist that data for this instance.
@return: True when metadata discovered indicates OpenStack datasource.
False when unable to contact metadata service or when metadata
format is invalid or disabled.
"""
if self.perform_dhcp_setup: # Setup networking in init-local stage.
try:
with EphemeralDHCPv4(self.fallback_interface):
results = util.log_time(logfunc=LOG.debug,
msg='Crawl of metadata service',
func=self._crawl_metadata)
except (NoDHCPLeaseError, sources.InvalidMetaDataException) as e:
util.logexc(LOG, str(e))
return False
else:
try:
results = self._crawl_metadata()
except sources.InvalidMetaDataException as e:
util.logexc(LOG, str(e))
return False
self.dsmode = self._determine_dsmode([results.get('dsmode')])
if self.dsmode == sources.DSMODE_DISABLED:
return False
md = results.get('metadata', {})
md = util.mergemanydict([md, DEFAULT_METADATA])
self.metadata = md
self.ec2_metadata = results.get('ec2-metadata')
self.network_json = results.get('networkdata')
self.userdata_raw = results.get('userdata')
self.version = results['version']
self.files.update(results.get('files', {}))
vd = results.get('vendordata')
self.vendordata_pure = vd
try:
self.vendordata_raw = sources.convert_vendordata(vd)
except ValueError as e:
LOG.warning("Invalid content in vendor-data: %s", e)
self.vendordata_raw = None
return True
def _get_data(self):
with EphemeralDHCPv4(get_first_physical_interface()):
response = url_helper.readurl(self.metadata_address, timeout=self.timeout,
sec_between=self.wait_retry,
retries=self.retries)
if not response.ok():
raise RuntimeError("unable to read metadata at %s" % self.metadata_address)
md = json.loads(response.contents.decode())
LOG.debug("Vultr metadata: %s", pprint.pformat(md))
self.metadata_full = md
self.metadata['instance-id'] = md['instanceid']
self.metadata['local-hostname'] = md['hostname']
self.metadata['interfaces'] = md.get('interfaces')
self.metadata['public-keys'] = md.get('public-keys')
self.metadata['availability_zone'] = md.get('region', {}).get('EWR', 'unknown')
return True
def _poll_imds(self, report_ready=True):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "?api-version=2017-04-02"
headers = {"Metadata": "true"}
LOG.debug("Start polling IMDS")
def sleep_cb(response, loop_n):
return 1
def exception_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return
LOG.warning("Exception during polling. Will try DHCP.",
exc_info=True)
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
raise exception
need_report = report_ready
for i in range(IMDS_RETRIES):
try:
with EphemeralDHCPv4() as lease:
if need_report:
self._report_ready(lease=lease)
need_report = False
wait_for_url([url],
max_wait=None,
timeout=60,
status_cb=LOG.info,
headers_cb=lambda url: headers,
sleep_time=1,
exception_cb=exception_cb,
sleep_time_cb=sleep_cb)
return str(readurl(url, headers=headers))
except Exception:
LOG.debug("Exception during polling-retrying dhcp" +
" %d more time(s).", (IMDS_RETRIES - i),
exc_info=True)
def get_metadata(url, timeout, retries, sec_between, agent):
# Bring up interface (and try untill one works)
exception = RuntimeError("Failed to DHCP")
# Seek iface with DHCP
for iface in net.get_interfaces():
# Skip dummy, lo interfaces
if "dummy" in iface[0]:
continue
if "lo" == iface[0]:
continue
try:
with EphemeralDHCPv4(
iface=iface[0], connectivity_url_data={"url": url}
):
# Fetch the metadata
v1 = read_metadata(url, timeout, retries, sec_between, agent)
return json.loads(v1)
except (NoDHCPLeaseError, subp.ProcessExecutionError) as exc:
LOG.error("DHCP Exception: %s", exc)
exception = exc
raise exception
def __init__(self, reporter, nic=None):
self.reporter = reporter
self.ephemeralDHCPv4 = EphemeralDHCPv4(
iface=nic, dhcp_log_func=dhcp_log_cb)
def __init__(self, reporter, nic=None):
self.reporter = reporter
self.ephemeralDHCPv4 = EphemeralDHCPv4(iface=nic)
def crawl_metadata(self):
"""Walk all instance metadata sources returning a dict on success.
@return: A dictionary of any metadata content for this instance.
@raise: InvalidMetaDataException when the expected metadata service is
unavailable, broken or disabled.
"""
crawled_data = {}
# azure removes/ejects the cdrom containing the ovf-env.xml
# file on reboot. So, in order to successfully reboot we
# need to look in the datadir and consider that valid
ddir = self.ds_cfg['data_dir']
candidates = [self.seed_dir]
if os.path.isfile(REPROVISION_MARKER_FILE):
candidates.insert(0, "IMDS")
candidates.extend(list_possible_azure_ds_devs())
if ddir:
candidates.append(ddir)
found = None
reprovision = False
for cdev in candidates:
try:
if cdev == "IMDS":
ret = None
reprovision = True
elif cdev.startswith("/dev/"):
if util.is_FreeBSD():
ret = util.mount_cb(cdev,
load_azure_ds_dir,
mtype="udf",
sync=False)
else:
ret = util.mount_cb(cdev, load_azure_ds_dir)
else:
ret = load_azure_ds_dir(cdev)
except NonAzureDataSource:
continue
except BrokenAzureDataSource as exc:
msg = 'BrokenAzureDataSource: %s' % exc
raise sources.InvalidMetaDataException(msg)
except util.MountFailedError:
LOG.warning("%s was not mountable", cdev)
continue
perform_reprovision = reprovision or self._should_reprovision(ret)
if perform_reprovision:
if util.is_FreeBSD():
msg = "Free BSD is not supported for PPS VMs"
LOG.error(msg)
raise sources.InvalidMetaDataException(msg)
ret = self._reprovision()
imds_md = get_metadata_from_imds(self.fallback_interface,
retries=10)
(md, userdata_raw, cfg, files) = ret
self.seed = cdev
crawled_data.update({
'cfg':
cfg,
'files':
files,
'metadata':
util.mergemanydict([md, {
'imds': imds_md
}]),
'userdata_raw':
userdata_raw
})
found = cdev
LOG.debug("found datasource in %s", cdev)
break
if not found:
raise sources.InvalidMetaDataException('No Azure metadata found')
if found == ddir:
LOG.debug("using files cached in %s", ddir)
seed = _get_random_seed()
if seed:
crawled_data['metadata']['random_seed'] = seed
crawled_data['metadata']['instance-id'] = util.read_dmi_data(
'system-uuid')
if perform_reprovision:
LOG.info("Reporting ready to Azure after getting ReprovisionData")
use_cached_ephemeral = (net.is_up(self.fallback_interface) and
getattr(self, '_ephemeral_dhcp_ctx', None))
if use_cached_ephemeral:
self._report_ready(lease=self._ephemeral_dhcp_ctx.lease)
self._ephemeral_dhcp_ctx.clean_network() # Teardown ephemeral
else:
with EphemeralDHCPv4() as lease:
self._report_ready(lease=lease)
return crawled_data
