def has_url_connectivity(url_data: Dict[str, Any]) -> bool:
"""Return true when the instance has access to the provided URL.
Logs a warning if url is not the expected format.
url_data is a dictionary of kwargs to send to readurl. E.g.:
has_url_connectivity({
"url": "http://example.invalid",
"headers": {"some": "header"},
"timeout": 10
})
"""
if "url" not in url_data:
LOG.warning(
"Ignoring connectivity check. No 'url' to check in %s", url_data
)
return False
url = url_data["url"]
if not any([url.startswith("http://"), url.startswith("https://")]):
LOG.warning(
"Ignoring connectivity check. Expected URL beginning with http*://"
" received '%s'",
url,
)
return False
if "timeout" not in url_data:
url_data["timeout"] = 5
try:
readurl(**url_data)
except UrlError:
return False
return True
def _get_metadata(self):
resp = url_helper.readurl(
self.metadata_address,
data=None,
timeout=self.timeout,
retries=self.retries
)
metadata = json.loads(util.decode_binary(resp.contents))
LOG.debug('metadata downloaded')
# try to make a request on the first privileged port available
for port in range(1, 1024):
try:
session = requests.Session()
session.mount(
'http://',
SourceAddressAdapter(source_address=('0.0.0.0', port))
)
resp = url_helper.readurl(
self.userdata_address,
data=None,
timeout=self.timeout,
retries=self.retries,
session=session
)
user_data = util.decode_binary(resp.contents)
LOG.debug('user-data downloaded')
return metadata, user_data
except url_helper.UrlError: # try next port
pass
def read_metadata(endpoint_base=METADATA_ENDPOINT,
sys_uuid=None,
version='2013-10-17'):
"""Read metadata, return a dictionary.
Each path listed in the index will be represented in the dictionary.
If the path ends in .json, then the content will be decoded and
populated into the dictionary.
The system uuid (/sys/class/dmi/id/product_uuid) is also populated.
Example: given paths = ('user_data', 'meta_data.json')
This would return:
{version: {'user_data': b'blob', 'meta_data': json.loads(blob.decode())
'system_uuid': '3b54f2e0-3ab2-458d-b770-af9926eee3b2'}}
"""
endpoint = combine_url(endpoint_base, version) + "/"
if sys_uuid is None:
sys_uuid = _read_system_uuid()
if not sys_uuid:
raise sources.BrokenMetadata("Failed to read system uuid.")
try:
resp = readurl(endpoint)
if not resp.ok():
raise sources.BrokenMetadata("Bad response from %s: %s" %
(endpoint, resp.code))
except UrlError as e:
raise sources.BrokenMetadata("Failed to read index at %s: %s" %
(endpoint, e))
entries = _load_index(resp.contents.decode('utf-8'))
LOG.debug("index url %s contained: %s", endpoint, entries)
# meta_data.json is required.
mdj = 'meta_data.json'
if mdj not in entries:
raise sources.BrokenMetadata(
"Required field '%s' missing in index at %s" % (mdj, endpoint))
ret = {'system_uuid': sys_uuid}
for path in entries:
response = readurl(combine_url(endpoint, path))
if path.endswith(".json"):
ret[path.rpartition(".")[0]] = (json.loads(
response.contents.decode('utf-8')))
else:
ret[path] = response.contents
return {version: ret}
def has_url_connectivity(url):
"""Return true when the instance has access to the provided URL
Logs a warning if url is not the expected format.
"""
if not any([url.startswith('http://'), url.startswith('https://')]):
LOG.warning(
"Ignoring connectivity check. Expected URL beginning with http*://"
" received '%s'", url)
return False
try:
readurl(url, timeout=5)
except UrlError:
return False
return True
def has_url_connectivity(url):
"""Return true when the instance has access to the provided URL
Logs a warning if url is not the expected format.
"""
if not any([url.startswith('http://'), url.startswith('https://')]):
LOG.warning(
"Ignoring connectivity check. Expected URL beginning with http*://"
" received '%s'", url)
return False
try:
readurl(url, timeout=5)
except UrlError:
return False
return True
def _poll_imds(self):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
headers = {"Metadata": "true"}
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
LOG.debug("Start polling IMDS")
while True:
try:
# Save our EphemeralDHCPv4 context so we avoid repeated dhcp
self._ephemeral_dhcp_ctx = EphemeralDHCPv4()
lease = self._ephemeral_dhcp_ctx.obtain_lease()
if report_ready:
path = REPORTED_READY_MARKER_FILE
LOG.info("Creating a marker file to report ready: %s",
path)
util.write_file(
path, "{pid}: {time}\n".format(pid=os.getpid(),
time=time()))
self._report_ready(lease=lease)
report_ready = False
return readurl(url,
timeout=1,
headers=headers,
exception_cb=retry_on_url_exc,
infinite=True,
log_req_resp=False).contents
except UrlError:
# Teardown our EphemeralDHCPv4 context on failure as we retry
self._ephemeral_dhcp_ctx.clean_network()
pass
def install_chef(cloud, chef_cfg, log):
# If chef is not installed, we install chef based on 'install_type'
install_type = util.get_cfg_option_str(chef_cfg, 'install_type',
'packages')
run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False)
if install_type == "gems":
# This will install and run the chef-client from gems
chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version',
RUBY_VERSION_DEFAULT)
install_chef_from_gems(cloud.distro, ruby_version, chef_version)
# Retain backwards compat, by preferring True instead of False
# when not provided/overriden...
run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True)
elif install_type == 'packages':
# This will install and run the chef-client from packages
cloud.distro.install_packages(('chef',))
elif install_type == 'omnibus':
# This will install as a omnibus unified package
url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL)
retries = max(0, util.get_cfg_option_int(chef_cfg,
"omnibus_url_retries",
default=OMNIBUS_URL_RETRIES))
content = url_helper.readurl(url=url, retries=retries)
with util.tempdir() as tmpd:
# Use tmpdir over tmpfile to avoid 'text file busy' on execute
tmpf = "%s/chef-omnibus-install" % tmpd
util.write_file(tmpf, str(content), mode=0700)
util.subp([tmpf], capture=False)
else:
log.warn("Unknown chef install type '%s'", install_type)
run = False
return run
def install_puppet_aio(url=AIO_INSTALL_URL,
version=None,
collection=None,
cleanup=True):
"""Install puppet-agent from the puppetlabs repositories using the one-shot
shell script
:param url: URL from where to download the install script
:param version: version to install, blank defaults to latest
:param collection: collection to install, blank defaults to latest
:param cleanup: whether to purge the puppetlabs repo after installation
"""
args = []
if version is not None:
args = ['-v', version]
if collection is not None:
args += ['-c', collection]
# Purge puppetlabs repos after installation
if cleanup:
args += ['--cleanup']
content = url_helper.readurl(url=url, retries=5).contents
# Use tmpdir over tmpfile to avoid 'text file busy' on execute
with temp_utils.tempdir(needs_exe=True) as tmpd:
tmpf = os.path.join(tmpd, 'puppet-install')
util.write_file(tmpf, content, mode=0o700)
return subp.subp([tmpf] + args, capture=False)
def get_metadata_api_version(self):
"""Get the best supported api version from the metadata service.
Loop through all extended support metadata versions in order and
return the most-fully featured metadata api version discovered.
If extended_metadata_versions aren't present, return the datasource's
min_metadata_version.
"""
# Assumes metadata service is already up
url_tmpl = "{0}/{1}/meta-data/instance-id"
headers = self._get_headers()
for api_ver in self.extended_metadata_versions:
url = url_tmpl.format(self.metadata_address, api_ver)
try:
resp = uhelp.readurl(url=url,
headers=headers,
headers_redact=AWS_TOKEN_REDACT)
except uhelp.UrlError as e:
LOG.debug("url %s raised exception %s", url, e)
else:
if resp.code == 200:
LOG.debug("Found preferred metadata version %s", api_ver)
return api_ver
elif resp.code == 404:
msg = "Metadata api version %s not present. Headers: %s"
LOG.debug(msg, api_ver, resp.headers)
return self.min_metadata_version
def post(self, url, data=None, extra_headers=None):
headers = self.headers
if extra_headers is not None:
headers = self.headers.copy()
headers.update(extra_headers)
return url_helper.readurl(url, data=data, headers=headers,
timeout=5, retries=10, sec_between=5)
def get_metadata_api_version(self):
"""Get the best supported api version from the metadata service.
Loop through all extended support metadata versions in order and
return the most-fully featured metadata api version discovered.
If extended_metadata_versions aren't present, return the datasource's
min_metadata_version.
"""
# Assumes metadata service is already up
for api_ver in self.extended_metadata_versions:
url = '{0}/{1}/meta-data/instance-id'.format(
self.metadata_address, api_ver)
try:
resp = uhelp.readurl(url=url)
except uhelp.UrlError as e:
LOG.debug('url %s raised exception %s', url, e)
else:
if resp.code == 200:
LOG.debug('Found preferred metadata version %s', api_ver)
return api_ver
elif resp.code == 404:
msg = 'Metadata api version %s not present. Headers: %s'
LOG.debug(msg, api_ver, resp.headers)
return self.min_metadata_version
def install_chef(cloud, chef_cfg, log):
# If chef is not installed, we install chef based on 'install_type'
install_type = util.get_cfg_option_str(chef_cfg, 'install_type',
'packages')
run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False)
if install_type == "gems":
# This will install and run the chef-client from gems
chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version',
RUBY_VERSION_DEFAULT)
install_chef_from_gems(ruby_version, chef_version, cloud.distro)
# Retain backwards compat, by preferring True instead of False
# when not provided/overriden...
run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True)
elif install_type == 'packages':
# This will install and run the chef-client from packages
cloud.distro.install_packages(('chef', ))
elif install_type == 'omnibus':
# This will install as a omnibus unified package
url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL)
retries = max(
0,
util.get_cfg_option_int(chef_cfg,
"omnibus_url_retries",
default=OMNIBUS_URL_RETRIES))
content = url_helper.readurl(url=url, retries=retries)
with util.tempdir() as tmpd:
# Use tmpdir over tmpfile to avoid 'text file busy' on execute
tmpf = "%s/chef-omnibus-install" % tmpd
util.write_file(tmpf, content, mode=0o700)
util.subp([tmpf], capture=False)
else:
log.warn("Unknown chef install type '%s'", install_type)
run = False
return run
def read_userdata(url, timeout=2, sec_between=2, retries=30):
response = url_helper.readurl(
url, timeout=timeout, sec_between=sec_between, retries=retries
)
if not response.ok():
raise RuntimeError("unable to read userdata at %s" % url)
return response.contents
def read_metadata(url, timeout=2, sec_between=2, retries=30):
response = url_helper.readurl(
url, timeout=timeout, sec_between=sec_between, retries=retries
)
if not response.ok():
raise RuntimeError("unable to read metadata at %s" % url)
return util.load_yaml(response.contents.decode())
def query_data_api_once(api_address, timeout, requests_session):
"""
Retrieve user data or vendor data.
Scaleway user/vendor data API returns HTTP/404 if user/vendor data is not
set.
This function calls `url_helper.readurl` but instead of considering
HTTP/404 as an error that requires a retry, it considers it as empty
user/vendor data.
Also, be aware the user data/vendor API requires the source port to be
below 1024 to ensure the client is root (since non-root users can't bind
ports below 1024). If requests raises ConnectionError (EADDRINUSE), the
caller should retry to call this function on an other port.
"""
try:
resp = url_helper.readurl(
api_address,
data=None,
timeout=timeout,
# It's the caller's responsability to recall this function in case
# of exception. Don't let url_helper.readurl() retry by itself.
retries=0,
session=requests_session,
# If the error is a HTTP/404 or a ConnectionError, go into raise
# block below and don't bother retrying.
exception_cb=lambda _, exc: exc.code != 404 and
(not isinstance(exc.cause, requests.exceptions.ConnectionError)))
return util.decode_binary(resp.contents)
except url_helper.UrlError as exc:
# Empty user data.
if exc.code == 404:
return None
raise
def _poll_imds(self, report_ready=True):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "?api-version=2017-04-02"
headers = {"Metadata": "true"}
LOG.debug("Start polling IMDS")
def exc_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return True
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
return False
need_report = report_ready
while True:
try:
with EphemeralDHCPv4() as lease:
if need_report:
self._report_ready(lease=lease)
need_report = False
return readurl(url,
timeout=1,
headers=headers,
exception_cb=exc_cb,
infinite=True).contents
except UrlError:
pass
def _poll_imds(self):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
headers = {"Metadata": "true"}
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
LOG.debug("Start polling IMDS")
def exc_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return True
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
return False
while True:
try:
with EphemeralDHCPv4() as lease:
if report_ready:
path = REPORTED_READY_MARKER_FILE
LOG.info("Creating a marker file to report ready: %s",
path)
util.write_file(
path, "{pid}: {time}\n".format(pid=os.getpid(),
time=time()))
self._report_ready(lease=lease)
report_ready = False
return readurl(url,
timeout=1,
headers=headers,
exception_cb=exc_cb,
infinite=True).contents
except UrlError:
pass
def get(self, url, secure=False):
headers = self.headers
if secure:
headers = self.headers.copy()
headers.update(self.extra_secure_headers)
return url_helper.readurl(url, headers=headers,
timeout=5, retries=10, sec_between=5)
def try_read_url(url, distro_name, reset_net=True):
raw_data = url_helper.readurl(url, timeout=3, retries=3).contents
if type(raw_data) is bytes:
raw_data = raw_data.decode()
return raw_data
def _fetch(metadata_version: int, path: str) -> dict:
headers = {
"Authorization": "Bearer Oracle"
} if metadata_version > 1 else None
return readurl(
url=METADATA_PATTERN.format(version=metadata_version, path=path),
headers=headers,
retries=retries,
)._response.json()
def _write_host_key_to_guest_attributes(key_type, key_value):
url = '%s/%s/%s' % (GUEST_ATTRIBUTES_URL, HOSTKEY_NAMESPACE, key_type)
key_value = key_value.encode('utf-8')
resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS,
request_method='PUT', check_status=False)
if resp.ok():
LOG.debug('Wrote %s host key to guest attributes.', key_type)
else:
LOG.debug('Unable to write %s host key to guest attributes.', key_type)
def read_opc_metadata():
"""
Fetch metadata from the /opc/ routes.
:return:
The JSON-decoded value of the /opc/v1/instance/ endpoint on the IMDS.
"""
# retries=1 as requested by Oracle to address a potential race condition
return json.loads(readurl(METADATA_ENDPOINT, retries=1)._response.text)
def _poll_imds(self):
"""Poll IMDS for the new provisioning data until we get a valid
response. Then return the returned JSON object."""
url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
headers = {"Metadata": "true"}
nl_sock = None
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
def exc_cb(msg, exception):
if isinstance(exception, UrlError) and exception.code == 404:
return True
# If we get an exception while trying to call IMDS, we
# call DHCP and setup the ephemeral network to acquire the new IP.
return False
LOG.debug("Wait for vnetswitch to happen")
while True:
try:
# Save our EphemeralDHCPv4 context so we avoid repeated dhcp
self._ephemeral_dhcp_ctx = EphemeralDHCPv4()
lease = self._ephemeral_dhcp_ctx.obtain_lease()
if report_ready:
try:
nl_sock = netlink.create_bound_netlink_socket()
except netlink.NetlinkCreateSocketError as e:
LOG.warning(e)
self._ephemeral_dhcp_ctx.clean_network()
return
path = REPORTED_READY_MARKER_FILE
LOG.info("Creating a marker file to report ready: %s",
path)
util.write_file(
path, "{pid}: {time}\n".format(pid=os.getpid(),
time=time()))
self._report_ready(lease=lease)
report_ready = False
try:
netlink.wait_for_media_disconnect_connect(
nl_sock, lease['interface'])
except AssertionError as error:
LOG.error(error)
return
self._ephemeral_dhcp_ctx.clean_network()
else:
return readurl(url,
timeout=1,
headers=headers,
exception_cb=exc_cb,
infinite=True,
log_req_resp=False).contents
except UrlError:
# Teardown our EphemeralDHCPv4 context on failure as we retry
self._ephemeral_dhcp_ctx.clean_network()
pass
finally:
if nl_sock:
nl_sock.close()
def _get_data(self, apply_filter=False):
url = get_url_from_file()
if url is None:
return False
response = url_helper.readurl(url)
decoded = json.loads(response.contents.decode())
self.metadata = decoded["metadata"]
self.vendordata_raw = decoded["vendordata_raw"]
self.userdata_raw = decoded["userdata_raw"]
return True
def get_data(self, apply_filter=False):
url = get_url_from_file()
if url is None:
return False
response = url_helper.readurl(url)
decoded = json.loads(response.contents)
self.metadata = decoded["metadata"]
self.vendordata_raw = decoded["vendordata_raw"]
self.userdata_raw = decoded["userdata_raw"]
return True
def handle(name, _cfg, cloud, log, _args):
try:
ud = cloud.get_userdata_raw()
except Exception:
log.debug("Failed to get raw userdata in module %s", name)
return
try:
mdict = parse_qs(ud)
if not mdict or MY_HOOKNAME not in mdict:
log.debug(
"Skipping module %s, did not find %s in parsed raw userdata",
name,
MY_HOOKNAME,
)
return
except Exception:
util.logexc(log, "Failed to parse query string %s into a dictionary",
ud)
raise
wrote_fns = []
captured_excps = []
# These will eventually be then ran by the cc_scripts_user
# TODO(harlowja): maybe this should just be a new user data handler??
# Instead of a late module that acts like a user data handler?
scripts_d = cloud.get_ipath_cur("scripts")
urls = mdict[MY_HOOKNAME]
for (i, url) in enumerate(urls):
fname = os.path.join(scripts_d, "rightscale-%02i" % (i))
try:
resp = uhelp.readurl(url)
# Ensure its a valid http response (and something gotten)
if resp.ok() and resp.contents:
util.write_file(fname, resp, mode=0o700)
wrote_fns.append(fname)
except Exception as e:
captured_excps.append(e)
util.logexc(log, "%s failed to read %s and write %s", MY_NAME, url,
fname)
if wrote_fns:
log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns))
if len(wrote_fns) != len(urls):
skipped = len(urls) - len(wrote_fns)
log.debug("%s urls were skipped or failed", skipped)
if captured_excps:
log.warning(
"%s failed with exceptions, re-raising the last one",
len(captured_excps),
)
raise captured_excps[-1]
def get_data(self):
# IBGCE metadata server requires a custom header since v1
headers = {'X-Google-Metadata-Request': str(True)}
# url_map: (our-key, path, required)
url_map = [
('instance-id', 'instance/id', True),
('availability-zone', 'instance/zone', True),
('local-hostname', 'instance/hostname', True),
('public-keys', 'project/attributes/sshKeys', False),
('user-data', 'instance/attributes/user-data', False),
]
# if we cannot resolve the metadata server, then no point in trying
if not util.is_resolvable_url(self.metadata_address):
LOG.debug("%s is not resolvable", self.metadata_address)
return False
# iterate over url_map keys to get metadata items
found = False
for (mkey, path, required) in url_map:
try:
resp = url_helper.readurl(url=self.metadata_address + path,
headers=headers)
if resp.code == 200:
found = True
self.metadata[mkey] = resp.contents
else:
if required:
msg = "required url %s returned code %s. not IBGCE"
if not found:
LOG.debug(msg, path, resp.code)
else:
LOG.warn(msg, path, resp.code)
return False
else:
self.metadata[mkey] = None
except url_helper.UrlError as e:
if required:
msg = "required url %s raised exception %s. not IBGCE"
if not found:
LOG.debug(msg, path, e)
else:
LOG.warn(msg, path, e)
return False
msg = "Failed to get %s metadata item: %s."
LOG.debug(msg, path, e)
self.metadata[mkey] = None
if self.metadata['public-keys']:
lines = self.metadata['public-keys'].splitlines()
self.metadata['public-keys'] = [self._trim_key(k) for k in lines]
return found
def get_data(self):
# GCE metadata server requires a custom header since v1
headers = {"X-Google-Metadata-Request": True}
# url_map: (our-key, path, required)
url_map = [
("instance-id", "instance/id", True),
("availability-zone", "instance/zone", True),
("local-hostname", "instance/hostname", True),
("public-keys", "project/attributes/sshKeys", False),
("user-data", "instance/attributes/user-data", False),
]
# if we cannot resolve the metadata server, then no point in trying
if not util.is_resolvable_url(self.metadata_address):
LOG.debug("%s is not resolvable", self.metadata_address)
return False
# iterate over url_map keys to get metadata items
found = False
for (mkey, path, required) in url_map:
try:
resp = url_helper.readurl(url=self.metadata_address + path, headers=headers)
if resp.code == 200:
found = True
self.metadata[mkey] = resp.contents
else:
if required:
msg = "required url %s returned code %s. not GCE"
if not found:
LOG.debug(msg, path, resp.code)
else:
LOG.warn(msg, path, resp.code)
return False
else:
self.metadata[mkey] = None
except url_helper.UrlError as e:
if required:
msg = "required url %s raised exception %s. not GCE"
if not found:
LOG.debug(msg, path, e)
else:
LOG.warn(msg, path, e)
return False
msg = "Failed to get %s metadata item: %s."
LOG.debug(msg, path, e)
self.metadata[mkey] = None
if self.metadata["public-keys"]:
lines = self.metadata["public-keys"].splitlines()
self.metadata["public-keys"] = [self._trim_key(k) for k in lines]
return found
def _crawl_metadata(self):
resp = url_helper.readurl(self.metadata_address,
timeout=self.timeout,
retries=self.retries)
self.metadata = json.loads(util.decode_binary(resp.contents))
self.userdata_raw = query_data_api('user-data', self.userdata_address,
self.retries, self.timeout)
self.vendordata_raw = query_data_api('vendor-data',
self.vendordata_address,
self.retries, self.timeout)
def read_metadata(url, timeout, retries, sec_between):
url = "%s/v1.json" % url
response = url_helper.readurl(url,
timeout=timeout,
retries=retries,
headers={'Metadata-Token': 'vultr'},
sec_between=sec_between)
if not response.ok():
raise RuntimeError("Failed to connect to %s: Code: %s" % url,
response.code)
return response.contents.decode()
def handle(name, _cfg, cloud, log, _args):
try:
ud = cloud.get_userdata_raw()
except Exception:
log.debug("Failed to get raw userdata in module %s", name)
return
try:
mdict = parse_qs(ud)
if not mdict or MY_HOOKNAME not in mdict:
log.debug(("Skipping module %s, "
"did not find %s in parsed"
" raw userdata"), name, MY_HOOKNAME)
return
except Exception:
util.logexc(log, "Failed to parse query string %s into a dictionary",
ud)
raise
wrote_fns = []
captured_excps = []
# These will eventually be then ran by the cc_scripts_user
# TODO(harlowja): maybe this should just be a new user data handler??
# Instead of a late module that acts like a user data handler?
scripts_d = cloud.get_ipath_cur('scripts')
urls = mdict[MY_HOOKNAME]
for (i, url) in enumerate(urls):
fname = os.path.join(scripts_d, "rightscale-%02i" % (i))
try:
resp = uhelp.readurl(url)
# Ensure its a valid http response (and something gotten)
if resp.ok() and resp.contents:
util.write_file(fname, resp, mode=0o700)
wrote_fns.append(fname)
except Exception as e:
captured_excps.append(e)
util.logexc(log, "%s failed to read %s and write %s", MY_NAME, url,
fname)
if wrote_fns:
log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns))
if len(wrote_fns) != len(urls):
skipped = len(urls) - len(wrote_fns)
log.debug("%s urls were skipped or failed", skipped)
if captured_excps:
log.warn("%s failed with exceptions, re-raising the last one",
len(captured_excps))
raise captured_excps[-1]
def _crawl_metadata(self):
resp = url_helper.readurl(self.metadata_address,
timeout=self.timeout,
retries=self.retries)
self.metadata = json.loads(util.decode_binary(resp.contents))
self.userdata_raw = query_data_api(
'user-data', self.userdata_address,
self.retries, self.timeout
)
self.vendordata_raw = query_data_api(
'vendor-data', self.vendordata_address,
self.retries, self.timeout
)
def has_url_connectivity(url_data: Dict[str, Any]) -> bool:
"""Return true when the instance has access to the provided URL.
Logs a warning if url is not the expected format.
url_data is a dictionary of kwargs to send to readurl. E.g.:
has_url_connectivity({
"url": "http://example.invalid",
"headers": {"some": "header"},
"timeout": 10
})
"""
if "url" not in url_data:
LOG.warning(
"Ignoring connectivity check. No 'url' to check in %s", url_data
)
return False
url = url_data["url"]
try:
result = urlparse(url)
if not any([result.scheme == "http", result.scheme == "https"]):
LOG.warning(
"Ignoring connectivity check. Invalid URL scheme %s",
url.scheme,
)
return False
except ValueError as err:
LOG.warning("Ignoring connectivity check. Invalid URL %s", err)
return False
if "timeout" not in url_data:
url_data["timeout"] = 5
try:
readurl(**url_data)
except UrlError:
return False
return True
def _get_data(self):
if not on_scaleway():
return False
resp = url_helper.readurl(self.metadata_address,
timeout=self.timeout,
retries=self.retries)
self.metadata = json.loads(util.decode_binary(resp.contents))
self.userdata_raw = query_data_api('user-data', self.userdata_address,
self.retries, self.timeout)
self.vendordata_raw = query_data_api('vendor-data',
self.vendordata_address,
self.retries, self.timeout)
return True
def _crawl_metadata(self):
# Stay backward compatible with classes w/o these attributes
self.headers = getattr(self, 'headers', None)
self.headers_redact = getattr(self, 'headers_redact', None)
resp = url_helper.readurl(
self.metadata_address, headers=self.headers, headers_redact=self.headers_redact, timeout=self.timeout, retries=self.retries
)
self.metadata = json.loads(util.decode_binary(resp.contents))
self.userdata_raw = query_data_api(
"user-data", self.userdata_address, self.headers, self.headers_redact, self.retries, self.timeout
)
self.vendordata_raw = query_data_api(
"vendor-data", self.vendordata_address, self.headers, self.headers_redact, self.retries, self.timeout
)
def get_value(self, path, is_text):
value = None
try:
resp = url_helper.readurl(url=self.metadata_address + path,
headers=self.headers)
except url_helper.UrlError as exc:
msg = "url %s raised exception %s"
LOG.debug(msg, path, exc)
else:
if resp.code == 200:
if is_text:
value = util.decode_binary(resp.contents)
else:
value = resp.contents
else:
LOG.debug("url %s returned code %s", path, resp.code)
return value
def http_with_retries(url, **kwargs) -> url_helper.UrlResponse:
"""Wrapper around url_helper.readurl() with custom telemetry logging
that url_helper.readurl() does not provide.
"""
max_readurl_attempts = 240
default_readurl_timeout = 5
sleep_duration_between_retries = 5
periodic_logging_attempts = 12
if "timeout" not in kwargs:
kwargs["timeout"] = default_readurl_timeout
# remove kwargs that cause url_helper.readurl to retry,
# since we are already implementing our own retry logic.
if kwargs.pop("retries", None):
LOG.warning("Ignoring retries kwarg passed in for "
"communication with Azure endpoint.")
if kwargs.pop("infinite", None):
LOG.warning("Ignoring infinite kwarg passed in for communication "
"with Azure endpoint.")
for attempt in range(1, max_readurl_attempts + 1):
try:
ret = url_helper.readurl(url, **kwargs)
report_diagnostic_event(
"Successful HTTP request with Azure endpoint %s after "
"%d attempts" % (url, attempt),
logger_func=LOG.debug,
)
return ret
except Exception as e:
if attempt % periodic_logging_attempts == 0:
report_diagnostic_event(
"Failed HTTP request with Azure endpoint %s during "
"attempt %d with exception: %s" % (url, attempt, e),
logger_func=LOG.debug,
)
if attempt == max_readurl_attempts:
raise
time.sleep(sleep_duration_between_retries)
raise RuntimeError("Failed to return in http_with_retries")
def get_value(self, path, is_text):
value = None
try:
resp = url_helper.readurl(url=self.metadata_address + path,
headers=self.headers)
except url_helper.UrlError as exc:
msg = "url %s raised exception %s"
LOG.debug(msg, path, exc)
else:
if resp.code == 200:
if is_text:
value = util.decode_binary(resp.contents)
else:
value = resp.contents
else:
LOG.debug("url %s returned code %s", path, resp.code)
return value
def _get_metadata_from_imds(retries):
url = IMDS_URL + "instance?api-version=2017-12-01"
headers = {"Metadata": "true"}
try:
response = readurl(
url, timeout=1, headers=headers, retries=retries,
exception_cb=retry_on_url_exc)
except Exception as e:
LOG.debug('Ignoring IMDS instance metadata: %s', e)
return {}
try:
return util.load_json(str(response))
except json.decoder.JSONDecodeError:
LOG.warning(
'Ignoring non-json IMDS instance metadata: %s', str(response))
return {}
def _path_read(self, path):
def should_retry_cb(_request_args, cause):
try:
code = int(cause.code)
if code >= 400:
return False
except (TypeError, ValueError):
# Older versions of requests didn't have a code.
pass
return True
response = url_helper.readurl(path,
retries=self.retries,
ssl_details=self.ssl_details,
timeout=self.timeout,
exception_cb=should_retry_cb)
return response.contents
def get_value(self, path, is_text, is_recursive=False):
value = None
try:
url = self.metadata_address + path
if is_recursive:
url += '/?recursive=True'
resp = url_helper.readurl(url=url, headers=self.headers)
except url_helper.UrlError as exc:
msg = "url %s raised exception %s"
LOG.debug(msg, path, exc)
else:
if resp.code == 200:
if is_text:
value = util.decode_binary(resp.contents)
else:
value = resp.contents.decode('utf-8')
else:
LOG.debug("url %s returned code %s", path, resp.code)
return value
def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None):
"""Install an omnibus unified package from url.
@param url: URL where blob of chef content may be downloaded. Defaults to
OMNIBUS_URL.
@param retries: Number of retries to perform when attempting to read url.
Defaults to OMNIBUS_URL_RETRIES
@param omnibus_version: Optional version string to require for omnibus
install.
"""
if url is None:
url = OMNIBUS_URL
if retries is None:
retries = OMNIBUS_URL_RETRIES
if omnibus_version is None:
args = []
else:
args = ['-v', omnibus_version]
content = url_helper.readurl(url=url, retries=retries).contents
return util.subp_blob_in_tempfile(
blob=content, args=args,
basename='chef-omnibus-install', capture=False)
def query_data_api_once(api_address, timeout, requests_session):
"""
Retrieve user data or vendor data.
Scaleway user/vendor data API returns HTTP/404 if user/vendor data is not
set.
This function calls `url_helper.readurl` but instead of considering
HTTP/404 as an error that requires a retry, it considers it as empty
user/vendor data.
Also, be aware the user data/vendor API requires the source port to be
below 1024 to ensure the client is root (since non-root users can't bind
ports below 1024). If requests raises ConnectionError (EADDRINUSE), the
caller should retry to call this function on an other port.
"""
try:
resp = url_helper.readurl(
api_address,
data=None,
timeout=timeout,
# It's the caller's responsability to recall this function in case
# of exception. Don't let url_helper.readurl() retry by itself.
retries=0,
session=requests_session,
# If the error is a HTTP/404 or a ConnectionError, go into raise
# block below and don't bother retrying.
exception_cb=lambda _, exc: exc.code != 404 and (
not isinstance(exc.cause, requests.exceptions.ConnectionError)
)
)
return util.decode_binary(resp.contents)
except url_helper.UrlError as exc:
# Empty user data.
if exc.code == 404:
return None
raise
def handle(name, cfg, cloud, log, _args):
# If there isn't a chef key in the configuration don't do anything
if 'chef' not in cfg:
log.debug(("Skipping module named %s,"
" no 'chef' key in configuration"), name)
return
chef_cfg = cfg['chef']
# Ensure the chef directories we use exist
for d in CHEF_DIRS:
util.ensure_dir(d)
# Set the validation key based on the presence of either 'validation_key'
# or 'validation_cert'. In the case where both exist, 'validation_key'
# takes precedence
for key in ('validation_key', 'validation_cert'):
if key in chef_cfg and chef_cfg[key]:
util.write_file('/etc/chef/validation.pem', chef_cfg[key])
break
# Create the chef config from template
template_fn = cloud.get_template_filename('chef_client.rb')
if template_fn:
iid = str(cloud.datasource.get_instance_id())
params = {
'server_url': chef_cfg['server_url'],
'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid),
'environment': util.get_cfg_option_str(chef_cfg, 'environment',
'_default'),
'validation_name': chef_cfg['validation_name']
}
templater.render_to_file(template_fn, '/etc/chef/client.rb', params)
else:
log.warn("No template found, not rendering to /etc/chef/client.rb")
# set the firstboot json
initial_json = {}
if 'run_list' in chef_cfg:
initial_json['run_list'] = chef_cfg['run_list']
if 'initial_attributes' in chef_cfg:
initial_attributes = chef_cfg['initial_attributes']
for k in list(initial_attributes.keys()):
initial_json[k] = initial_attributes[k]
util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json))
# If chef is not installed, we install chef based on 'install_type'
if (not os.path.isfile('/usr/bin/chef-client') or
util.get_cfg_option_bool(chef_cfg,
'force_install', default=False)):
install_type = util.get_cfg_option_str(chef_cfg, 'install_type',
'packages')
if install_type == "gems":
# this will install and run the chef-client from gems
chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version',
RUBY_VERSION_DEFAULT)
install_chef_from_gems(cloud.distro, ruby_version, chef_version)
# and finally, run chef-client
log.debug('Running chef-client')
util.subp(['/usr/bin/chef-client',
'-d', '-i', '1800', '-s', '20'], capture=False)
elif install_type == 'packages':
# this will install and run the chef-client from packages
cloud.distro.install_packages(('chef',))
elif install_type == 'omnibus':
url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL)
content = url_helper.readurl(url=url, retries=5)
with util.tempdir() as tmpd:
# use tmpd over tmpfile to avoid 'Text file busy' on execute
tmpf = "%s/chef-omnibus-install" % tmpd
util.write_file(tmpf, str(content), mode=0700)
util.subp([tmpf], capture=False)
else:
log.warn("Unknown chef install type %s", install_type)
def read_userdata(url, timeout=2, sec_between=2, retries=30):
response = url_helper.readurl(url, timeout=timeout,
sec_between=sec_between, retries=retries)
if not response.ok():
raise RuntimeError("unable to read userdata at %s" % url)
return response.contents
def _path_read(self, path):
response = url_helper.readurl(path,
retries=self.retries,
ssl_details=self.ssl_details,
timeout=self.timeout)
return response.contents
def read_metadata(url, timeout=2, sec_between=2, retries=30):
response = url_helper.readurl(url, timeout=timeout,
sec_between=sec_between, retries=retries)
if not response.ok():
raise RuntimeError("unable to read metadata at %s" % url)
return json.loads(response.contents.decode())
def get_data(self):
# GCE metadata server requires a custom header since v1
headers = {'X-Google-Metadata-Request': True}
# url_map: (our-key, path, required, is_text)
url_map = [
('instance-id', 'instance/id', True, True),
('availability-zone', 'instance/zone', True, True),
('local-hostname', 'instance/hostname', True, True),
('public-keys', 'project/attributes/sshKeys', False, True),
('user-data', 'instance/attributes/user-data', False, False),
('user-data-encoding', 'instance/attributes/user-data-encoding',
False, True),
]
# if we cannot resolve the metadata server, then no point in trying
if not util.is_resolvable_url(self.metadata_address):
LOG.debug("%s is not resolvable", self.metadata_address)
return False
# iterate over url_map keys to get metadata items
found = False
for (mkey, path, required, is_text) in url_map:
try:
resp = url_helper.readurl(url=self.metadata_address + path,
headers=headers)
if resp.code == 200:
found = True
if is_text:
self.metadata[mkey] = util.decode_binary(resp.contents)
else:
self.metadata[mkey] = resp.contents
else:
if required:
msg = "required url %s returned code %s. not GCE"
if not found:
LOG.debug(msg, path, resp.code)
else:
LOG.warn(msg, path, resp.code)
return False
else:
self.metadata[mkey] = None
except url_helper.UrlError as e:
if required:
msg = "required url %s raised exception %s. not GCE"
if not found:
LOG.debug(msg, path, e)
else:
LOG.warn(msg, path, e)
return False
msg = "Failed to get %s metadata item: %s."
LOG.debug(msg, path, e)
self.metadata[mkey] = None
if self.metadata['public-keys']:
lines = self.metadata['public-keys'].splitlines()
self.metadata['public-keys'] = [self._trim_key(k) for k in lines]
encoding = self.metadata.get('user-data-encoding')
if encoding:
if encoding == 'base64':
self.metadata['user-data'] = b64decode(
self.metadata['user-data'])
else:
LOG.warn('unknown user-data-encoding: %s, ignoring', encoding)
return found
