def _bringup_router(self):
"""Perform the ip commands to fully setup the router if needed."""
# Check if a default route exists and exit if it does
out, _ = subp.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True)
if 'default' in out:
LOG.debug(
'Skip ephemeral route setup. %s already has default route: %s',
self.interface, out.strip())
return
subp.subp(
['ip', '-4', 'route', 'add', self.router, 'dev', self.interface,
'src', self.ip], capture=True)
self.cleanup_cmds.insert(
0,
['ip', '-4', 'route', 'del', self.router, 'dev', self.interface,
'src', self.ip])
subp.subp(
['ip', '-4', 'route', 'add', 'default', 'via', self.router,
'dev', self.interface], capture=True)
self.cleanup_cmds.insert(
0, ['ip', '-4', 'route', 'del', 'default', 'dev', self.interface])
def get_interfaces_by_mac_on_netbsd():
ret = {}
re_field_match = (r"(?P<ifname>\w+).*address:\s"
r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*")
(out, _) = subp.subp(['ifconfig', '-a'])
if_lines = re.sub(r'\n\s+', ' ', out).splitlines()
for line in if_lines:
m = re.match(re_field_match, line)
if m:
fields = m.groupdict()
ret[fields['mac']] = fields['ifname']
return ret
def is_lvm_lv(devpath):
if util.is_Linux():
# all lvm lvs will have a realpath as a 'dm-*' name.
rpath = os.path.realpath(devpath)
if not os.path.basename(rpath).startswith("dm-"):
return False
out, _ = subp.subp("udevadm", "info", devpath)
# lvs should have DM_LV_NAME=<lvmuuid> and also DM_VG_NAME
return 'DM_LV_NAME=' in out
else:
LOG.info("Not an LVM Logical Volume partition")
return False
def get_interfaces_by_mac_on_openbsd(blacklist_drivers=None) -> dict:
ret = {}
re_field_match = (r"(?P<ifname>\w+).*lladdr\s"
r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*")
(out, _) = subp.subp(["ifconfig", "-a"])
if_lines = re.sub(r"\n\s+", " ", out).splitlines()
for line in if_lines:
m = re.match(re_field_match, line)
if m:
fields = m.groupdict()
ret[fields["mac"]] = fields["ifname"]
return ret
def available(self):
myenv = os.environ.copy()
myenv['LANG'] = 'C'
try:
(_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1])
if re.search(r"gpart recover ", err):
return True
except subp.ProcessExecutionError:
pass
return False
def available(self):
myenv = os.environ.copy()
myenv['LANG'] = 'C'
try:
(out, _err) = subp.subp(["growpart", "--help"], env=myenv)
if re.search(r"--update\s+", out):
return True
except subp.ProcessExecutionError:
pass
return False
def package_command(self, command, args=None, pkgs=None):
if pkgs is None:
pkgs = []
cmd = ['apk']
# Redirect output
cmd.append("--quiet")
if args and isinstance(args, str):
cmd.append(args)
elif args and isinstance(args, list):
cmd.extend(args)
if command:
cmd.append(command)
pkglist = util.expand_package_list('%s-%s', pkgs)
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)
def handle(name, cfg, _cloud, log, _args):
mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {}))
if not mycfg:
mycfg = {}
enabled = mycfg.get("enabled", True)
if util.is_false(enabled):
log.debug("%s disabled by config grub_dpkg/enabled=%s", name, enabled)
return
idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None)
if idevs is None:
idevs = fetch_idevs(log)
idevs_empty = mycfg.get("grub-pc/install_devices_empty")
if idevs_empty is None:
idevs_empty = not idevs
elif not isinstance(idevs_empty, bool):
log.warning(
"DEPRECATED: grub_dpkg: grub-pc/install_devices_empty value of "
f"'{idevs_empty}' is not boolean. Use of non-boolean values "
"will be removed in a future version of cloud-init.")
idevs_empty = util.translate_bool(idevs_empty)
idevs_empty = str(idevs_empty).lower()
# now idevs and idevs_empty are set to determined values
# or, those set by user
dconf_sel = ("grub-pc grub-pc/install_devices string %s\n"
"grub-pc grub-pc/install_devices_empty boolean %s\n" %
(idevs, idevs_empty))
log.debug("Setting grub debconf-set-selections with '%s','%s'" %
(idevs, idevs_empty))
try:
subp.subp(["debconf-set-selections"], dconf_sel)
except Exception:
util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg")
def exec_mkpart_gpt(device, layout):
try:
subp.subp([SGDISK_CMD, "-Z", device])
for index, (partition_type, (start, end)) in enumerate(layout):
index += 1
subp.subp(
[
SGDISK_CMD,
"-n",
"{}:{}:{}".format(index, start, end),
device,
]
)
if partition_type is not None:
# convert to a 4 char (or more) string right padded with 0
# 82 -> 8200. 'Linux' -> 'Linux'
pinput = str(partition_type).ljust(4, "0")
subp.subp(
[SGDISK_CMD, "-t", "{}:{}".format(index, pinput), device]
)
except Exception:
LOG.warning("Failed to partition device %s", device)
raise
read_parttbl(device)
def netdev_info(empty=""):
devs = {}
if util.is_NetBSD():
(ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1])
devs = _netdev_info_ifconfig_netbsd(ifcfg_out)
elif subp.which('ip'):
# Try iproute first of all
(ipaddr_out, _err) = subp.subp(["ip", "addr", "show"])
devs = _netdev_info_iproute(ipaddr_out)
elif subp.which('ifconfig'):
# Fall back to net-tools if iproute2 is not present
(ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1])
devs = _netdev_info_ifconfig(ifcfg_out)
else:
LOG.warning(
"Could not print networks: missing 'ip' and 'ifconfig' commands")
if empty == "":
return devs
recurse_types = (dict, tuple, list)
def fill(data, new_val="", empty_vals=("", b"")):
"""Recursively replace 'empty_vals' in data (dict, tuple, list)
with new_val"""
if isinstance(data, dict):
myiter = data.items()
elif isinstance(data, (tuple, list)):
myiter = enumerate(data)
else:
raise TypeError("Unexpected input to fill")
for key, val in myiter:
if val in empty_vals:
data[key] = new_val
elif isinstance(val, recurse_types):
fill(val, new_val)
fill(devs, new_val=empty)
return devs
def get_underlying_partition(blockdev):
command = ["dmsetup", "deps", "--options=devname", blockdev]
dep: str = subp.subp(command)[0] # pyright: ignore
# Returned result should look something like:
# 1 dependencies : (vdb1)
if not dep.startswith("1 depend"):
raise RuntimeError(
f"Expecting '1 dependencies' from 'dmsetup'. Received: {dep}")
try:
return f'/dev/{dep.split(": (")[1].split(")")[0]}'
except IndexError as e:
raise RuntimeError(
f"Ran `{command}`, but received unexpected stdout: `{dep}`") from e
def package_command(self, command, args=None, pkgs=None):
if pkgs is None:
pkgs = []
cmd = ["pacman", "-Sy", "--quiet", "--noconfirm"]
# Redirect output
if args and isinstance(args, str):
cmd.append(args)
elif args and isinstance(args, list):
cmd.extend(args)
if command == "upgrade":
command = "-u"
if command:
cmd.append(command)
pkglist = util.expand_package_list("%s-%s", pkgs)
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)
def set_passwd(self, user, passwd, hashed=False):
if hashed:
hashed_pw = passwd
elif not hasattr(crypt, "METHOD_BLOWFISH"):
# crypt.METHOD_BLOWFISH comes with Python 3.7 which is available
# on NetBSD 7 and 8.
LOG.error(
"Cannot set non-encrypted password for user %s. "
"Python >= 3.7 is required.",
user,
)
return
else:
method = crypt.METHOD_BLOWFISH # pylint: disable=E1101
hashed_pw = crypt.crypt(passwd, crypt.mksalt(method))
try:
subp.subp(["usermod", "-p", hashed_pw, user])
except Exception:
util.logexc(LOG, "Failed to set password for %s", user)
raise
self.unlock_passwd(user)
def handle(_name, cfg, cloud, log, _args):
validate_cloudconfig_schema(cfg, schema)
network_hotplug_enabled = ("updates" in cfg and "network" in cfg["updates"]
and "when" in cfg["updates"]["network"] and
"hotplug" in cfg["updates"]["network"]["when"])
hotplug_supported = EventType.HOTPLUG in (
cloud.datasource.get_supported_events([EventType.HOTPLUG
]).get(EventScope.NETWORK,
set()))
hotplug_enabled = stages.update_event_enabled(
datasource=cloud.datasource,
cfg=cfg,
event_source_type=EventType.HOTPLUG,
scope=EventScope.NETWORK,
)
if not (hotplug_supported and hotplug_enabled):
if os.path.exists(HOTPLUG_UDEV_PATH):
log.debug("Uninstalling hotplug, not enabled")
util.del_file(HOTPLUG_UDEV_PATH)
subp.subp(["udevadm", "control", "--reload-rules"])
elif network_hotplug_enabled:
log.warning("Hotplug is unsupported by current datasource. "
"Udev rules will NOT be installed.")
else:
log.debug("Skipping hotplug install, not enabled")
return
if not subp.which("udevadm"):
log.debug("Skipping hotplug install, udevadm not found")
return
# This may need to turn into a distro property at some point
libexecdir = "/usr/libexec/cloud-init"
if not os.path.exists(libexecdir):
libexecdir = "/usr/lib/cloud-init"
util.write_file(
filename=HOTPLUG_UDEV_PATH,
content=HOTPLUG_UDEV_RULES_TEMPLATE.format(libexecdir=libexecdir),
)
subp.subp(["udevadm", "control", "--reload-rules"])
def _bring_up_interface(self, device_name):
cmd = ['ifup', device_name]
LOG.debug("Attempting to run bring up interface %s using command %s",
device_name, cmd)
try:
(_out, err) = subp.subp(cmd)
if len(err):
LOG.warning("Running %s resulted in stderr output: %s", cmd,
err)
return True
except subp.ProcessExecutionError:
util.logexc(LOG, "Running interface command %s failed", cmd)
return False
def features(self):
if self._features is None:
try:
info_blob, _err = subp.subp(self.NETPLAN_INFO, capture=True)
info = util.load_yaml(info_blob)
self._features = info["netplan.io"]["features"]
except subp.ProcessExecutionError:
# if the info subcommand is not present then we don't have any
# new features
pass
except (TypeError, KeyError) as e:
LOG.debug("Failed to list features from netplan info: %s", e)
return self._features
def guestinfo_set_value(key, value, vmware_rpctool=VMWARE_RPCTOOL):
"""
Sets a guestinfo value for the specified key. Set value to an empty string
to clear an existing guestinfo key.
"""
# If value is an empty string then set it to a single space as it is not
# possible to set a guestinfo key to an empty string. Setting a guestinfo
# key to a single space is as close as it gets to clearing an existing
# guestinfo key.
if value == "":
value = " "
LOG.debug("Setting guestinfo key=%s to value=%s", key, value)
try:
subp([
vmware_rpctool,
("info-set %s %s" % (get_guestinfo_key_name(key), value)),
])
return True
except ProcessExecutionError as error:
util.logexc(
LOG,
"Failed to set guestinfo key=%s to value=%s: %s",
key,
value,
error,
)
except Exception:
util.logexc(
LOG,
"Unexpected error while trying to set " +
"guestinfo key=%s to value=%s",
key,
value,
)
return None
def recv_key(key, keyserver, retries=(1, 1)):
"""Receive gpg key from the specified keyserver.
Retries are done by default because keyservers can be unreliable.
Additionally, there is no way to determine the difference between
a non-existant key and a failure. In both cases gpg (at least 2.2.4)
exits with status 2 and stderr: "keyserver receive failed: No data"
It is assumed that a key provided to cloud-init exists on the keyserver
so re-trying makes better sense than failing.
@param key: a string key fingerprint (as passed to gpg --recv-keys).
@param keyserver: the keyserver to request keys from.
@param retries: an iterable of sleep lengths for retries.
Use None to indicate no retries."""
LOG.debug("Importing key '%s' from keyserver '%s'", key, keyserver)
cmd = ["gpg", "--keyserver=%s" % keyserver, "--recv-keys", key]
if retries is None:
retries = []
trynum = 0
error = None
sleeps = iter(retries)
while True:
trynum += 1
try:
subp.subp(cmd, capture=True)
LOG.debug("Imported key '%s' from keyserver '%s' on try %d", key,
keyserver, trynum)
return
except subp.ProcessExecutionError as e:
error = e
try:
naplen = next(sleeps)
LOG.debug("Import failed with exit code %d, will try again in %ss",
error.exit_code, naplen)
time.sleep(naplen)
except StopIteration:
raise ValueError(
("Failed to import key '%s' from keyserver '%s' "
"after %d tries: %s") % (key, keyserver, trynum, error))
def set_route():
# Get routes, confirm entry does not exist
routes = netinfo.route_info()
# If no tools exist and empty dict is returned
if "ipv4" not in routes:
return
# We only care about IPv4
routes = routes["ipv4"]
# Searchable list
dests = []
# Parse each route into a more searchable format
for route in routes:
dests.append(route["destination"])
gw_present = "100.64.0.0" in dests or "100.64.0.0/10" in dests
dest_present = "169.254.169.254" in dests
# If not IPv6 only (No link local)
# or the route is already present
if not gw_present or dest_present:
return
# Set metadata route
if subp.which("ip"):
subp.subp([
"ip",
"route",
"add",
"169.254.169.254/32",
"dev",
net.find_fallback_nic(),
])
elif subp.which("route"):
subp.subp(["route", "add", "-net", "169.254.169.254/32", "100.64.0.1"])
def resize(self, diskdev, partnum, partdev):
"""
GPT disks store metadata at the beginning (primary) and at the
end (secondary) of the disk. When launching an image with a
larger disk compared to the original image, the secondary copy
is lost. Thus, the metadata will be marked CORRUPT, and need to
be recovered.
"""
try:
subp.subp(["gpart", "recover", diskdev])
except subp.ProcessExecutionError as e:
if e.exit_code != 0:
util.logexc(LOG, "Failed: gpart recover %s", diskdev)
raise ResizeFailedException(e) from e
before = get_size(partdev)
try:
subp.subp(["gpart", "resize", "-i", partnum, diskdev])
except subp.ProcessExecutionError as e:
util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
raise ResizeFailedException(e) from e
return (before, get_size(partdev))
def package_command(self, command, args=None, pkgs=None):
if pkgs is None:
pkgs = []
# No user interaction possible, enable non-interactive mode
cmd = ['zypper', '--non-interactive']
# Command is the operation, such as install
if command == 'upgrade':
command = 'update'
cmd.append(command)
# args are the arguments to the command, not global options
if args and isinstance(args, str):
cmd.append(args)
elif args and isinstance(args, list):
cmd.extend(args)
pkglist = util.expand_package_list('%s-%s', pkgs)
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)
def dpkg_reconfigure(packages, target=None):
# For any packages that are already installed, but have preseed data
# we populate the debconf database, but the filesystem configuration
# would be preferred on a subsequent dpkg-reconfigure.
# so, what we have to do is "know" information about certain packages
# to unconfigure them.
unhandled = []
to_config = []
for pkg in packages:
if pkg in CONFIG_CLEANERS:
LOG.debug("unconfiguring %s", pkg)
CONFIG_CLEANERS[pkg](target)
to_config.append(pkg)
else:
unhandled.append(pkg)
if len(unhandled):
LOG.warning("The following packages were installed and preseeded, "
"but cannot be unconfigured: %s", unhandled)
if len(to_config):
subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
list(to_config), data=None, target=target, capture=True)
def assign_ipv4_link_local(nic=None):
"""Bring up NIC using an address using link-local (ip4LL) IPs. On
DigitalOcean, the link-local domain is per-droplet routed, so there
is no risk of collisions. However, to be more safe, the ip4LL
address is random.
"""
if not nic:
nic = get_link_local_nic()
LOG.debug("selected interface '%s' for reading metadata", nic)
if not nic:
raise RuntimeError("unable to find interfaces to access the"
"meta-data server. This droplet is broken.")
addr = "169.254.{0}.{1}/16".format(random.randint(1, 168),
random.randint(0, 255))
ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic]
ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up']
if not subp.which('ip'):
raise RuntimeError("No 'ip' command available to configure ip4LL "
"address")
try:
subp.subp(ip_addr_cmd)
LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic)
subp.subp(ip_link_cmd)
LOG.debug("brought device '%s' up", nic)
except Exception:
util.logexc(
LOG, "ip4LL address assignment of '%s' to '%s' failed."
" Droplet networking will be broken", addr, nic)
raise
return nic
def run_commands(commands):
"""Run the provided commands provided in snap:commands configuration.
Commands are run individually. Any errors are collected and reported
after attempting all commands.
@param commands: A list or dict containing commands to run. Keys of a
dict will be used to order the commands provided as dict values.
"""
if not commands:
return
LOG.debug("Running user-provided snap commands")
if isinstance(commands, dict):
# Sort commands based on dictionary key
commands = [v for _, v in sorted(commands.items())]
elif not isinstance(commands, list):
raise TypeError(
"commands parameter was not a list or dict: {commands}".format(
commands=commands
)
)
fixed_snap_commands = prepend_base_command("snap", commands)
cmd_failures = []
for command in fixed_snap_commands:
shell = isinstance(command, str)
try:
subp.subp(command, shell=shell, status_cb=sys.stderr.write)
except subp.ProcessExecutionError as e:
cmd_failures.append(str(e))
if cmd_failures:
msg = "Failures running snap commands:\n{cmd_failures}".format(
cmd_failures=cmd_failures
)
util.logexc(LOG, msg)
raise RuntimeError(msg)
def get_pvs_for_lv(devpath):
myenv = {'LANG': 'C'}
if not util.is_Linux():
LOG.info("No support for LVM on %s", platform.system())
return None
if not subp.which('lvm'):
LOG.info("No 'lvm' command present")
return None
try:
(out, _err) = subp.subp(
["lvm", "lvs", devpath, "--options=vgname", "--noheadings"],
update_env=myenv)
vgname = out.strip()
except subp.ProcessExecutionError as e:
if e.exit_code != 0:
util.logexc(
LOG, "Failed: can't get Volume Group information "
"from %s", devpath)
raise ResizeFailedException(e) from e
try:
(out, _err) = subp.subp(
["lvm", "vgs", vgname, "--options=pvname", "--noheadings"],
update_env=myenv)
pvs = [p.strip() for p in out.splitlines()]
if len(pvs) > 1:
LOG.info("Do not know how to resize multiple Physical" " Volumes")
else:
return pvs[0]
except subp.ProcessExecutionError as e:
if e.exit_code != 0:
util.logexc(
LOG, "Failed: can't get Physical Volume "
"information from Volume Group %s", vgname)
raise ResizeFailedException(e) from e
def _read_hostname(self, filename, default=None):
if self.uses_systemd() and filename.endswith("/previous-hostname"):
return util.load_file(filename).strip()
elif self.uses_systemd():
(out, _err) = subp.subp(["hostname"])
if len(out):
return out
else:
return default
else:
(_exists, contents) = rhel_util.read_sysconfig_file(filename)
if "HOSTNAME" in contents:
return contents["HOSTNAME"]
else:
return default
def do_register(
server,
profile_name,
ca_cert_path=def_ca_cert_path,
proxy=None,
log=None,
activation_key=None,
):
if log is not None:
log.info(
"Registering using `rhnreg_ks` profile '%s' into server '%s'",
profile_name,
server,
)
cmd = ["rhnreg_ks"]
cmd.extend(["--serverUrl", "https://%s/XMLRPC" % server])
cmd.extend(["--profilename", str(profile_name)])
if proxy:
cmd.extend(["--proxy", str(proxy)])
if ca_cert_path:
cmd.extend(["--sslCACert", str(ca_cert_path)])
if activation_key:
cmd.extend(["--activationkey", str(activation_key)])
subp.subp(cmd, capture=False)
def package_command(self, command, args=None, pkgs=None):
if pkgs is None:
pkgs = []
cmd = ["apk"]
# Redirect output
cmd.append("--quiet")
if args and isinstance(args, str):
cmd.append(args)
elif args and isinstance(args, list):
cmd.extend(args)
if command:
cmd.append(command)
if command == "upgrade":
cmd.extend(["--update-cache", "--available"])
pkglist = util.expand_package_list("%s-%s", pkgs)
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
subp.subp(cmd, capture=False)
def subp(self):
"""
Make a subp call based on set args and handle errors by setting
failure code
:return: whether the subp call failed or not
"""
try:
value, err = subp.subp(self.args, capture=True)
if err:
return err
self.epoch = value
return None
except Exception as systemctl_fail:
return systemctl_fail
def create_group(self, name, members=None):
if util.is_group(name):
LOG.warning("Skipping creation of existing group '%s'", name)
else:
group_add_cmd = self.group_add_cmd_prefix + [name]
try:
subp.subp(group_add_cmd)
LOG.info("Created new group %s", name)
except Exception:
util.logexc(LOG, "Failed to create group %s", name)
if not members:
members = []
for member in members:
if not util.is_user(member):
LOG.warning("Unable to add group member '%s' to group '%s'"
"; user does not exist.", member, name)
continue
try:
subp.subp(self._get_add_member_to_group_cmd(member, name))
LOG.info("Added user '%s' to group '%s'", member, name)
except Exception:
util.logexc(LOG, "Failed to add user '%s' to group '%s'",
member, name)
