def _should_configure_on_empty_apt():
# if no config was provided, should apt configuration be done?
if util.system_is_snappy():
return False, "system is snappy."
if not (util.which('apt-get') or util.which('apt')):
return False, "no apt commands."
return True, "Apt is available."
def _should_configure_on_empty_apt():
# if no config was provided, should apt configuration be done?
if util.system_is_snappy():
return False, "system is snappy."
if not (util.which('apt-get') or util.which('apt')):
return False, "no apt commands."
return True, "Apt is available."
def set_snappy_command():
global SNAPPY_CMD
if util.which("snappy-go"):
SNAPPY_CMD = "snappy-go"
elif util.which("snappy"):
SNAPPY_CMD = "snappy"
else:
SNAPPY_CMD = "snap"
LOG.debug("snappy command is '%s'", SNAPPY_CMD)
def route_info():
routes = {}
if util.which('ip'):
# Try iproute first of all
(iproute_out, _err) = util.subp(["ip", "-o", "route", "list"])
routes = _netdev_route_info_iproute(iproute_out)
elif util.which('netstat'):
# Fall back to net-tools if iproute2 is not present
(route_out, _err) = util.subp(
["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1])
routes = _netdev_route_info_netstat(route_out)
else:
LOG.warning(
"Could not print routes: missing 'ip' and 'netstat' commands")
return routes
def route_info():
routes = {}
if util.which('ip'):
# Try iproute first of all
(iproute_out, _err) = util.subp(["ip", "-o", "route", "list"])
routes = _netdev_route_info_iproute(iproute_out)
elif util.which('netstat'):
# Fall back to net-tools if iproute2 is not present
(route_out, _err) = util.subp(
["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1])
routes = _netdev_route_info_netstat(route_out)
else:
LOG.warning(
"Could not print routes: missing 'ip' and 'netstat' commands")
return routes
def handle(name, cfg, cloud, log, args):
# Get config
lxd_cfg = cfg.get('lxd')
if not lxd_cfg:
log.debug("Skipping module named %s, not present or disabled by cfg")
return
if not isinstance(lxd_cfg, dict):
log.warn("lxd config must be a dictionary. found a '%s'",
type(lxd_cfg))
return
init_cfg = lxd_cfg.get('init')
if not isinstance(init_cfg, dict):
log.warn("lxd/init config must be a dictionary. found a '%s'",
type(init_cfg))
init_cfg = {}
if not init_cfg:
log.debug("no lxd/init config. disabled.")
return
packages = []
# Ensure lxd is installed
if not util.which("lxd"):
packages.append('lxd')
# if using zfs, get the utils
if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'):
packages.append('zfs')
if len(packages):
try:
cloud.distro.install_packages(packages)
except util.ProcessExecutionError as exc:
log.warn("failed to install packages %s: %s", packages, exc)
return
# Set up lxd if init config is given
init_keys = (
'network_address', 'network_port', 'storage_backend',
'storage_create_device', 'storage_create_loop',
'storage_pool', 'trust_password')
cmd = ['lxd', 'init', '--auto']
for k in init_keys:
if init_cfg.get(k):
cmd.extend(["--%s=%s" %
(k.replace('_', '-'), str(init_cfg[k]))])
util.subp(cmd)
def perform_hostname_bounce(hostname, cfg, prev_hostname):
# set the hostname to 'hostname' if it is not already set to that.
# then, if policy is not off, bounce the interface using command
# Returns True if the network was bounced, False otherwise.
command = cfg['command']
interface = cfg['interface']
policy = cfg['policy']
msg = ("hostname=%s policy=%s interface=%s" %
(hostname, policy, interface))
env = os.environ.copy()
env['interface'] = interface
env['hostname'] = hostname
env['old_hostname'] = prev_hostname
if command == "builtin":
if util.is_FreeBSD():
command = BOUNCE_COMMAND_FREEBSD
elif util.which('ifup'):
command = BOUNCE_COMMAND_IFUP
else:
LOG.debug(
"Skipping network bounce: ifupdown utils aren't present.")
# Don't bounce as networkd handles hostname DDNS updates
return False
LOG.debug("pubhname: publishing hostname [%s]", msg)
shell = not isinstance(command, (list, tuple))
# capture=False, see comments in bug 1202758 and bug 1206164.
util.log_time(logfunc=LOG.debug, msg="publishing hostname",
get_uptime=True, func=util.subp,
kwargs={'args': command, 'shell': shell, 'capture': False,
'env': env})
return True
def package_command(self, command, args=None, pkgs=None):
if pkgs is None:
pkgs = []
if util.which('dnf'):
LOG.debug('Using DNF for package management')
cmd = ['dnf']
else:
LOG.debug('Using YUM for package management')
# the '-t' argument makes yum tolerant of errors on the command
# line with regard to packages.
#
# For example: if you request to install foo, bar and baz and baz
# is installed; yum won't error out complaining that baz is already
# installed.
cmd = ['yum', '-t']
# Determines whether or not yum prompts for confirmation
# of critical actions. We don't want to prompt...
cmd.append("-y")
if args and isinstance(args, str):
cmd.append(args)
elif args and isinstance(args, list):
cmd.extend(args)
cmd.append(command)
pkglist = util.expand_package_list('%s-%s', pkgs)
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
util.subp(cmd, capture=False)
def get_tools_config(section, key, defaultVal):
""" Return the value of [section] key from VMTools configuration.
@param section: String of section to read from VMTools config
@returns: String value from key in [section] or defaultVal if
[section] is not present or vmware-toolbox-cmd is
not installed.
"""
if not util.which('vmware-toolbox-cmd'):
logger.debug(
'vmware-toolbox-cmd not installed, returning default value')
return defaultVal
retValue = defaultVal
cmd = ['vmware-toolbox-cmd', 'config', 'get', section, key]
try:
(outText, _) = util.subp(cmd)
m = re.match(r'([^=]+)=(.*)', outText)
if m:
retValue = m.group(2).strip()
logger.debug("Get tools config: [%s] %s = %s", section, key,
retValue)
else:
logger.debug(
"Tools config: [%s] %s is not found, return default value: %s",
section, key, retValue)
except util.ProcessExecutionError as e:
logger.error("Failed running %s[%s]", cmd, e.exit_code)
logger.exception(e)
return retValue
def install_ntp(install_func, packages=None, check_exe="ntpd"):
if util.which(check_exe):
return
if packages is None:
packages = ['ntp']
install_func(packages)
def available(target=None):
expected = ['netplan']
search = ['/usr/sbin', '/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
return True
def set_snappy_command():
global SNAPPY_CMD
if util.which("snappy-go"):
SNAPPY_CMD = "snappy-go"
else:
SNAPPY_CMD = "snappy"
LOG.debug("snappy command is '%s'", SNAPPY_CMD)
def handle(name, cfg, _cloud, log, _args):
disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False)
if disabled:
reject_cmd = None
if util.which('ip'):
reject_cmd = REJECT_CMD_IP
elif util.which('ifconfig'):
reject_cmd = REJECT_CMD_IF
else:
log.error(('Neither "route" nor "ip" command found, unable to '
'manipulate routing table'))
return
util.subp(reject_cmd, capture=False)
else:
log.debug(("Skipping module named %s,"
" disabling the ec2 route not enabled"), name)
def maybe_perform_dhcp_discovery(nic=None):
"""Perform dhcp discovery if nic valid and dhclient command exists.
If the nic is invalid or undiscoverable or dhclient command is not found,
skip dhcp_discovery and return an empty dict.
@param nic: Name of the network interface we want to run dhclient on.
@return: A list of dicts representing dhcp options for each lease obtained
from the dhclient discovery if run, otherwise an empty list is
returned.
"""
if nic is None:
nic = find_fallback_nic()
if nic is None:
LOG.debug('Skip dhcp_discovery: Unable to find fallback nic.')
return []
elif nic not in get_devicelist():
LOG.debug(
'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic)
return []
dhclient_path = util.which('dhclient')
if not dhclient_path:
LOG.debug('Skip dhclient configuration: No dhclient command found.')
return []
with temp_utils.tempdir(rmtree_ignore_errors=True,
prefix='cloud-init-dhcp-',
needs_exe=True) as tdir:
# Use /var/tmp because /run/cloud-init/tmp is mounted noexec
return dhcp_discovery(dhclient_path, nic, tdir)
def install_ntp(install_func, packages=None, check_exe="ntpd"):
if util.which(check_exe):
return
if packages is None:
packages = ['ntp']
install_func(packages)
def maybe_perform_dhcp_discovery(nic=None):
"""Perform dhcp discovery if nic valid and dhclient command exists.
If the nic is invalid or undiscoverable or dhclient command is not found,
skip dhcp_discovery and return an empty dict.
@param nic: Name of the network interface we want to run dhclient on.
@return: A list of dicts representing dhcp options for each lease obtained
from the dhclient discovery if run, otherwise an empty list is
returned.
"""
if nic is None:
nic = find_fallback_nic()
if nic is None:
LOG.debug('Skip dhcp_discovery: Unable to find fallback nic.')
return []
elif nic not in get_devicelist():
LOG.debug(
'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic)
return []
dhclient_path = util.which('dhclient')
if not dhclient_path:
LOG.debug('Skip dhclient configuration: No dhclient command found.')
return []
with temp_utils.tempdir(rmtree_ignore_errors=True,
prefix='cloud-init-dhcp-',
needs_exe=True) as tdir:
# Use /var/tmp because /run/cloud-init/tmp is mounted noexec
return dhcp_discovery(dhclient_path, nic, tdir)
def add_local_ip(nic=None):
nic = get_local_ip_nic()
LOG.debug("selected interface '%s' for reading metadata", nic)
if not nic:
raise RuntimeError("unable to find interfaces to access the"
"meta-data server. This droplet is broken.")
ip_addr_cmd = ['ip', 'addr', 'add', '169.254.0.1/16', 'dev', nic]
ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up']
if not util.which('ip'):
raise RuntimeError("No 'ip' command available to configure local ip "
"address")
try:
(result, _err) = util.subp(ip_addr_cmd)
LOG.debug("assigned local ip to '%s'", nic)
(result, _err) = util.subp(ip_link_cmd)
LOG.debug("brought device '%s' up", nic)
except Exception:
util.logexc(LOG, "local ip address assignment to '%s' failed.", nic)
raise
return nic
def maybe_perform_dhcp_discovery(nic=None):
"""Perform dhcp discovery if nic valid and dhclient command exists.
If the nic is invalid or undiscoverable or dhclient command is not found,
skip dhcp_discovery and return an empty dict.
@param nic: Name of the network interface we want to run dhclient on.
@return: A dict of dhcp options from the dhclient discovery if run,
otherwise an empty dict is returned.
"""
if nic is None:
nic = find_fallback_nic()
if nic is None:
LOG.debug('Skip dhcp_discovery: Unable to find fallback nic.')
return {}
elif nic not in get_devicelist():
LOG.debug('Skip dhcp_discovery: nic %s not found in get_devicelist.',
nic)
return {}
dhclient_path = util.which('dhclient')
if not dhclient_path:
LOG.debug('Skip dhclient configuration: No dhclient command found.')
return {}
with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir:
return dhcp_discovery(dhclient_path, nic, tmpdir)
def package_command(self, command, args=None, pkgs=None):
if pkgs is None:
pkgs = []
if util.which('dnf'):
LOG.debug('Using DNF for package management')
cmd = ['dnf']
else:
LOG.debug('Using YUM for package management')
# the '-t' argument makes yum tolerant of errors on the command
# line with regard to packages.
#
# For example: if you request to install foo, bar and baz and baz
# is installed; yum won't error out complaining that baz is already
# installed.
cmd = ['yum', '-t']
# Determines whether or not yum prompts for confirmation
# of critical actions. We don't want to prompt...
cmd.append("-y")
if args and isinstance(args, str):
cmd.append(args)
elif args and isinstance(args, list):
cmd.extend(args)
cmd.append(command)
pkglist = util.expand_package_list('%s-%s', pkgs)
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
util.subp(cmd, capture=False)
def available(target=None):
expected = ['netplan']
search = ['/usr/sbin', '/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
return True
def _get_wrapper_prefix(cmd, mode):
if isinstance(cmd, str):
cmd = [str(cmd)]
if util.is_true(mode) or (str(mode).lower() == "auto" and cmd[0] and util.which(cmd[0])):
return cmd
else:
return []
def handle(name, cfg, cloud, log, args):
"""Handler method activated by cloud-init."""
if not isinstance(cloud.distro, ubuntu.Distro):
log.debug("%s: distro is '%s', not ubuntu. returning",
name, cloud.distro.__class__)
return
cfg = util.mergemanydict([cfg, DEFAULT_CONFIG])
target = cfg['init_switch']['target']
reboot = cfg['init_switch']['reboot']
if len(args) != 0:
target = args[0]
if len(args) > 1:
reboot = util.is_true(args[1])
if not target:
log.debug("%s: target=%s. nothing to do", name, target)
return
if not util.which('dpkg'):
log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name)
return
supported = ('upstart', 'systemd')
if target not in supported:
log.warn("%s: target set to %s, expected one of: %s",
name, target, str(supported))
if os.path.exists("/run/systemd/system"):
current = "systemd"
else:
current = "upstart"
if current == target:
log.debug("%s: current = target = %s. nothing to do", name, target)
return
try:
util.subp(['sh', '-s', target], data=SWITCH_INIT)
except util.ProcessExecutionError as e:
log.warn("%s: Failed to switch to init '%s'. %s", name, target, e)
return
if util.is_false(reboot):
log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.",
name, current, target)
return
try:
log.warn("%s: switched '%s' to '%s'. rebooting.",
name, current, target)
logging.flushLoggers(log)
_fire_reboot(log, wait_attempts=4, initial_sleep=4)
except Exception as e:
util.logexc(log, "Requested reboot did not happen!")
raise
def handle(name, cfg, cloud, log, args):
"""Handler method activated by cloud-init."""
if not isinstance(cloud.distro, ubuntu.Distro):
log.debug("%s: distro is '%s', not ubuntu. returning", name,
cloud.distro.__class__)
return
cfg = util.mergemanydict([cfg, DEFAULT_CONFIG])
target = cfg['init_switch']['target']
reboot = cfg['init_switch']['reboot']
if len(args) != 0:
target = args[0]
if len(args) > 1:
reboot = util.is_true(args[1])
if not target:
log.debug("%s: target=%s. nothing to do", name, target)
return
if not util.which('dpkg'):
log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name)
return
supported = ('upstart', 'systemd')
if target not in supported:
log.warn("%s: target set to %s, expected one of: %s", name, target,
str(supported))
if os.path.exists("/run/systemd/system"):
current = "systemd"
else:
current = "upstart"
if current == target:
log.debug("%s: current = target = %s. nothing to do", name, target)
return
try:
util.subp(['sh', '-s', target], data=SWITCH_INIT)
except util.ProcessExecutionError as e:
log.warn("%s: Failed to switch to init '%s'. %s", name, target, e)
return
if util.is_false(reboot):
log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.",
name, current, target)
return
try:
log.warn("%s: switched '%s' to '%s'. rebooting.", name, current,
target)
logging.flushLoggers(log)
_fire_reboot(log, wait_attempts=4, initial_sleep=4)
except Exception as e:
util.logexc(log, "Requested reboot did not happen!")
raise
def _get_wrapper_prefix(cmd, mode):
if isinstance(cmd, str):
cmd = [str(cmd)]
if (util.is_true(mode) or
(str(mode).lower() == "auto" and cmd[0] and util.which(cmd[0]))):
return cmd
else:
return []
def apply_apt(cfg, cloud, target):
# cfg is the 'apt' top level dictionary already in 'v3' format.
if not cfg:
# no config was provided. If apt configuration does not seem
# necessary on this system, then return.
if util.system_is_snappy():
LOG.debug("Nothing to do: No apt config and running on snappy")
return
if not (util.which('apt-get') or util.which('apt')):
LOG.debug("Nothing to do: No apt config and no apt commands")
return
LOG.debug("handling apt config: %s", cfg)
release = util.lsb_release(target=target)['codename']
arch = util.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
LOG.debug("Apt Mirror info: %s", mirrors)
if util.is_false(cfg.get('preserve_sources_list', False)):
generate_sources_list(cfg, release, mirrors, cloud)
rename_apt_lists(mirrors, target)
try:
apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'],
cloud,
target=target,
template_params=params,
aa_repo_match=matcher)
def install_drivers(cfg, pkg_install_func):
if not isinstance(cfg, dict):
raise TypeError(
"'drivers' config expected dict, found '%s': %s" %
(type_utils.obj_name(cfg), cfg))
cfgpath = 'nvidia/license-accepted'
# Call translate_bool to ensure that we treat string values like "yes" as
# acceptance and _don't_ treat string values like "nah" as acceptance
# because they're True-ish
nv_acc = util.translate_bool(util.get_cfg_by_path(cfg, cfgpath))
if not nv_acc:
LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
return
if not util.which('ubuntu-drivers'):
LOG.debug("'ubuntu-drivers' command not available. "
"Installing ubuntu-drivers-common")
pkg_install_func(['ubuntu-drivers-common'])
driver_arg = 'nvidia'
version_cfg = util.get_cfg_by_path(cfg, 'nvidia/version')
if version_cfg:
driver_arg += ':{}'.format(version_cfg)
LOG.debug("Installing and activating NVIDIA drivers (%s=%s, version=%s)",
cfgpath, nv_acc, version_cfg if version_cfg else 'latest')
# Register and set debconf selection linux/nvidia/latelink = true
tdir = temp_utils.mkdtemp(needs_exe=True)
debconf_file = os.path.join(tdir, 'nvidia.template')
debconf_script = os.path.join(tdir, 'nvidia-debconf.sh')
try:
util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT)
util.write_file(
debconf_script,
util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT),
mode=0o755)
util.subp([debconf_script, debconf_file])
except Exception as e:
util.logexc(
LOG, "Failed to register NVIDIA debconf template: %s", str(e))
raise
finally:
if os.path.isdir(tdir):
util.del_dir(tdir)
try:
util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
except util.ProcessExecutionError as exc:
if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr:
LOG.warning('the available version of ubuntu-drivers is'
' too old to perform requested driver installation')
elif 'No drivers found for installation.' in exc.stdout:
LOG.warning('ubuntu-drivers found no drivers for installation')
raise
def can_dev_be_reformatted(devpath):
# determine if the ephemeral block device path devpath
# is newly formatted after a resize.
if not os.path.exists(devpath):
return False, 'device %s does not exist' % devpath
realpath = os.path.realpath(devpath)
LOG.debug('Resolving realpath of %s -> %s', devpath, realpath)
# it is possible that the block device might exist, but the kernel
# have not yet read the partition table and sent events. we udevadm settle
# to hope to resolve that. Better here would probably be to test and see,
# and then settle if we didn't find anything and try again.
if util.which("udevadm"):
util.subp(["udevadm", "settle"])
# devpath of /dev/sd[a-z] or /dev/disk/cloud/azure_resource
# where partitions are "<devpath>1" or "<devpath>-part1" or "<devpath>p1"
part1path = None
for suff in ("-part", "p", ""):
cand = devpath + suff + "1"
if os.path.exists(cand):
if os.path.exists(devpath + suff + "2"):
msg = ('device %s had more than 1 partition: %s, %s' % devpath,
cand, devpath + suff + "2")
return False, msg
part1path = cand
break
if part1path is None:
return False, 'device %s was not partitioned' % devpath
real_part1path = os.path.realpath(part1path)
ntfs_devices = util.find_devs_with("TYPE=ntfs", no_cache=True)
LOG.debug('ntfs_devices found = %s', ntfs_devices)
if real_part1path not in ntfs_devices:
msg = ('partition 1 (%s -> %s) on device %s was not ntfs formatted' %
(part1path, real_part1path, devpath))
return False, msg
def count_files(mp):
ignored = {'dataloss_warning_readme.txt'}
return len([f for f in os.listdir(mp) if f.lower() not in ignored])
bmsg = ('partition 1 (%s -> %s) on device %s was ntfs formatted' %
(part1path, real_part1path, devpath))
try:
file_count = util.mount_cb(part1path, count_files)
except util.MountFailedError as e:
return False, bmsg + ' but mount of %s failed: %s' % (part1path, e)
if file_count != 0:
return False, bmsg + ' but had %d files on it.' % file_count
return True, bmsg + ' and had no important files. Safe for reformatting.'
def available(target=None):
expected = ['ifquery', 'ifup', 'ifdown']
search = ['/sbin', '/usr/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
eni = util.target_path(target, 'etc/network/interfaces')
if not os.path.isfile(eni):
return False
return True
def available(target=None):
expected = ['ifquery', 'ifup', 'ifdown']
search = ['/sbin', '/usr/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
eni = util.target_path(target, 'etc/network/interfaces')
if not os.path.isfile(eni):
return False
return True
def netdev_info(empty=""):
devs = {}
if util.which('ip'):
# Try iproute first of all
(ipaddr_out, _err) = util.subp(["ip", "addr", "show"])
devs = _netdev_info_iproute(ipaddr_out)
elif util.which('ifconfig'):
# Fall back to net-tools if iproute2 is not present
(ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1])
devs = _netdev_info_ifconfig(ifcfg_out)
else:
LOG.warning(
"Could not print networks: missing 'ip' and 'ifconfig' commands")
if empty != "":
for (_devname, dev) in devs.items():
for field in dev:
if dev[field] == "":
dev[field] = empty
return devs
def netdev_info(empty=""):
devs = {}
if util.is_NetBSD():
(ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1])
devs = _netdev_info_ifconfig_netbsd(ifcfg_out)
elif util.which('ip'):
# Try iproute first of all
(ipaddr_out, _err) = util.subp(["ip", "addr", "show"])
devs = _netdev_info_iproute(ipaddr_out)
elif util.which('ifconfig'):
# Fall back to net-tools if iproute2 is not present
(ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1])
devs = _netdev_info_ifconfig(ifcfg_out)
else:
LOG.warning(
"Could not print networks: missing 'ip' and 'ifconfig' commands")
if empty == "":
return devs
recurse_types = (dict, tuple, list)
def fill(data, new_val="", empty_vals=("", b"")):
"""Recursively replace 'empty_vals' in data (dict, tuple, list)
with new_val"""
if isinstance(data, dict):
myiter = data.items()
elif isinstance(data, (tuple, list)):
myiter = enumerate(data)
else:
raise TypeError("Unexpected input to fill")
for key, val in myiter:
if val in empty_vals:
data[key] = new_val
elif isinstance(val, recurse_types):
fill(val, new_val)
fill(devs, new_val=empty)
return devs
def can_dev_be_reformatted(devpath):
# determine if the ephemeral block device path devpath
# is newly formatted after a resize.
if not os.path.exists(devpath):
return False, "device %s does not exist" % devpath
realpath = os.path.realpath(devpath)
LOG.debug("Resolving realpath of %s -> %s", devpath, realpath)
# it is possible that the block device might exist, but the kernel
# have not yet read the partition table and sent events. we udevadm settle
# to hope to resolve that. Better here would probably be to test and see,
# and then settle if we didn't find anything and try again.
if util.which("udevadm"):
util.subp(["udevadm", "settle"])
# devpath of /dev/sd[a-z] or /dev/disk/cloud/azure_resource
# where partitions are "<devpath>1" or "<devpath>-part1" or "<devpath>p1"
part1path = None
for suff in ("-part", "p", ""):
cand = devpath + suff + "1"
if os.path.exists(cand):
if os.path.exists(devpath + suff + "2"):
msg = ("device %s had more than 1 partition: %s, %s" % devpath, cand, devpath + suff + "2")
return False, msg
part1path = cand
break
if part1path is None:
return False, "device %s was not partitioned" % devpath
real_part1path = os.path.realpath(part1path)
ntfs_devices = util.find_devs_with("TYPE=ntfs", no_cache=True)
LOG.debug("ntfs_devices found = %s", ntfs_devices)
if real_part1path not in ntfs_devices:
msg = "partition 1 (%s -> %s) on device %s was not ntfs formatted" % (part1path, real_part1path, devpath)
return False, msg
def count_files(mp):
ignored = set(["dataloss_warning_readme.txt"])
return len([f for f in os.listdir(mp) if f.lower() not in ignored])
bmsg = "partition 1 (%s -> %s) on device %s was ntfs formatted" % (part1path, real_part1path, devpath)
try:
file_count = util.mount_cb(part1path, count_files)
except util.MountFailedError as e:
return False, bmsg + " but mount of %s failed: %s" % (part1path, e)
if file_count != 0:
return False, bmsg + " but had %d files on it." % file_count
return True, bmsg + " and had no important files. Safe for reformatting."
def maybe_install_ua_tools(cloud):
"""Install ubuntu-advantage-tools if not present."""
if util.which('ubuntu-advantage'):
return
try:
cloud.distro.update_package_sources()
except Exception as e:
util.logexc(LOG, "Package update failed")
raise
try:
cloud.distro.install_packages(['ubuntu-advantage-tools'])
except Exception as e:
util.logexc(LOG, "Failed to install ubuntu-advantage-tools")
raise
def maybe_install_ua_tools(cloud):
"""Install ubuntu-advantage-tools if not present."""
if util.which('ua'):
return
try:
cloud.distro.update_package_sources()
except Exception:
util.logexc(LOG, "Package update failed")
raise
try:
cloud.distro.install_packages(['ubuntu-advantage-tools'])
except Exception:
util.logexc(LOG, "Failed to install ubuntu-advantage-tools")
raise
def available_sysconfig(target=None):
expected = ['ifup', 'ifdown']
search = ['/sbin', '/usr/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
expected_paths = [
'etc/sysconfig/network-scripts/network-functions',
'etc/sysconfig/network-scripts/ifdown-eth']
for p in expected_paths:
if not os.path.isfile(util.target_path(target, p)):
return False
return True
def available_sysconfig(target=None):
expected = ['ifup', 'ifdown']
search = ['/sbin', '/usr/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
expected_paths = [
'etc/sysconfig/network-scripts/network-functions',
'etc/sysconfig/config']
for p in expected_paths:
if os.path.isfile(util.target_path(target, p)):
return True
return False
def netdev_info(empty=""):
devs = {}
if util.which('ip'):
# Try iproute first of all
(ipaddr_out, _err) = util.subp(["ip", "addr", "show"])
devs = _netdev_info_iproute(ipaddr_out)
elif util.which('ifconfig'):
# Fall back to net-tools if iproute2 is not present
(ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1])
devs = _netdev_info_ifconfig(ifcfg_out)
else:
LOG.warning(
"Could not print networks: missing 'ip' and 'ifconfig' commands")
if empty == "":
return devs
recurse_types = (dict, tuple, list)
def fill(data, new_val="", empty_vals=("", b"")):
"""Recursively replace 'empty_vals' in data (dict, tuple, list)
with new_val"""
if isinstance(data, dict):
myiter = data.items()
elif isinstance(data, (tuple, list)):
myiter = enumerate(data)
else:
raise TypeError("Unexpected input to fill")
for key, val in myiter:
if val in empty_vals:
data[key] = new_val
elif isinstance(val, recurse_types):
fill(val, new_val)
fill(devs, new_val=empty)
return devs
def handle_random_seed_command(command, required, env=None):
if not command and required:
raise ValueError("no command found but required=true")
elif not command:
LOG.debug("no command provided")
return
cmd = command[0]
if not util.which(cmd):
if required:
raise ValueError("command '%s' not found but required=true", cmd)
else:
LOG.debug("command '%s' not found for seed_command", cmd)
return
util.subp(command, env=env, capture=False)
def install_ntp_client(install_func, packages=None, check_exe="ntpd"):
"""Install ntp client package if not already installed.
@param install_func: function. This parameter is invoked with the contents
of the packages parameter.
@param packages: list. This parameter defaults to ['ntp'].
@param check_exe: string. The name of a binary that indicates the package
the specified package is already installed.
"""
if util.which(check_exe):
return
if packages is None:
packages = ['ntp']
install_func(packages)
def install_ntp_client(install_func, packages=None, check_exe="ntpd"):
"""Install ntp client package if not already installed.
@param install_func: function. This parameter is invoked with the contents
of the packages parameter.
@param packages: list. This parameter defaults to ['ntp'].
@param check_exe: string. The name of a binary that indicates the package
the specified package is already installed.
"""
if util.which(check_exe):
return
if packages is None:
packages = ['ntp']
install_func(packages)
def transport_vmware_guestinfo():
rpctool = "vmware-rpctool"
not_found = None
if not util.which(rpctool):
return not_found
cmd = [rpctool, "info-get guestinfo.ovfEnv"]
try:
out, _err = util.subp(cmd)
if out:
return out
LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out)
except util.ProcessExecutionError as e:
if e.exit_code != 1:
LOG.warning("%s exited with code %d", rpctool, e.exit_code)
LOG.debug(e)
return not_found
def transport_vmware_guestinfo():
rpctool = "vmware-rpctool"
not_found = None
if not util.which(rpctool):
return not_found
cmd = [rpctool, "info-get guestinfo.ovfEnv"]
try:
out, _err = util.subp(cmd)
if out:
return out
LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out)
except util.ProcessExecutionError as e:
if e.exit_code != 1:
LOG.warning("%s exited with code %d", rpctool, e.exit_code)
LOG.debug(e)
return not_found
def lock_passwd(self, name):
"""
Lock the password of a user, i.e., disable password logins
"""
# passwd must use short '-l' due to SLES11 lacking long form '--lock'
lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
try:
cmd = next(l for l in lock_tools if util.which(l[0]))
except StopIteration:
raise RuntimeError(
("Unable to lock user account '%s'. No tools available. "
" Tried: %s.") % (name, [c[0] for c in lock_tools]))
try:
util.subp(cmd)
except Exception as e:
util.logexc(LOG, 'Failed to disable password for user %s', name)
raise e
def lock_passwd(self, name):
"""
Lock the password of a user, i.e., disable password logins
"""
# passwd must use short '-l' due to SLES11 lacking long form '--lock'
lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
try:
cmd = next(l for l in lock_tools if util.which(l[0]))
except StopIteration:
raise RuntimeError((
"Unable to lock user account '%s'. No tools available. "
" Tried: %s.") % (name, [c[0] for c in lock_tools]))
try:
util.subp(cmd)
except Exception as e:
util.logexc(LOG, 'Failed to disable password for user %s', name)
raise e
def console_log_method(self):
if self._console_log_method is not None:
return self._console_log_method
client = which('lxc')
if not client:
raise PlatformError("No 'lxc' client.")
elif _has_proper_console_support():
self._console_log_method = 'show-log'
elif client.startswith("/snap"):
self._console_log_method = 'logfile-snap'
else:
self._console_log_method = 'logfile-tmp'
LOG.debug("Set console log method to %s", self._console_log_method)
return self._console_log_method
def available(target=None):
expected = ['ifup', 'ifdown']
search = ['/sbin', '/usr/sbin']
for p in expected:
if not util.which(p, search=search, target=target):
return False
expected_paths = [
'etc/net/scripts/functions', 'etc/net/scripts/functions-eth',
'etc/net/scripts/functions-ip', 'etc/net/scripts/functions-ipv4',
'etc/net/scripts/functions-ipv6', 'etc/net/scripts/functions-vlan',
'etc/net/scripts/ifdown'
]
for p in expected_paths:
if not os.path.isfile(util.target_path(target, p)):
return False
return True
def console_log_method(self):
if self._console_log_method is not None:
return self._console_log_method
client = which('lxc')
if not client:
raise PlatformError("No 'lxc' client.")
elif _has_proper_console_support():
self._console_log_method = 'show-log'
elif client.startswith("/snap"):
self._console_log_method = 'logfile-snap'
else:
self._console_log_method = 'logfile-tmp'
LOG.debug("Set console log method to %s", self._console_log_method)
return self._console_log_method
def handle(name, cfg, cloud, log, args):
cfgin = cfg.get('fan')
if not cfgin:
cfgin = {}
mycfg = util.mergemanydict([cfgin, BUILTIN_CFG])
if not mycfg.get('config'):
LOG.debug("%s: no 'fan' config entry. disabling", name)
return
util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w")
distro = cloud.distro
if not util.which('fanctl'):
distro.install_packages(['ubuntu-fan'])
stop_update_start(
service='ubuntu-fan', config_file=mycfg.get('config_path'),
content=mycfg.get('config'), systemd=distro.uses_systemd())
def handle(name, cfg, cloud, log, args):
cfgin = cfg.get('fan')
if not cfgin:
cfgin = {}
mycfg = util.mergemanydict([cfgin, BUILTIN_CFG])
if not mycfg.get('config'):
LOG.debug("%s: no 'fan' config entry. disabling", name)
return
util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w")
distro = cloud.distro
if not util.which('fanctl'):
distro.install_packages(['ubuntu-fan'])
stop_update_start(
service='ubuntu-fan', config_file=mycfg.get('config_path'),
content=mycfg.get('config'), systemd=distro.uses_systemd())
class TestParseTimestamp(CiTestCase):
def test_parse_timestamp_handles_cloud_init_default_format(self):
"""Logs with cloud-init detailed formats will be properly parsed."""
trusty_fmt = '%Y-%m-%d %H:%M:%S,%f'
trusty_stamp = '2016-09-12 14:39:20,839'
dt = datetime.strptime(trusty_stamp, trusty_fmt)
self.assertEqual(float(dt.strftime('%s.%f')),
parse_timestamp(trusty_stamp))
def test_parse_timestamp_handles_syslog_adding_year(self):
"""Syslog timestamps lack a year. Add year and properly parse."""
syslog_fmt = '%b %d %H:%M:%S %Y'
syslog_stamp = 'Aug 08 15:12:51'
# convert stamp ourselves by adding the missing year value
year = datetime.now().year
dt = datetime.strptime(syslog_stamp + " " + str(year), syslog_fmt)
self.assertEqual(float(dt.strftime('%s.%f')),
parse_timestamp(syslog_stamp))
def test_parse_timestamp_handles_journalctl_format_adding_year(self):
"""Journalctl precise timestamps lack a year. Add year and parse."""
journal_fmt = '%b %d %H:%M:%S.%f %Y'
journal_stamp = 'Aug 08 17:15:50.606811'
# convert stamp ourselves by adding the missing year value
year = datetime.now().year
dt = datetime.strptime(journal_stamp + " " + str(year), journal_fmt)
self.assertEqual(float(dt.strftime('%s.%f')),
parse_timestamp(journal_stamp))
@skipIf(not which("date"), "'date' command not available.")
def test_parse_unexpected_timestamp_format_with_date_command(self):
"""Dump sends unexpected timestamp formats to date for processing."""
new_fmt = '%H:%M %m/%d %Y'
new_stamp = '17:15 08/08'
# convert stamp ourselves by adding the missing year value
year = datetime.now().year
dt = datetime.strptime(new_stamp + " " + str(year), new_fmt)
# use date(1)
with self.allow_subp(["date"]):
self.assertEqual(float(dt.strftime('%s.%f')),
parse_timestamp(new_stamp))
def select_ntp_client(ntp_client, distro):
"""Determine which ntp client is to be used, consulting the distro
for its preference.
@param ntp_client: String name of the ntp client to use.
@param distro: Distro class instance.
@returns: Dict of the selected ntp client or {} if none selected.
"""
# construct distro-specific ntp_client_config dict
distro_cfg = distro_ntp_client_configs(distro.name)
# user specified client, return its config
if ntp_client and ntp_client != 'auto':
LOG.debug('Selected NTP client "%s" via user-data configuration',
ntp_client)
return distro_cfg.get(ntp_client, {})
# default to auto if unset in distro
distro_ntp_client = distro.get_option('ntp_client', 'auto')
clientcfg = {}
if distro_ntp_client == "auto":
for client in distro.preferred_ntp_clients:
cfg = distro_cfg.get(client)
if util.which(cfg.get('check_exe')):
LOG.debug('Selected NTP client "%s", already installed',
client)
clientcfg = cfg
break
if not clientcfg:
client = distro.preferred_ntp_clients[0]
LOG.debug(
'Selected distro preferred NTP client "%s", not yet installed',
client)
clientcfg = distro_cfg.get(client)
else:
LOG.debug('Selected NTP client "%s" via distro system config',
distro_ntp_client)
clientcfg = distro_cfg.get(distro_ntp_client, {})
return clientcfg
def dmi_data():
sys_uuid, sys_type = None, None
dmidecode_path = util.which('dmidecode')
if not dmidecode_path:
return False
sys_uuid_cmd = [dmidecode_path, "-s", "system-uuid"]
try:
LOG.debug("Getting hostname from dmidecode")
(sys_uuid, _err) = util.subp(sys_uuid_cmd)
except Exception as e:
util.logexc(LOG, "Failed to get system UUID", e)
sys_type_cmd = [dmidecode_path, "-s", "system-product-name"]
try:
LOG.debug("Determining hypervisor product name via dmidecode")
(sys_type, _err) = util.subp(sys_type_cmd)
except Exception as e:
util.logexc(LOG, "Failed to get system UUID", e)
return (sys_uuid.lower().strip(), sys_type.strip())
def assign_ipv4_link_local(nic=None):
"""Bring up NIC using an address using link-local (ip4LL) IPs. On
DigitalOcean, the link-local domain is per-droplet routed, so there
is no risk of collisions. However, to be more safe, the ip4LL
address is random.
"""
if not nic:
for cdev in sorted(cloudnet.get_devicelist()):
if cloudnet.is_physical(cdev):
nic = cdev
LOG.debug("assigned nic '%s' for link-local discovery", nic)
break
if not nic:
raise RuntimeError("unable to find interfaces to access the"
"meta-data server. This droplet is broken.")
addr = "169.254.{0}.{1}/16".format(random.randint(1, 168),
random.randint(0, 255))
ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic]
ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up']
if not util.which('ip'):
raise RuntimeError("No 'ip' command available to configure ip4LL "
"address")
try:
(result, _err) = util.subp(ip_addr_cmd)
LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic)
(result, _err) = util.subp(ip_link_cmd)
LOG.debug("brought device '%s' up", nic)
except Exception:
util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed."
" Droplet networking will be broken", addr, nic)
raise
return nic
def is_running_in_cloudsigma(self):
"""
Uses dmidecode to detect if this instance of cloud-init is running
in the CloudSigma's infrastructure.
"""
uname_arch = os.uname()[4]
if uname_arch.startswith("arm") or uname_arch == "aarch64":
# Disabling because dmidecode in CMD_DMI_SYSTEM crashes kvm process
LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)")
return False
dmidecode_path = util.which('dmidecode')
if not dmidecode_path:
return False
LOG.debug("Determining hypervisor product name via dmidecode")
try:
cmd = [dmidecode_path, "--string", "system-product-name"]
system_product_name, _ = util.subp(cmd)
return 'cloudsigma' in system_product_name.lower()
except:
LOG.warn("Failed to get hypervisor product name via dmidecode")
return False
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from cloudinit.settings import PER_INSTANCE
from cloudinit import util
import logging
import os
import shlex
frequency = PER_INSTANCE
# Define the commands to use
UDEVADM_CMD = util.which('udevadm')
SFDISK_CMD = util.which("sfdisk")
SGDISK_CMD = util.which("sgdisk")
LSBLK_CMD = util.which("lsblk")
BLKID_CMD = util.which("blkid")
BLKDEV_CMD = util.which("blockdev")
WIPEFS_CMD = util.which("wipefs")
LOG = logging.getLogger(__name__)
def handle(_name, cfg, cloud, log, _args):
"""
See doc/examples/cloud-config_disk-setup.txt for documentation on the
format.
"""
def mkfs(fs_cfg):
"""
Create a file system on the device.
label: defines the label to use on the device
fs_cfg: defines how the filesystem is to look
The following values are required generally:
device: which device or cloud defined default_device
filesystem: which file system type
overwrite: indiscriminately create the file system
partition: when device does not define a partition,
setting this to a number will mean
device + partition. When set to 'auto', the
first free device or the first device which
matches both label and type will be used.
'any' means the first filesystem that matches
on the device.
When 'cmd' is provided then no other parameter is required.
"""
label = fs_cfg.get('label')
device = fs_cfg.get('device')
partition = str(fs_cfg.get('partition', 'any'))
fs_type = fs_cfg.get('filesystem')
fs_cmd = fs_cfg.get('cmd', [])
fs_opts = fs_cfg.get('extra_opts', [])
fs_replace = fs_cfg.get('replace_fs', False)
overwrite = fs_cfg.get('overwrite', False)
# ensure that we get a real device rather than a symbolic link
device = os.path.realpath(device)
# This allows you to define the default ephemeral or swap
LOG.debug("Checking %s against default devices", device)
if not partition or partition.isdigit():
# Handle manual definition of partition
if partition.isdigit():
device = "%s%s" % (device, partition)
LOG.debug("Manual request of partition %s for %s",
partition, device)
# Check to see if the fs already exists
LOG.debug("Checking device %s", device)
check_label, check_fstype, _ = check_fs(device)
LOG.debug("Device %s has %s %s", device, check_label, check_fstype)
if check_label == label and check_fstype == fs_type:
LOG.debug("Existing file system found at %s", device)
if not overwrite:
LOG.debug("Device %s has required file system", device)
return
else:
LOG.warn("Destroying filesystem on %s", device)
else:
LOG.debug("Device %s is cleared for formating", device)
elif partition and str(partition).lower() in ('auto', 'any'):
# For auto devices, we match if the filesystem does exist
odevice = device
LOG.debug("Identifying device to create %s filesytem on", label)
# any mean pick the first match on the device with matching fs_type
label_match = True
if partition.lower() == 'any':
label_match = False
device, reuse = find_device_node(device, fs_type=fs_type, label=label,
label_match=label_match,
replace_fs=fs_replace)
LOG.debug("Automatic device for %s identified as %s", odevice, device)
if reuse:
LOG.debug("Found filesystem match, skipping formating.")
return
if not reuse and fs_replace and device:
LOG.debug("Replacing file system on %s as instructed." % device)
if not device:
LOG.debug("No device aviable that matches request. "
"Skipping fs creation for %s", fs_cfg)
return
elif not partition or str(partition).lower() == 'none':
LOG.debug("Using the raw device to place filesystem %s on" % label)
else:
LOG.debug("Error in device identification handling.")
return
LOG.debug("File system %s will be created on %s", label, device)
# Make sure the device is defined
if not device:
LOG.warn("Device is not known: %s", device)
return
# Check that we can create the FS
if not (fs_type or fs_cmd):
raise Exception("No way to create filesystem '%s'. fs_type or fs_cmd "
"must be set.", label)
# Create the commands
if fs_cmd:
fs_cmd = fs_cfg['cmd'] % {
'label': label,
'filesystem': fs_type,
'device': device,
}
else:
# Find the mkfs command
mkfs_cmd = util.which("mkfs.%s" % fs_type)
if not mkfs_cmd:
mkfs_cmd = util.which("mk%s" % fs_type)
if not mkfs_cmd:
LOG.warn("Cannot create fstype '%s'. No mkfs.%s command", fs_type,
fs_type)
return
fs_cmd = [mkfs_cmd, device]
if label:
fs_cmd.extend(["-L", label])
# File systems that support the -F flag
if overwrite or device_type(device) == "disk":
fs_cmd.append(lookup_force_flag(fs_type))
# Add the extends FS options
if fs_opts:
fs_cmd.extend(fs_opts)
LOG.debug("Creating file system %s on %s", label, device)
LOG.debug(" Using cmd: %s", " ".join(fs_cmd))
try:
util.subp(fs_cmd)
except Exception as e:
raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e))
def available(self):
if not util.which('gpart'):
return False
return True
import json
import six
import sys
import yaml
from cloudinit import importer, util
from cloudinit.tests import helpers
try:
from unittest import mock
except ImportError:
import mock
BASH = util.which('bash')
BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name'
class FakeSelinux(object):
def __init__(self, match_what):
self.match_what = match_what
self.restored = []
def matchpathcon(self, path, mode):
if path == self.match_what:
return
else:
raise OSError("No match!")