def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if files is None:
files = _get_klibc_net_cfg_files()
if 'network-config=' in cmdline:
data64 = None
for tok in cmdline.split():
if tok.startswith("network-config="):
data64 = tok.split("=", 1)[1]
if data64:
return util.load_yaml(_b64dgz(data64))
if not _is_initramfs_netconfig(files, cmdline):
return None
if mac_addrs is None:
mac_addrs = {}
for k in get_devicelist():
mac_addr = read_sys_net_safe(k, 'address')
if mac_addr:
mac_addrs[k] = mac_addr
return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
def _is_cloudinit_disabled(disable_file, paths):
"""Report whether cloud-init is disabled.
@param disable_file: The path to the cloud-init disable file.
@param paths: An initialized cloudinit.helpers.Paths object.
@returns: A tuple containing (bool, reason) about cloud-init's status and
why.
"""
is_disabled = False
cmdline_parts = get_cmdline().split()
if not uses_systemd():
reason = 'Cloud-init enabled on sysvinit'
elif 'cloud-init=enabled' in cmdline_parts:
reason = 'Cloud-init enabled by kernel command line cloud-init=enabled'
elif os.path.exists(disable_file):
is_disabled = True
reason = 'Cloud-init disabled by {0}'.format(disable_file)
elif 'cloud-init=disabled' in cmdline_parts:
is_disabled = True
reason = 'Cloud-init disabled by kernel parameter cloud-init=disabled'
elif not os.path.exists(os.path.join(paths.run_dir, 'enabled')):
is_disabled = True
reason = 'Cloud-init disabled by cloud-init-generator'
else:
reason = 'Cloud-init enabled by systemd cloud-init-generator'
return (is_disabled, reason)
def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if files is None:
files = _get_klibc_net_cfg_files()
if 'network-config=' in cmdline:
data64 = None
for tok in cmdline.split():
if tok.startswith("network-config="):
data64 = tok.split("=", 1)[1]
if data64:
return util.load_yaml(_b64dgz(data64))
if not _is_initramfs_netconfig(files, cmdline):
return None
if mac_addrs is None:
mac_addrs = {}
for k in get_devicelist():
mac_addr = read_sys_net_safe(k, 'address')
if mac_addr:
mac_addrs[k] = mac_addr
return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
def _is_cloudinit_disabled(disable_file, paths):
"""Report whether cloud-init is disabled.
@param disable_file: The path to the cloud-init disable file.
@param paths: An initialized cloudinit.helpers.Paths object.
@returns: A tuple containing (bool, reason) about cloud-init's status and
why.
"""
is_disabled = False
cmdline_parts = get_cmdline().split()
if not uses_systemd():
reason = 'Cloud-init enabled on sysvinit'
elif 'cloud-init=enabled' in cmdline_parts:
reason = 'Cloud-init enabled by kernel command line cloud-init=enabled'
elif os.path.exists(disable_file):
is_disabled = True
reason = 'Cloud-init disabled by {0}'.format(disable_file)
elif 'cloud-init=disabled' in cmdline_parts:
is_disabled = True
reason = 'Cloud-init disabled by kernel parameter cloud-init=disabled'
elif not os.path.exists(os.path.join(paths.run_dir, 'enabled')):
is_disabled = True
reason = 'Cloud-init disabled by cloud-init-generator'
else:
reason = 'Cloud-init enabled by systemd cloud-init-generator'
return (is_disabled, reason)
def maybe_get_writable_device_path(devpath, info, log):
"""Return updated devpath if the devpath is a writable block device.
@param devpath: Requested path to the root device we want to resize.
@param info: String representing information about the requested device.
@param log: Logger to which logs will be added upon error.
@returns devpath or updated devpath per kernel commandline if the device
path is a writable block device, returns None otherwise.
"""
container = util.is_container()
# Ensure the path is a block device.
if (devpath == "/dev/root" and not os.path.exists(devpath)
and not container):
devpath = util.rootdev_from_cmdline(util.get_cmdline())
if devpath is None:
log.warn("Unable to find device '/dev/root'")
return None
log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath)
if devpath == 'overlayroot':
log.debug("Not attempting to resize devpath '%s': %s", devpath, info)
return None
# FreeBSD zpool can also just use gpt/<label>
# with that in mind we can not do an os.stat on "gpt/whatever"
# therefore return the devpath already here.
if devpath.startswith('gpt/'):
log.debug('We have a gpt label - just go ahead')
return devpath
try:
statret = os.stat(devpath)
except OSError as exc:
if container and exc.errno == errno.ENOENT:
log.debug(
"Device '%s' did not exist in container. "
"cannot resize: %s", devpath, info)
elif exc.errno == errno.ENOENT:
log.warn("Device '%s' did not exist. cannot resize: %s", devpath,
info)
else:
raise exc
return None
if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
if container:
log.debug("device '%s' not a block device in container."
" cannot resize: %s" % (devpath, info))
else:
log.warn("device '%s' not a block device. cannot resize: %s" %
(devpath, info))
return None
return devpath # The writable block devpath
def is_vultr():
# VC2, VDC, and HFC use DMI
sysinfo = get_sysinfo()
if sysinfo["manufacturer"] == "Vultr":
return True
# Baremetal requires a kernel parameter
if "vultr" in util.get_cmdline().split():
return True
return False
def read_kernel_cmdline_config(cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if 'network-config=' in cmdline:
data64 = None
for tok in cmdline.split():
if tok.startswith("network-config="):
data64 = tok.split("=", 1)[1]
if data64:
return util.load_yaml(_b64dgz(data64))
return None
def read_kernel_cmdline_config(cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if 'network-config=' in cmdline:
data64 = None
for tok in cmdline.split():
if tok.startswith("network-config="):
data64 = tok.split("=", 1)[1]
if data64:
if data64 == KERNEL_CMDLINE_NETWORK_CONFIG_DISABLED:
return {"config": "disabled"}
return util.load_yaml(_b64dgz(data64))
return None
def devent2dev(devent):
if devent.startswith("/dev/"):
return devent
else:
result = util.get_mount_info(devent)
if not result:
raise ValueError("Could not determine device of '%s' % dev_ent")
dev = result[0]
container = util.is_container()
# Ensure the path is a block device.
if (dev == "/dev/root" and not os.path.exists(dev) and not container):
dev = util.rootdev_from_cmdline(util.get_cmdline())
if dev is None:
raise ValueError("Unable to find device '/dev/root'")
return dev
def __init__(self, _files=None, _mac_addrs=None, _cmdline=None):
self._files = _files
self._mac_addrs = _mac_addrs
self._cmdline = _cmdline
# Set defaults here, as they require computation that we don't want to
# do at method definition time
if self._files is None:
self._files = _get_klibc_net_cfg_files()
if self._cmdline is None:
self._cmdline = util.get_cmdline()
if self._mac_addrs is None:
self._mac_addrs = {}
for k in get_devicelist():
mac_addr = read_sys_net_safe(k, 'address')
if mac_addr:
self._mac_addrs[k] = mac_addr
def read_initramfs_config(files=None, mac_addrs=None, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if files is None:
files = _get_klibc_net_cfg_files()
if not _is_initramfs_netconfig(files, cmdline):
return None
if mac_addrs is None:
mac_addrs = {}
for k in get_devicelist():
mac_addr = read_sys_net_safe(k, 'address')
if mac_addr:
mac_addrs[k] = mac_addr
return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if 'network-config=' in cmdline:
data64 = None
for tok in cmdline.split():
if tok.startswith("network-config="):
data64 = tok.split("=", 1)[1]
if data64:
return util.load_yaml(_b64dgz(data64))
if 'ip=' not in cmdline:
return None
if mac_addrs is None:
mac_addrs = {k: sys_netdev_info(k, 'address')
for k in get_devicelist()}
return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
if 'network-config=' in cmdline:
data64 = None
for tok in cmdline.split():
if tok.startswith("network-config="):
data64 = tok.split("=", 1)[1]
if data64:
return util.load_yaml(_b64dgz(data64))
if 'ip=' not in cmdline and 'ip6=' not in cmdline:
return None
if mac_addrs is None:
mac_addrs = dict(
(k, sys_netdev_info(k, 'address')) for k in get_devicelist())
return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
def devent2dev(devent):
if devent.startswith("/dev/"):
return devent
else:
result = util.get_mount_info(devent)
if not result:
raise ValueError("Could not determine device of '%s' % dev_ent")
dev = result[0]
container = util.is_container()
# Ensure the path is a block device.
if (dev == "/dev/root" and not container):
dev = util.rootdev_from_cmdline(util.get_cmdline())
if dev is None:
if os.path.exists(dev):
# if /dev/root exists, but we failed to convert
# that to a "real" /dev/ path device, then return it.
return dev
raise ValueError("Unable to find device '/dev/root'")
return dev
def on_scaleway():
"""
There are three ways to detect if you are on Scaleway:
* check DMI data: not yet implemented by Scaleway, but the check is made to
be future-proof.
* the initrd created the file /var/run/scaleway.
* "scaleway" is in the kernel cmdline.
"""
vendor_name = util.read_dmi_data('system-manufacturer')
if vendor_name == 'Scaleway':
return True
if os.path.exists('/var/run/scaleway'):
return True
cmdline = util.get_cmdline()
if 'scaleway' in cmdline:
return True
return False
def devent2dev(devent):
if devent.startswith("/dev/"):
return devent
else:
result = util.get_mount_info(devent)
if not result:
raise ValueError("Could not determine device of '%s' % dev_ent")
dev = result[0]
container = util.is_container()
# Ensure the path is a block device.
if dev == "/dev/root" and not container:
dev = util.rootdev_from_cmdline(util.get_cmdline())
if dev is None:
if os.path.exists(dev):
# if /dev/root exists, but we failed to convert
# that to a "real" /dev/ path device, then return it.
return dev
raise ValueError("Unable to find device '/dev/root'")
return dev
def on_scaleway():
"""
There are three ways to detect if you are on Scaleway:
* check DMI data: not yet implemented by Scaleway, but the check is made to
be future-proof.
* the initrd created the file /var/run/scaleway.
* "scaleway" is in the kernel cmdline.
"""
vendor_name = dmi.read_dmi_data('system-manufacturer')
if vendor_name == 'Scaleway':
return True
if os.path.exists('/var/run/scaleway'):
return True
cmdline = util.get_cmdline()
if 'scaleway' in cmdline:
return True
return False
def get_cmdline_url(names=('cloud-config-url', 'url'),
starts="#cloud-config", cmdline=None):
if cmdline == None:
cmdline = util.get_cmdline()
data = util.keyval_str_to_dict(cmdline)
url = None
key = None
for key in names:
if key in data:
url = data[key]
break
if url == None:
return (None, None, None)
contents = util.readurl(url)
if contents.startswith(starts):
return (key, url, contents)
return (key, url, None)
def parse_cmdline_data(ds_id, fill, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
cmdline = " %s " % cmdline
if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline):
return False
argline = ""
# cmdline can contain:
# ds=nocloud[;key=val;key=val]
for tok in cmdline.split():
if tok.startswith(ds_id):
argline = tok.split("=", 1)
# argline array is now 'nocloud' followed optionally by
# a ';' and then key=value pairs also terminated with ';'
tmp = argline[1].split(";")
if len(tmp) > 1:
kvpairs = tmp[1:]
else:
kvpairs = ()
# short2long mapping to save cmdline typing
s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"}
for item in kvpairs:
if item == "":
continue
try:
(k, v) = item.split("=", 1)
except Exception:
k = item
v = None
if k in s2l:
k = s2l[k]
fill[k] = v
return True
def parse_cmdline_data(ds_id, fill, cmdline=None):
if cmdline is None:
cmdline = util.get_cmdline()
cmdline = " %s " % cmdline
if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline):
return False
argline = ""
# cmdline can contain:
# ds=nocloud[;key=val;key=val]
for tok in cmdline.split():
if tok.startswith(ds_id):
argline = tok.split("=", 1)
# argline array is now 'nocloud' followed optionally by
# a ';' and then key=value pairs also terminated with ';'
tmp = argline[1].split(";")
if len(tmp) > 1:
kvpairs = tmp[1:]
else:
kvpairs = ()
# short2long mapping to save cmdline typing
s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"}
for item in kvpairs:
if item == "":
continue
try:
(k, v) = item.split("=", 1)
except:
k = item
v = None
if k in s2l:
k = s2l[k]
fill[k] = v
return True
def get_cmdline_url(
names=('cloud-config-url', 'url'), starts="#cloud-config",
cmdline=None):
if cmdline == None:
cmdline = util.get_cmdline()
data = util.keyval_str_to_dict(cmdline)
url = None
key = None
for key in names:
if key in data:
url = data[key]
break
if url == None:
return (None, None, None)
contents = util.readurl(url)
if contents.startswith(starts):
return (key, url, contents)
return (key, url, None)
def attempt_cmdline_url(path, network=True, cmdline=None):
"""Write data from url referenced in command line to path.
path: a file to write content to if downloaded.
network: should network access be assumed.
cmdline: the cmdline to parse for cloud-config-url.
This is used in MAAS datasource, in "ephemeral" (read-only root)
environment where the instance netboots to iscsi ro root.
and the entity that controls the pxe config has to configure
the maas datasource.
An attempt is made on network urls even in local datasource
for case of network set up in initramfs.
Return value is a tuple of a logger function (logging.DEBUG)
and a message indicating what happened.
"""
if cmdline is None:
cmdline = util.get_cmdline()
try:
cmdline_name, url = parse_cmdline_url(cmdline)
except KeyError:
return (logging.DEBUG, "No kernel command line url found.")
path_is_local = url.startswith("file://") or url.startswith("/")
if path_is_local and os.path.exists(path):
if network:
m = ("file '%s' existed, possibly from local stage download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.INFO
if path_is_local:
level = logging.DEBUG
else:
m = ("file '%s' existed, possibly from previous boot download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.WARN
return (level, m)
kwargs = {'url': url, 'timeout': 10, 'retries': 2}
if network or path_is_local:
level = logging.WARN
kwargs['sec_between'] = 1
else:
level = logging.DEBUG
kwargs['sec_between'] = .1
data = None
header = b'#cloud-config'
try:
resp = url_helper.read_file_or_url(**kwargs)
if resp.ok():
data = resp.contents
if not resp.contents.startswith(header):
if cmdline_name == 'cloud-config-url':
level = logging.WARN
else:
level = logging.INFO
return (level, "contents of '%s' did not start with %s" %
(url, header))
else:
return (level,
"url '%s' returned code %s. Ignoring." % (url, resp.code))
except url_helper.UrlError as e:
return (level, "retrieving url '%s' failed: %s" % (url, e))
util.write_file(path, data, mode=0o600)
return (logging.INFO, "wrote cloud-config data from %s='%s' to %s" %
(cmdline_name, url, path))
def main():
util.close_stdin()
cmds = ("start", "start-local")
deps = {"start": (ds.DEP_FILESYSTEM, ds.DEP_NETWORK),
"start-local": (ds.DEP_FILESYSTEM, )}
cmd = ""
if len(sys.argv) > 1:
cmd = sys.argv[1]
cfg_path = None
if len(sys.argv) > 2:
# this is really for debugging only
# but you can invoke on development system with ./config/cloud.cfg
cfg_path = sys.argv[2]
if not cmd in cmds:
sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds))
sys.exit(1)
now = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime())
try:
uptimef = open("/proc/uptime")
uptime = uptimef.read().split(" ")[0]
uptimef.close()
except IOError as e:
warn("unable to open /proc/uptime\n")
uptime = "na"
cmdline_msg = None
cmdline_exc = None
if cmd == "start":
target = "%s.d/%s" % (cloudinit.system_config,
"91_kernel_cmdline_url.cfg")
if os.path.exists(target):
cmdline_msg = "cmdline: %s existed" % target
else:
cmdline = util.get_cmdline()
try:
(key, url, content) = cloudinit.get_cmdline_url(
cmdline=cmdline)
if key and content:
util.write_file(target, content, mode=0600)
cmdline_msg = ("cmdline: wrote %s from %s, %s" %
(target, key, url))
elif key:
cmdline_msg = ("cmdline: %s, %s had no cloud-config" %
(key, url))
except Exception:
cmdline_exc = ("cmdline: '%s' raised exception\n%s" %
(cmdline, traceback.format_exc()))
warn(cmdline_exc)
try:
cfg = cloudinit.get_base_cfg(cfg_path)
except Exception as e:
warn("Failed to get base config. falling back to builtin: %s\n" % e)
try:
cfg = cloudinit.get_builtin_cfg()
except Exception as e:
warn("Unable to load builtin config\n")
raise
try:
(outfmt, errfmt) = CC.get_output_cfg(cfg, "init")
CC.redirect_output(outfmt, errfmt)
except Exception as e:
warn("Failed to get and set output config: %s\n" % e)
cloudinit.logging_set_from_cfg(cfg)
log = logging.getLogger()
if cmdline_exc:
log.debug(cmdline_exc)
elif cmdline_msg:
log.debug(cmdline_msg)
try:
cloudinit.initfs()
except Exception as e:
warn("failed to initfs, likely bad things to come: %s\n" % str(e))
nonet_path = "%s/%s" % (cloudinit.get_cpath("data"), "no-net")
if cmd == "start":
print netinfo.debug_info()
stop_files = (cloudinit.get_ipath_cur("obj_pkl"), nonet_path)
# if starting as the network start, there are cases
# where everything is already done for us, and it makes
# most sense to exit early and silently
for f in stop_files:
try:
fp = open(f, "r")
fp.close()
except:
continue
log.debug("no need for cloud-init start to run (%s)\n", f)
sys.exit(0)
elif cmd == "start-local":
# cache is not instance specific, so it has to be purged
# but we want 'start' to benefit from a cache if
# a previous start-local populated one
manclean = util.get_cfg_option_bool(cfg, 'manual_cache_clean', False)
if manclean:
log.debug("not purging cache, manual_cache_clean = True")
cloudinit.purge_cache(not manclean)
try:
os.unlink(nonet_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime)
sys.stderr.write(msg + "\n")
sys.stderr.flush()
log.info(msg)
cloud = cloudinit.CloudInit(ds_deps=deps[cmd])
try:
cloud.get_data_source()
except cloudinit.DataSourceNotFoundException as e:
sys.stderr.write("no instance data found in %s\n" % cmd)
sys.exit(0)
# set this as the current instance
cloud.set_cur_instance()
# store the metadata
cloud.update_cache()
msg = "found data source: %s" % cloud.datasource
sys.stderr.write(msg + "\n")
log.debug(msg)
# parse the user data (ec2-run-userdata.py)
try:
ran = cloud.sem_and_run("consume_userdata", cloudinit.per_instance,
cloud.consume_userdata, [cloudinit.per_instance], False)
if not ran:
cloud.consume_userdata(cloudinit.per_always)
except:
warn("consuming user data failed!\n")
raise
cfg_path = cloudinit.get_ipath_cur("cloud_config")
cc = CC.CloudConfig(cfg_path, cloud)
# if the output config changed, update output and err
try:
outfmt_orig = outfmt
errfmt_orig = errfmt
(outfmt, errfmt) = CC.get_output_cfg(cc.cfg, "init")
if outfmt_orig != outfmt or errfmt_orig != errfmt:
warn("stdout, stderr changing to (%s,%s)" % (outfmt, errfmt))
CC.redirect_output(outfmt, errfmt)
except Exception as e:
warn("Failed to get and set output config: %s\n" % e)
# send the cloud-config ready event
cc_path = cloudinit.get_ipath_cur('cloud_config')
cc_ready = cc.cfg.get("cc_ready_cmd",
['initctl', 'emit', 'cloud-config',
'%s=%s' % (cloudinit.cfg_env_name, cc_path)])
if cc_ready:
if isinstance(cc_ready, str):
cc_ready = ['sh', '-c', cc_ready]
subprocess.Popen(cc_ready).communicate()
module_list = CC.read_cc_modules(cc.cfg, "cloud_init_modules")
failures = []
if len(module_list):
failures = CC.run_cc_modules(cc, module_list, log)
else:
msg = "no cloud_init_modules to run"
sys.stderr.write(msg + "\n")
log.debug(msg)
sys.exit(0)
sys.exit(len(failures))
def find_fallback_nic_on_linux(blacklist_drivers=None):
"""Return the name of the 'fallback' network device on Linux."""
if not blacklist_drivers:
blacklist_drivers = []
if 'net.ifnames=0' in util.get_cmdline():
LOG.debug('Stable ifnames disabled by net.ifnames=0 in /proc/cmdline')
else:
unstable = [
device for device in get_devicelist()
if device != 'lo' and not is_renamed(device)
]
if len(unstable):
LOG.debug('Found unstable nic names: %s; calling udevadm settle',
unstable)
msg = 'Waiting for udev events to settle'
util.log_time(LOG.debug, msg, func=util.udevadm_settle)
# get list of interfaces that could have connections
invalid_interfaces = set(['lo'])
potential_interfaces = set([
device for device in get_devicelist()
if device_driver(device) not in blacklist_drivers
])
potential_interfaces = potential_interfaces.difference(invalid_interfaces)
# sort into interfaces with carrier, interfaces which could have carrier,
# and ignore interfaces that are definitely disconnected
connected = []
possibly_connected = []
for interface in potential_interfaces:
if interface.startswith("veth"):
continue
if is_bridge(interface):
# skip any bridges
continue
if is_bond(interface):
# skip any bonds
continue
if is_netfailover(interface):
# ignore netfailover primary/standby interfaces
continue
carrier = read_sys_net_int(interface, 'carrier')
if carrier:
connected.append(interface)
continue
# check if nic is dormant or down, as this may make a nick appear to
# not have a carrier even though it could acquire one when brought
# online by dhclient
dormant = read_sys_net_int(interface, 'dormant')
if dormant:
possibly_connected.append(interface)
continue
operstate = read_sys_net_safe(interface, 'operstate')
if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']:
possibly_connected.append(interface)
continue
# don't bother with interfaces that might not be connected if there are
# some that definitely are
if connected:
potential_interfaces = connected
else:
potential_interfaces = possibly_connected
# if eth0 exists use it above anything else, otherwise get the interface
# that we can read 'first' (using the sorted definition of first).
names = list(sorted(potential_interfaces, key=natural_sort_key))
if DEFAULT_PRIMARY_INTERFACE in names:
names.remove(DEFAULT_PRIMARY_INTERFACE)
names.insert(0, DEFAULT_PRIMARY_INTERFACE)
# pick the first that has a mac-address
for name in names:
if read_sys_net_safe(name, 'address'):
return name
return None
def test_cmdline_reads_debug_env(self):
os.environ["DEBUG_PROC_CMDLINE"] = "abcd 123"
self.assertEqual(os.environ["DEBUG_PROC_CMDLINE"], util.get_cmdline())
def test_cmdline_reads_debug_env(self):
os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123'
self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], util.get_cmdline())
def handle(name, cfg, _cloud, log, args):
if len(args) != 0:
resize_root = args[0]
else:
resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True)
if not util.translate_bool(resize_root, addons=[NOBLOCK]):
log.debug("Skipping module named %s, resizing disabled", name)
return
# TODO(harlowja) is the directory ok to be used??
resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run")
util.ensure_dir(resize_root_d)
# TODO(harlowja): allow what is to be resized to be configurable??
resize_what = "/"
result = util.get_mount_info(resize_what, log)
if not result:
log.warn("Could not determine filesystem type of %s", resize_what)
return
(devpth, fs_type, mount_point) = result
# Ensure the path is a block device.
info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
log.debug("resize_info: %s" % info)
container = util.is_container()
if (devpth == "/dev/root" and not os.path.exists(devpth) and
not container):
devpth = rootdev_from_cmdline(util.get_cmdline())
if devpth is None:
log.warn("Unable to find device '/dev/root'")
return
log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth)
try:
statret = os.stat(devpth)
except OSError as exc:
if container and exc.errno == errno.ENOENT:
log.debug("Device '%s' did not exist in container. "
"cannot resize: %s" % (devpth, info))
elif exc.errno == errno.ENOENT:
log.warn("Device '%s' did not exist. cannot resize: %s" %
(devpth, info))
else:
raise exc
return
if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
if container:
log.debug("device '%s' not a block device in container."
" cannot resize: %s" % (devpth, info))
else:
log.warn("device '%s' not a block device. cannot resize: %s" %
(devpth, info))
return
resizer = None
fstype_lc = fs_type.lower()
for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS:
if fstype_lc.startswith(pfix):
resizer = root_cmd
break
if not resizer:
log.warn("Not resizing unknown filesystem type %s for %s",
fs_type, resize_what)
return
resize_cmd = resizer(resize_what, devpth)
log.debug("Resizing %s (%s) using %s", resize_what, fs_type,
' '.join(resize_cmd))
if resize_root == NOBLOCK:
# Fork to a child that will run
# the resize command
util.fork_cb(
util.log_time(logfunc=log.debug, msg="backgrounded Resizing",
func=do_resize, args=(resize_cmd, log)))
else:
util.log_time(logfunc=log.debug, msg="Resizing",
func=do_resize, args=(resize_cmd, log))
action = 'Resized'
if resize_root == NOBLOCK:
action = 'Resizing (via forking)'
log.debug("%s root filesystem (type=%s, val=%s)", action, fs_type,
resize_root)
def maybe_get_writable_device_path(devpath, info, log):
"""Return updated devpath if the devpath is a writable block device.
@param devpath: Requested path to the root device we want to resize.
@param info: String representing information about the requested device.
@param log: Logger to which logs will be added upon error.
@returns devpath or updated devpath per kernel commandline if the device
path is a writable block device, returns None otherwise.
"""
container = util.is_container()
# Ensure the path is a block device.
if (devpath == "/dev/root" and not os.path.exists(devpath) and
not container):
devpath = util.rootdev_from_cmdline(util.get_cmdline())
if devpath is None:
log.warn("Unable to find device '/dev/root'")
return None
log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath)
if devpath == 'overlayroot':
log.debug("Not attempting to resize devpath '%s': %s", devpath, info)
return None
# FreeBSD zpool can also just use gpt/<label>
# with that in mind we can not do an os.stat on "gpt/whatever"
# therefore return the devpath already here.
if devpath.startswith('gpt/'):
log.debug('We have a gpt label - just go ahead')
return devpath
# Alternatively, our device could simply be a name as returned by gpart,
# such as da0p3
if not devpath.startswith('/dev/') and not os.path.exists(devpath):
fulldevpath = '/dev/' + devpath.lstrip('/')
log.debug("'%s' doesn't appear to be a valid device path. Trying '%s'",
devpath, fulldevpath)
devpath = fulldevpath
try:
statret = os.stat(devpath)
except OSError as exc:
if container and exc.errno == errno.ENOENT:
log.debug("Device '%s' did not exist in container. "
"cannot resize: %s", devpath, info)
elif exc.errno == errno.ENOENT:
log.warn("Device '%s' did not exist. cannot resize: %s",
devpath, info)
else:
raise exc
return None
if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
if container:
log.debug("device '%s' not a block device in container."
" cannot resize: %s" % (devpath, info))
else:
log.warn("device '%s' not a block device. cannot resize: %s" %
(devpath, info))
return None
return devpath # The writable block devpath
def find_fallback_nic(blacklist_drivers=None):
"""Return the name of the 'fallback' network device."""
if not blacklist_drivers:
blacklist_drivers = []
if 'net.ifnames=0' in util.get_cmdline():
LOG.debug('Stable ifnames disabled by net.ifnames=0 in /proc/cmdline')
else:
unstable = [device for device in get_devicelist()
if device != 'lo' and not is_renamed(device)]
if len(unstable):
LOG.debug('Found unstable nic names: %s; calling udevadm settle',
unstable)
msg = 'Waiting for udev events to settle'
util.log_time(LOG.debug, msg, func=util.udevadm_settle)
# get list of interfaces that could have connections
invalid_interfaces = set(['lo'])
potential_interfaces = set([device for device in get_devicelist()
if device_driver(device) not in
blacklist_drivers])
potential_interfaces = potential_interfaces.difference(invalid_interfaces)
# sort into interfaces with carrier, interfaces which could have carrier,
# and ignore interfaces that are definitely disconnected
connected = []
possibly_connected = []
for interface in potential_interfaces:
if interface.startswith("veth"):
continue
if is_bridge(interface):
# skip any bridges
continue
if is_bond(interface):
# skip any bonds
continue
carrier = read_sys_net_int(interface, 'carrier')
if carrier:
connected.append(interface)
continue
# check if nic is dormant or down, as this may make a nick appear to
# not have a carrier even though it could acquire one when brought
# online by dhclient
dormant = read_sys_net_int(interface, 'dormant')
if dormant:
possibly_connected.append(interface)
continue
operstate = read_sys_net_safe(interface, 'operstate')
if operstate in ['dormant', 'down', 'lowerlayerdown', 'unknown']:
possibly_connected.append(interface)
continue
# don't bother with interfaces that might not be connected if there are
# some that definitely are
if connected:
potential_interfaces = connected
else:
potential_interfaces = possibly_connected
# if eth0 exists use it above anything else, otherwise get the interface
# that we can read 'first' (using the sorted defintion of first).
names = list(sorted(potential_interfaces, key=natural_sort_key))
if DEFAULT_PRIMARY_INTERFACE in names:
names.remove(DEFAULT_PRIMARY_INTERFACE)
names.insert(0, DEFAULT_PRIMARY_INTERFACE)
# pick the first that has a mac-address
for name in names:
if read_sys_net_safe(name, 'address'):
return name
return None
def handle(name, cfg, _cloud, log, args):
if len(args) != 0:
resize_root = args[0]
else:
resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True)
if not util.translate_bool(resize_root, addons=[NOBLOCK]):
log.debug("Skipping module named %s, resizing disabled", name)
return
# TODO(harlowja) is the directory ok to be used??
resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run")
util.ensure_dir(resize_root_d)
# TODO(harlowja): allow what is to be resized to be configurable??
resize_what = "/"
result = util.get_mount_info(resize_what, log)
if not result:
log.warn("Could not determine filesystem type of %s", resize_what)
return
(devpth, fs_type, mount_point) = result
info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
log.debug("resize_info: %s" % info)
container = util.is_container()
# Ensure the path is a block device.
if (devpth == "/dev/root" and not os.path.exists(devpth)
and not container):
devpth = util.rootdev_from_cmdline(util.get_cmdline())
if devpth is None:
log.warn("Unable to find device '/dev/root'")
return
log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth)
try:
statret = os.stat(devpth)
except OSError as exc:
if container and exc.errno == errno.ENOENT:
log.debug(
"Device '%s' did not exist in container. "
"cannot resize: %s", devpth, info)
elif exc.errno == errno.ENOENT:
log.warn("Device '%s' did not exist. cannot resize: %s", devpth,
info)
else:
raise exc
return
if not os.access(devpth, os.W_OK):
if container:
log.debug("'%s' not writable in container. cannot resize: %s",
devpth, info)
else:
log.warn("'%s' not writable. cannot resize: %s", devpth, info)
return
if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
if container:
log.debug("device '%s' not a block device in container."
" cannot resize: %s" % (devpth, info))
else:
log.warn("device '%s' not a block device. cannot resize: %s" %
(devpth, info))
return
resizer = None
if can_skip_resize(fs_type, resize_what, devpth):
log.debug("Skip resize filesystem type %s for %s", fs_type,
resize_what)
return
fstype_lc = fs_type.lower()
for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS:
if fstype_lc.startswith(pfix):
resizer = root_cmd
break
if not resizer:
log.warn("Not resizing unknown filesystem type %s for %s", fs_type,
resize_what)
return
resize_cmd = resizer(resize_what, devpth)
log.debug("Resizing %s (%s) using %s", resize_what, fs_type,
' '.join(resize_cmd))
if resize_root == NOBLOCK:
# Fork to a child that will run
# the resize command
util.fork_cb(util.log_time,
logfunc=log.debug,
msg="backgrounded Resizing",
func=do_resize,
args=(resize_cmd, log))
else:
util.log_time(logfunc=log.debug,
msg="Resizing",
func=do_resize,
args=(resize_cmd, log))
action = 'Resized'
if resize_root == NOBLOCK:
action = 'Resizing (via forking)'
log.debug("%s root filesystem (type=%s, val=%s)", action, fs_type,
resize_root)
def main():
util.close_stdin()
cmds = ("start", "start-local")
deps = {
"start": (ds.DEP_FILESYSTEM, ds.DEP_NETWORK),
"start-local": (ds.DEP_FILESYSTEM, )
}
cmd = ""
if len(sys.argv) > 1:
cmd = sys.argv[1]
cfg_path = None
if len(sys.argv) > 2:
# this is really for debugging only
# but you can invoke on development system with ./config/cloud.cfg
cfg_path = sys.argv[2]
if not cmd in cmds:
sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds))
sys.exit(1)
now = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime())
try:
uptimef = open("/proc/uptime")
uptime = uptimef.read().split(" ")[0]
uptimef.close()
except IOError as e:
warn("unable to open /proc/uptime\n")
uptime = "na"
cmdline_msg = None
cmdline_exc = None
if cmd == "start":
target = "%s.d/%s" % (cloudinit.system_config,
"91_kernel_cmdline_url.cfg")
if os.path.exists(target):
cmdline_msg = "cmdline: %s existed" % target
else:
cmdline = util.get_cmdline()
try:
(key, url,
content) = cloudinit.get_cmdline_url(cmdline=cmdline)
if key and content:
util.write_file(target, content, mode=0600)
cmdline_msg = ("cmdline: wrote %s from %s, %s" %
(target, key, url))
elif key:
cmdline_msg = ("cmdline: %s, %s had no cloud-config" %
(key, url))
except Exception:
cmdline_exc = ("cmdline: '%s' raised exception\n%s" %
(cmdline, traceback.format_exc()))
warn(cmdline_exc)
try:
cfg = cloudinit.get_base_cfg(cfg_path)
except Exception as e:
warn("Failed to get base config. falling back to builtin: %s\n" % e)
try:
cfg = cloudinit.get_builtin_cfg()
except Exception as e:
warn("Unable to load builtin config\n")
raise
try:
(outfmt, errfmt) = CC.get_output_cfg(cfg, "init")
CC.redirect_output(outfmt, errfmt)
except Exception as e:
warn("Failed to get and set output config: %s\n" % e)
cloudinit.logging_set_from_cfg(cfg)
log = logging.getLogger()
if cmdline_exc:
log.debug(cmdline_exc)
elif cmdline_msg:
log.debug(cmdline_msg)
try:
cloudinit.initfs()
except Exception as e:
warn("failed to initfs, likely bad things to come: %s\n" % str(e))
nonet_path = "%s/%s" % (cloudinit.get_cpath("data"), "no-net")
if cmd == "start":
print netinfo.debug_info()
stop_files = (cloudinit.get_ipath_cur("obj_pkl"), nonet_path)
# if starting as the network start, there are cases
# where everything is already done for us, and it makes
# most sense to exit early and silently
for f in stop_files:
try:
fp = open(f, "r")
fp.close()
except:
continue
log.debug("no need for cloud-init start to run (%s)\n", f)
sys.exit(0)
elif cmd == "start-local":
# cache is not instance specific, so it has to be purged
# but we want 'start' to benefit from a cache if
# a previous start-local populated one
manclean = util.get_cfg_option_bool(cfg, 'manual_cache_clean', False)
if manclean:
log.debug("not purging cache, manual_cache_clean = True")
cloudinit.purge_cache(not manclean)
try:
os.unlink(nonet_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime)
sys.stderr.write(msg + "\n")
sys.stderr.flush()
log.info(msg)
cloud = cloudinit.CloudInit(ds_deps=deps[cmd])
try:
cloud.get_data_source()
except cloudinit.DataSourceNotFoundException as e:
sys.stderr.write("no instance data found in %s\n" % cmd)
sys.exit(0)
# set this as the current instance
cloud.set_cur_instance()
# store the metadata
cloud.update_cache()
msg = "found data source: %s" % cloud.datasource
sys.stderr.write(msg + "\n")
log.debug(msg)
# parse the user data (ec2-run-userdata.py)
try:
ran = cloud.sem_and_run("consume_userdata", cloudinit.per_instance,
cloud.consume_userdata,
[cloudinit.per_instance], False)
if not ran:
cloud.consume_userdata(cloudinit.per_always)
except:
warn("consuming user data failed!\n")
raise
cfg_path = cloudinit.get_ipath_cur("cloud_config")
cc = CC.CloudConfig(cfg_path, cloud)
# if the output config changed, update output and err
try:
outfmt_orig = outfmt
errfmt_orig = errfmt
(outfmt, errfmt) = CC.get_output_cfg(cc.cfg, "init")
if outfmt_orig != outfmt or errfmt_orig != errfmt:
warn("stdout, stderr changing to (%s,%s)" % (outfmt, errfmt))
CC.redirect_output(outfmt, errfmt)
except Exception as e:
warn("Failed to get and set output config: %s\n" % e)
# send the cloud-config ready event
cc_path = cloudinit.get_ipath_cur('cloud_config')
cc_ready = cc.cfg.get("cc_ready_cmd", [
'initctl', 'emit', 'cloud-config',
'%s=%s' % (cloudinit.cfg_env_name, cc_path)
])
if cc_ready:
if isinstance(cc_ready, str):
cc_ready = ['sh', '-c', cc_ready]
subprocess.Popen(cc_ready).communicate()
module_list = CC.read_cc_modules(cc.cfg, "cloud_init_modules")
failures = []
if len(module_list):
failures = CC.run_cc_modules(cc, module_list, log)
else:
msg = "no cloud_init_modules to run"
sys.stderr.write(msg + "\n")
log.debug(msg)
sys.exit(0)
sys.exit(len(failures))
def test_cmdline_reads_debug_env(self):
with mock.patch.dict("os.environ",
values={'DEBUG_PROC_CMDLINE': 'abcd 123'}):
ret = util.get_cmdline()
self.assertEqual("abcd 123", ret)
def test_cmdline_reads_debug_env(self):
os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123'
self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], get_cmdline())
def attempt_cmdline_url(path, network=True, cmdline=None):
"""Write data from url referenced in command line to path.
path: a file to write content to if downloaded.
network: should network access be assumed.
cmdline: the cmdline to parse for cloud-config-url.
This is used in MAAS datasource, in "ephemeral" (read-only root)
environment where the instance netboots to iscsi ro root.
and the entity that controls the pxe config has to configure
the maas datasource.
An attempt is made on network urls even in local datasource
for case of network set up in initramfs.
Return value is a tuple of a logger function (logging.DEBUG)
and a message indicating what happened.
"""
if cmdline is None:
cmdline = util.get_cmdline()
try:
cmdline_name, url = parse_cmdline_url(cmdline)
except KeyError:
return (logging.DEBUG, "No kernel command line url found.")
path_is_local = url.startswith("file://") or url.startswith("/")
if path_is_local and os.path.exists(path):
if network:
m = ("file '%s' existed, possibly from local stage download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.INFO
if path_is_local:
level = logging.DEBUG
else:
m = ("file '%s' existed, possibly from previous boot download"
" of command line url '%s'. Not re-writing." % (path, url))
level = logging.WARN
return (level, m)
kwargs = {'url': url, 'timeout': 10, 'retries': 2}
if network or path_is_local:
level = logging.WARN
kwargs['sec_between'] = 1
else:
level = logging.DEBUG
kwargs['sec_between'] = .1
data = None
header = b'#cloud-config'
try:
resp = url_helper.read_file_or_url(**kwargs)
if resp.ok():
data = resp.contents
if not resp.contents.startswith(header):
if cmdline_name == 'cloud-config-url':
level = logging.WARN
else:
level = logging.INFO
return (
level,
"contents of '%s' did not start with %s" % (url, header))
else:
return (level,
"url '%s' returned code %s. Ignoring." % (url, resp.code))
except url_helper.UrlError as e:
return (level, "retrieving url '%s' failed: %s" % (url, e))
util.write_file(path, data, mode=0o600)
return (logging.INFO,
"wrote cloud-config data from %s='%s' to %s" %
(cmdline_name, url, path))
def test_cmdline_reads_debug_env(self):
with mock.patch.dict("os.environ",
values={'DEBUG_PROC_CMDLINE': 'abcd 123'}):
ret = util.get_cmdline()
self.assertEqual("abcd 123", ret)
def find_candidate_nics_on_linux(
blacklist_drivers: Optional[List[str]] = None, ) -> List[str]:
"""Get the names of the candidate network devices on Linux.
@param blacklist_drivers: Filter out NICs with these drivers.
@return List of sorted interfaces.
"""
if not blacklist_drivers:
blacklist_drivers = []
if "net.ifnames=0" in util.get_cmdline():
LOG.debug("Stable ifnames disabled by net.ifnames=0 in /proc/cmdline")
else:
unstable = [
device for device in get_devicelist()
if device != "lo" and not is_renamed(device)
]
if len(unstable):
LOG.debug(
"Found unstable nic names: %s; calling udevadm settle",
unstable,
)
msg = "Waiting for udev events to settle"
util.log_time(LOG.debug, msg, func=util.udevadm_settle)
# sort into interfaces with carrier, interfaces which could have carrier,
# and ignore interfaces that are definitely disconnected
connected = []
possibly_connected = []
for interface in get_devicelist():
if interface == "lo":
continue
driver = device_driver(interface)
if driver in blacklist_drivers:
LOG.debug("Ignoring interface with %s driver: %s", driver,
interface)
continue
if not read_sys_net_safe(interface, "address"):
LOG.debug("Ignoring interface without mac: %s", interface)
continue
if interface.startswith("veth"):
LOG.debug("Ignoring veth interface: %s", interface)
continue
if is_bridge(interface):
LOG.debug("Ignoring bridge interface: %s", interface)
continue
if is_bond(interface):
LOG.debug("Ignoring bond interface: %s", interface)
continue
if is_netfailover(interface):
LOG.debug("Ignoring failover interface: %s", interface)
continue
carrier = read_sys_net_int(interface, "carrier")
if carrier:
connected.append(interface)
continue
LOG.debug("Interface has no carrier: %s", interface)
# check if nic is dormant or down, as this may make a nick appear to
# not have a carrier even though it could acquire one when brought
# online by dhclient
dormant = read_sys_net_int(interface, "dormant")
if dormant:
possibly_connected.append(interface)
continue
operstate = read_sys_net_safe(interface, "operstate")
if operstate in ["dormant", "down", "lowerlayerdown", "unknown"]:
possibly_connected.append(interface)
continue
LOG.debug("Interface ignored: %s", interface)
# Order the NICs:
# 1. DEFAULT_PRIMARY_INTERFACE, if connected.
# 2. Remaining connected interfaces, naturally sorted.
# 3. DEFAULT_PRIMARY_INTERFACE, if possibly connected.
# 4. Remaining possibly connected interfaces, naturally sorted.
sorted_interfaces = []
for interfaces in [connected, possibly_connected]:
interfaces = sorted(interfaces, key=natural_sort_key)
if DEFAULT_PRIMARY_INTERFACE in interfaces:
interfaces.remove(DEFAULT_PRIMARY_INTERFACE)
interfaces.insert(0, DEFAULT_PRIMARY_INTERFACE)
sorted_interfaces += interfaces
return sorted_interfaces
