def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
ds_list = list_sources(cfg_list, ds_deps, pkg_list)
ds_names = [type_utils.obj_name(f) for f in ds_list]
mode = "network" if DEP_NETWORK in ds_deps else "local"
LOG.debug("Searching for %s data source in: %s", mode, ds_names)
for name, cls in zip(ds_names, ds_list):
myrep = events.ReportEventStack(
name="search-%s" % name.replace("DataSource", ""),
description="searching for %s data from %s" % (mode, name),
message="no %s data found from %s" % (mode, name),
parent=reporter)
try:
with myrep:
LOG.debug("Seeing if we can get any data from %s", cls)
s = cls(sys_cfg, distro, paths)
if s.get_data():
myrep.message = "found %s data from %s" % (mode, name)
return (s, type_utils.obj_name(cls))
except Exception:
util.logexc(LOG, "Getting data from %s failed", cls)
msg = ("Did not find any data source,"
" searched classes: (%s)") % (", ".join(ds_names))
raise DataSourceNotFoundException(msg)
def replacer(match):
# Only 1 of the 2 groups will actually have a valid entry.
name = match.group(1)
if name is None:
name = match.group(2)
if name is None:
raise RuntimeError("Match encountered but no valid group present")
path = collections.deque(name.split("."))
selected_params = params
while len(path) > 1:
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError("Can not traverse into"
" non-dictionary '%s' of type %s while"
" looking for subkey '%s'"
% (selected_params,
tu.obj_name(selected_params),
key))
selected_params = selected_params[key]
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError("Can not extract key '%s' from non-dictionary"
" '%s' of type %s"
% (key, selected_params,
tu.obj_name(selected_params)))
return str(selected_params[key])
def merge(self, source, merge_with):
type_name = type_utils.obj_name(source)
type_name = type_name.lower()
method_name = "_on_%s" % (type_name)
meth = None
args = [source, merge_with]
if hasattr(self, method_name):
meth = getattr(self, method_name)
if not meth:
meth = self._handle_unknown
args.insert(0, method_name)
LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'",
type_name, type_utils.obj_name(merge_with),
meth.__name__, self)
return meth(*args)
def handle(name, cfg, cloud, log, args):
"""Handler method activated by cloud-init."""
verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True)
if args:
# if args are provided (from cmdline) then explicitly set verbose
out_file = args[0]
verbose = True
else:
out_file = util.get_cfg_by_path(cfg, ('debug', 'output'))
if not verbose:
log.debug(("Skipping module named %s,"
" verbose printing disabled"), name)
return
# Clean out some keys that we just don't care about showing...
dump_cfg = copy.deepcopy(cfg)
for k in SKIP_KEYS:
dump_cfg.pop(k, None)
all_keys = list(dump_cfg)
for k in all_keys:
if k.startswith("_"):
dump_cfg.pop(k, None)
# Now dump it...
to_print = StringIO()
to_print.write(_make_header("Config"))
to_print.write(_dumps(dump_cfg))
to_print.write("\n")
to_print.write(_make_header("MetaData"))
to_print.write(_dumps(cloud.datasource.metadata))
to_print.write("\n")
to_print.write(_make_header("Misc"))
to_print.write("Datasource: %s\n" %
(type_utils.obj_name(cloud.datasource)))
to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro)))
to_print.write("Hostname: %s\n" % (cloud.get_hostname(True)))
to_print.write("Instance ID: %s\n" % (cloud.get_instance_id()))
to_print.write("Locale: %s\n" % (cloud.get_locale()))
to_print.write("Launch IDX: %s\n" % (cloud.launch_index))
contents = to_print.getvalue()
content_to_file = []
for line in contents.splitlines():
line = "ci-info: %s\n" % (line)
content_to_file.append(line)
if out_file:
util.write_file(out_file, "".join(content_to_file), 0o644, "w")
else:
util.multi_log("".join(content_to_file), console=True, stderr=False)
def handle(name, cfg, cloud, log, _args):
"""
Enable and configure ntp
ntp:
pools: ['0.{{distro}}.pool.ntp.org', '1.{{distro}}.pool.ntp.org']
servers: ['192.168.2.1']
"""
ntp_cfg = cfg.get('ntp', {})
if not isinstance(ntp_cfg, (dict)):
raise RuntimeError(("'ntp' key existed in config,"
" but not a dictionary type,"
" is a %s %instead"), type_utils.obj_name(ntp_cfg))
if 'ntp' not in cfg:
LOG.debug("Skipping module named %s,"
"not present or disabled by cfg", name)
return True
install_ntp(cloud.distro.install_packages, packages=['ntp'],
check_exe="ntpd")
rename_ntp_conf()
write_ntp_config_template(ntp_cfg, cloud)
def handle(_name, cfg, cloud, log, _args):
"""
Basically turn a top level 'landscape' entry with a 'client' dict
and render it to ConfigObj format under '[client]' section in
/etc/landscape/client.conf
"""
ls_cloudcfg = cfg.get("landscape", {})
if not isinstance(ls_cloudcfg, (dict)):
raise RuntimeError(("'landscape' key existed in config,"
" but not a dictionary type,"
" is a %s instead"),
type_utils.obj_name(ls_cloudcfg))
if not ls_cloudcfg:
return
cloud.distro.install_packages(('landscape-client',))
merge_data = [
LSC_BUILTIN_CFG,
LSC_CLIENT_CFG_FILE,
ls_cloudcfg,
]
merged = merge_together(merge_data)
contents = StringIO()
merged.write(contents)
util.ensure_dir(os.path.dirname(LSC_CLIENT_CFG_FILE))
util.write_file(LSC_CLIENT_CFG_FILE, contents.getvalue())
log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE)
util.write_file(LS_DEFAULT_FILE, "RUN=1\n")
util.subp(["service", "landscape-client", "restart"])
def __init__(self, sys_cfg, distro, paths, ud_proc=None):
self.sys_cfg = sys_cfg
self.distro = distro
self.paths = paths
self.userdata = None
self.metadata = None
self.userdata_raw = None
self.vendordata = None
self.vendordata_raw = None
# find the datasource config name.
# remove 'DataSource' from classname on front, and remove 'Net' on end.
# Both Foo and FooNet sources expect config in cfg['sources']['Foo']
name = type_utils.obj_name(self)
if name.startswith(DS_PREFIX):
name = name[len(DS_PREFIX):]
if name.endswith('Net'):
name = name[0:-3]
self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
("datasource", name), {})
if not ud_proc:
self.ud_proc = ud.UserDataProcessor(self.paths)
else:
self.ud_proc = ud_proc
def write_sudo_rules(self, user, rules, sudo_file=None):
if not sudo_file:
sudo_file = self.ci_sudoers_fn
lines = ["", "# User rules for %s" % user]
if isinstance(rules, (list, tuple)):
for rule in rules:
lines.append("%s %s" % (user, rule))
elif isinstance(rules, (basestring, str)):
lines.append("%s %s" % (user, rules))
else:
msg = "Can not create sudoers rule addition with type %r"
raise TypeError(msg % (type_utils.obj_name(rules)))
content = "\n".join(lines)
content += "\n" # trailing newline
self.ensure_sudo_dir(os.path.dirname(sudo_file))
if not os.path.exists(sudo_file):
contents = [util.make_header(), content]
try:
util.write_file(sudo_file, "\n".join(contents), 0440)
except IOError as e:
util.logexc(LOG, "Failed to write sudoers file %s", sudo_file)
raise e
else:
try:
util.append_file(sudo_file, content)
except IOError as e:
util.logexc(LOG, "Failed to append sudoers file %s", sudo_file)
raise e
def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
ds_list = list_sources(cfg_list, ds_deps, pkg_list)
ds_names = [type_utils.obj_name(f) for f in ds_list]
LOG.debug("Searching for data source in: %s", ds_names)
for cls in ds_list:
try:
LOG.debug("Seeing if we can get any data from %s", cls)
s = cls(sys_cfg, distro, paths)
if s.get_data():
return (s, type_utils.obj_name(cls))
except Exception:
util.logexc(LOG, "Getting data from %s failed", cls)
msg = ("Did not find any data source,"
" searched classes: (%s)") % (", ".join(ds_names))
raise DataSourceNotFoundException(msg)
def _get_datasources(self):
# Any config provided???
pkg_list = self.cfg.get('datasource_pkg_list') or []
# Add the defaults at the end
for n in ['', type_utils.obj_name(sources)]:
if n not in pkg_list:
pkg_list.append(n)
cfg_list = self.cfg.get('datasource_list') or []
return (cfg_list, pkg_list)
def _get_datasources(self):
# Any config provided???
pkg_list = self.cfg.get('datasource_pkg_list') or []
# Add the defaults at the end
for n in ['', type_utils.obj_name(sources)]:
if n not in pkg_list:
pkg_list.append(n)
cfg_list = self.cfg.get('datasource_list') or []
return (cfg_list, pkg_list)
def install_drivers(cfg, pkg_install_func):
if not isinstance(cfg, dict):
raise TypeError(
"'drivers' config expected dict, found '%s': %s" %
(type_utils.obj_name(cfg), cfg))
cfgpath = 'nvidia/license-accepted'
# Call translate_bool to ensure that we treat string values like "yes" as
# acceptance and _don't_ treat string values like "nah" as acceptance
# because they're True-ish
nv_acc = util.translate_bool(util.get_cfg_by_path(cfg, cfgpath))
if not nv_acc:
LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
return
if not util.which('ubuntu-drivers'):
LOG.debug("'ubuntu-drivers' command not available. "
"Installing ubuntu-drivers-common")
pkg_install_func(['ubuntu-drivers-common'])
driver_arg = 'nvidia'
version_cfg = util.get_cfg_by_path(cfg, 'nvidia/version')
if version_cfg:
driver_arg += ':{}'.format(version_cfg)
LOG.debug("Installing and activating NVIDIA drivers (%s=%s, version=%s)",
cfgpath, nv_acc, version_cfg if version_cfg else 'latest')
# Register and set debconf selection linux/nvidia/latelink = true
tdir = temp_utils.mkdtemp(needs_exe=True)
debconf_file = os.path.join(tdir, 'nvidia.template')
debconf_script = os.path.join(tdir, 'nvidia-debconf.sh')
try:
util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT)
util.write_file(
debconf_script,
util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT),
mode=0o755)
util.subp([debconf_script, debconf_file])
except Exception as e:
util.logexc(
LOG, "Failed to register NVIDIA debconf template: %s", str(e))
raise
finally:
if os.path.isdir(tdir):
util.del_dir(tdir)
try:
util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
except util.ProcessExecutionError as exc:
if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr:
LOG.warning('the available version of ubuntu-drivers is'
' too old to perform requested driver installation')
elif 'No drivers found for installation.' in exc.stdout:
LOG.warning('ubuntu-drivers found no drivers for installation')
raise
def install_drivers(cfg, pkg_install_func):
if not isinstance(cfg, dict):
raise TypeError("'drivers' config expected dict, found '%s': %s" %
(type_utils.obj_name(cfg), cfg))
cfgpath = "nvidia/license-accepted"
# Call translate_bool to ensure that we treat string values like "yes" as
# acceptance and _don't_ treat string values like "nah" as acceptance
# because they're True-ish
nv_acc = util.translate_bool(util.get_cfg_by_path(cfg, cfgpath))
if not nv_acc:
LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
return
if not subp.which("ubuntu-drivers"):
LOG.debug("'ubuntu-drivers' command not available. "
"Installing ubuntu-drivers-common")
pkg_install_func(["ubuntu-drivers-common"])
driver_arg = "nvidia"
version_cfg = util.get_cfg_by_path(cfg, "nvidia/version")
if version_cfg:
driver_arg += ":{}".format(version_cfg)
LOG.debug(
"Installing and activating NVIDIA drivers (%s=%s, version=%s)",
cfgpath,
nv_acc,
version_cfg if version_cfg else "latest",
)
# Register and set debconf selection linux/nvidia/latelink = true
tdir = temp_utils.mkdtemp(needs_exe=True)
debconf_file = os.path.join(tdir, "nvidia.template")
try:
util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT)
with debconf.DebconfCommunicator("cloud-init") as dc:
dc.command(X_LOADTEMPLATEFILE, debconf_file)
except Exception as e:
util.logexc(LOG, "Failed to register NVIDIA debconf template: %s",
str(e))
raise
finally:
if os.path.isdir(tdir):
util.del_dir(tdir)
try:
subp.subp(["ubuntu-drivers", "install", "--gpgpu", driver_arg])
except subp.ProcessExecutionError as exc:
if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr:
LOG.warning("the available version of ubuntu-drivers is"
" too old to perform requested driver installation")
elif "No drivers found for installation." in exc.stdout:
LOG.warning("ubuntu-drivers found no drivers for installation")
raise
def _read_modules(self, name):
"""Read the modules from the config file given the specified name.
Returns a list of module definitions. E.g.,
[
{
"mod": "bootcmd",
"freq": "always"
"args": "some_arg",
}
]
Note that in the default case, only "mod" will be set.
"""
module_list = []
if name not in self.cfg:
return module_list
cfg_mods = self.cfg.get(name)
if not cfg_mods:
return module_list
for item in cfg_mods:
if not item:
continue
if isinstance(item, str):
module_list.append({
"mod": item.strip(),
})
elif isinstance(item, (list)):
contents = {}
# Meant to fall through...
if len(item) >= 1:
contents["mod"] = item[0].strip()
if len(item) >= 2:
contents["freq"] = item[1].strip()
if len(item) >= 3:
contents["args"] = item[2:]
if contents:
module_list.append(contents)
elif isinstance(item, (dict)):
contents = {}
valid = False
if "name" in item:
contents["mod"] = item["name"].strip()
valid = True
if "frequency" in item:
contents["freq"] = item["frequency"].strip()
if "args" in item:
contents["args"] = item["args"] or []
if contents and valid:
module_list.append(contents)
else:
raise TypeError(
"Failed to read '%s' item in config, unknown type %s" %
(item, type_utils.obj_name(item)))
return module_list
def merge(self, source, merge_with):
type_name = type_utils.obj_name(source)
type_name = type_name.lower()
method_name = "_on_%s" % (type_name)
meth = None
args = [source, merge_with]
if hasattr(self, method_name):
meth = getattr(self, method_name)
if not meth:
meth = self._handle_unknown
args.insert(0, method_name)
return meth(*args)
def merge(self, source, merge_with):
type_name = type_utils.obj_name(source)
type_name = type_name.lower()
method_name = "_on_%s" % (type_name)
meth = None
args = [source, merge_with]
if hasattr(self, method_name):
meth = getattr(self, method_name)
if not meth:
meth = self._handle_unknown
args.insert(0, method_name)
return meth(*args)
def _normalize_groups(grp_cfg):
if isinstance(grp_cfg, six.string_types):
grp_cfg = grp_cfg.strip().split(",")
if isinstance(grp_cfg, list):
c_grp_cfg = {}
for i in grp_cfg:
if isinstance(i, dict):
for k, v in i.items():
if k not in c_grp_cfg:
if isinstance(v, list):
c_grp_cfg[k] = list(v)
elif isinstance(v, six.string_types):
c_grp_cfg[k] = [v]
else:
raise TypeError("Bad group member type %s" %
type_utils.obj_name(v))
else:
if isinstance(v, list):
c_grp_cfg[k].extend(v)
elif isinstance(v, six.string_types):
c_grp_cfg[k].append(v)
else:
raise TypeError("Bad group member type %s" %
type_utils.obj_name(v))
elif isinstance(i, six.string_types):
if i not in c_grp_cfg:
c_grp_cfg[i] = []
else:
raise TypeError("Unknown group name type %s" %
type_utils.obj_name(i))
grp_cfg = c_grp_cfg
groups = {}
if isinstance(grp_cfg, dict):
for (grp_name, grp_members) in grp_cfg.items():
groups[grp_name] = util.uniq_merge_sorted(grp_members)
else:
raise TypeError(("Group config must be list, dict "
" or string types only and not %s") %
type_utils.obj_name(grp_cfg))
return groups
def _normalize_groups(grp_cfg):
if isinstance(grp_cfg, six.string_types):
grp_cfg = grp_cfg.strip().split(",")
if isinstance(grp_cfg, list):
c_grp_cfg = {}
for i in grp_cfg:
if isinstance(i, dict):
for k, v in i.items():
if k not in c_grp_cfg:
if isinstance(v, list):
c_grp_cfg[k] = list(v)
elif isinstance(v, six.string_types):
c_grp_cfg[k] = [v]
else:
raise TypeError("Bad group member type %s" %
type_utils.obj_name(v))
else:
if isinstance(v, list):
c_grp_cfg[k].extend(v)
elif isinstance(v, six.string_types):
c_grp_cfg[k].append(v)
else:
raise TypeError("Bad group member type %s" %
type_utils.obj_name(v))
elif isinstance(i, six.string_types):
if i not in c_grp_cfg:
c_grp_cfg[i] = []
else:
raise TypeError("Unknown group name type %s" %
type_utils.obj_name(i))
grp_cfg = c_grp_cfg
groups = {}
if isinstance(grp_cfg, dict):
for (grp_name, grp_members) in grp_cfg.items():
groups[grp_name] = util.uniq_merge_sorted(grp_members)
else:
raise TypeError(("Group config must be list, dict "
" or string types only and not %s") %
type_utils.obj_name(grp_cfg))
return groups
def _reflect_cur_instance(self):
# Remove the old symlink and attach a new one so
# that further reads/writes connect into the right location
idir = self._get_ipath()
util.del_file(self.paths.instance_link)
util.sym_link(idir, self.paths.instance_link)
# Ensures these dirs exist
dir_list = []
for d in self._get_instance_subdirs():
dir_list.append(os.path.join(idir, d))
util.ensure_dirs(dir_list)
# Write out information on what is being used for the current instance
# and what may have been used for a previous instance...
dp = self.paths.get_cpath('data')
# Write what the datasource was and is..
ds = "%s: %s" % (type_utils.obj_name(self.datasource), self.datasource)
previous_ds = None
ds_fn = os.path.join(idir, 'datasource')
try:
previous_ds = util.load_file(ds_fn).strip()
except Exception:
pass
if not previous_ds:
previous_ds = ds
util.write_file(ds_fn, "%s\n" % ds)
util.write_file(os.path.join(dp, 'previous-datasource'),
"%s\n" % (previous_ds))
# What the instance id was and is...
iid = self.datasource.get_instance_id()
previous_iid = None
iid_fn = os.path.join(dp, 'instance-id')
try:
previous_iid = util.load_file(iid_fn).strip()
except Exception:
pass
if not previous_iid:
previous_iid = iid
util.write_file(iid_fn, "%s\n" % iid)
util.write_file(os.path.join(dp, 'previous-instance-id'),
"%s\n" % (previous_iid))
# Ensure needed components are regenerated
# after change of instance which may cause
# change of configuration
self._reset()
return iid
def _read_modules(self, name):
module_list = []
if name not in self.cfg:
return module_list
cfg_mods = self.cfg.get(name)
if not cfg_mods:
return module_list
# Create 'module_list', an array of hashes
# Where hash['mod'] = module name
# hash['freq'] = frequency
# hash['args'] = arguments
for item in cfg_mods:
if not item:
continue
if isinstance(item, str):
module_list.append(
{
"mod": item.strip(),
}
)
elif isinstance(item, (list)):
contents = {}
# Meant to fall through...
if len(item) >= 1:
contents["mod"] = item[0].strip()
if len(item) >= 2:
contents["freq"] = item[1].strip()
if len(item) >= 3:
contents["args"] = item[2:]
if contents:
module_list.append(contents)
elif isinstance(item, (dict)):
contents = {}
valid = False
if "name" in item:
contents["mod"] = item["name"].strip()
valid = True
if "frequency" in item:
contents["freq"] = item["frequency"].strip()
if "args" in item:
contents["args"] = item["args"] or []
if contents and valid:
module_list.append(contents)
else:
raise TypeError(
"Failed to read '%s' item in config, unknown type %s"
% (item, type_utils.obj_name(item))
)
return module_list
def _reflect_cur_instance(self):
# Remove the old symlink and attach a new one so
# that further reads/writes connect into the right location
idir = self._get_ipath()
util.del_file(self.paths.instance_link)
util.sym_link(idir, self.paths.instance_link)
# Ensures these dirs exist
dir_list = []
for d in self._get_instance_subdirs():
dir_list.append(os.path.join(idir, d))
util.ensure_dirs(dir_list)
# Write out information on what is being used for the current instance
# and what may have been used for a previous instance...
dp = self.paths.get_cpath("data")
# Write what the datasource was and is..
ds = "%s: %s" % (type_utils.obj_name(self.datasource), self.datasource)
previous_ds = None
ds_fn = os.path.join(idir, "datasource")
try:
previous_ds = util.load_file(ds_fn).strip()
except Exception:
pass
if not previous_ds:
previous_ds = ds
util.write_file(ds_fn, "%s\n" % ds)
util.write_file(
os.path.join(dp, "previous-datasource"), "%s\n" % (previous_ds)
)
# What the instance id was and is...
iid = self.datasource.get_instance_id()
iid_fn = os.path.join(dp, "instance-id")
previous_iid = self.previous_iid()
util.write_file(iid_fn, "%s\n" % iid)
util.write_file(self.paths.get_runpath("instance_id"), "%s\n" % iid)
util.write_file(
os.path.join(dp, "previous-instance-id"), "%s\n" % (previous_iid)
)
self._write_to_cache()
# Ensure needed components are regenerated
# after change of instance which may cause
# change of configuration
self._reset()
return iid
def replacer(match):
# Only 1 of the 2 groups will actually have a valid entry.
name = match.group(1)
if name is None:
name = match.group(2)
if name is None:
raise RuntimeError("Match encountered but no valid group present")
path = collections.deque(name.split("."))
selected_params = params
while len(path) > 1:
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError(
"Can not traverse into"
" non-dictionary '%s' of type %s while"
" looking for subkey '%s'" %
(selected_params, tu.obj_name(selected_params), key))
selected_params = selected_params[key]
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError(
"Can not extract key '%s' from non-dictionary '%s' of type %s"
% (key, selected_params, tu.obj_name(selected_params)))
return str(selected_params[key])
def __init__(self, sys_cfg, distro, paths, ud_proc=None):
self.sys_cfg = sys_cfg
self.distro = distro
self.paths = paths
self.userdata = None
self.metadata = None
self.userdata_raw = None
name = type_utils.obj_name(self)
if name.startswith(DS_PREFIX):
name = name[len(DS_PREFIX):]
self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
("datasource", name), {})
if not ud_proc:
self.ud_proc = ud.UserDataProcessor(self.paths)
else:
self.ud_proc = ud_proc
def __init__(self, sys_cfg, distro, paths, ud_proc=None):
self.sys_cfg = sys_cfg
self.distro = distro
self.paths = paths
self.userdata = None
self.metadata = None
self.userdata_raw = None
name = type_utils.obj_name(self)
if name.startswith(DS_PREFIX):
name = name[len(DS_PREFIX):]
self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", name),
{})
if not ud_proc:
self.ud_proc = ud.UserDataProcessor(self.paths)
else:
self.ud_proc = ud_proc
def _read_modules(self, name):
module_list = []
if name not in self.cfg:
return module_list
cfg_mods = self.cfg.get(name)
if not cfg_mods:
return module_list
# Create 'module_list', an array of hashes
# Where hash['mod'] = module name
# hash['freq'] = frequency
# hash['args'] = arguments
for item in cfg_mods:
if not item:
continue
if isinstance(item, six.string_types):
module_list.append({
'mod': item.strip(),
})
elif isinstance(item, (list)):
contents = {}
# Meant to fall through...
if len(item) >= 1:
contents['mod'] = item[0].strip()
if len(item) >= 2:
contents['freq'] = item[1].strip()
if len(item) >= 3:
contents['args'] = item[2:]
if contents:
module_list.append(contents)
elif isinstance(item, (dict)):
contents = {}
valid = False
if 'name' in item:
contents['mod'] = item['name'].strip()
valid = True
if 'frequency' in item:
contents['freq'] = item['frequency'].strip()
if 'args' in item:
contents['args'] = item['args'] or []
if contents and valid:
module_list.append(contents)
else:
raise TypeError(("Failed to read '%s' item in config,"
" unknown type %s") %
(item, type_utils.obj_name(item)))
return module_list
def _fixup_modules(self, raw_mods) -> List[ModuleDetails]:
"""Convert list of returned from _read_modules() into new format.
Invalid modules and arguments are ingnored.
Also ensures that the module has the required meta fields.
"""
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod["mod"]
freq = raw_mod.get("freq")
run_args = raw_mod.get("args") or []
mod_name = form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warning(
"Config specified module %s has an unknown frequency %s",
raw_name,
freq,
)
# Misconfigured in /etc/cloud/cloud.cfg. Reset so cc_* module
# default meta attribute "frequency" value is used.
freq = None
mod_locs, looked_locs = importer.find_module(
mod_name, ["", type_utils.obj_name(config)], ["handle"])
if not mod_locs:
LOG.warning(
"Could not find module named %s (searched %s)",
mod_name,
looked_locs,
)
continue
mod = importer.import_module(mod_locs[0])
validate_module(mod, raw_name)
if freq is None:
# Use cc_* module default setting since no cloud.cfg overrides
freq = mod.meta["frequency"]
mostly_mods.append(
ModuleDetails(
module=mod,
name=raw_name,
frequency=freq,
run_args=run_args,
))
return mostly_mods
def _read_modules(self, name):
module_list = []
if name not in self.cfg:
return module_list
cfg_mods = self.cfg[name]
# Create 'module_list', an array of hashes
# Where hash['mod'] = module name
# hash['freq'] = frequency
# hash['args'] = arguments
for item in cfg_mods:
if not item:
continue
if isinstance(item, (str, basestring)):
module_list.append({
'mod': item.strip(),
})
elif isinstance(item, (list)):
contents = {}
# Meant to fall through...
if len(item) >= 1:
contents['mod'] = item[0].strip()
if len(item) >= 2:
contents['freq'] = item[1].strip()
if len(item) >= 3:
contents['args'] = item[2:]
if contents:
module_list.append(contents)
elif isinstance(item, (dict)):
contents = {}
valid = False
if 'name' in item:
contents['mod'] = item['name'].strip()
valid = True
if 'frequency' in item:
contents['freq'] = item['frequency'].strip()
if 'args' in item:
contents['args'] = item['args'] or []
if contents and valid:
module_list.append(contents)
else:
raise TypeError(
("Failed to read '%s' item in config,"
" unknown type %s") % (item, type_utils.obj_name(item)))
return module_list
def handle(name, cfg, cloud, log, _args):
"""Enable and configure ntp."""
if 'ntp' not in cfg:
LOG.debug("Skipping module named %s, not present or disabled by cfg",
name)
return
ntp_cfg = cfg['ntp']
if ntp_cfg is None:
ntp_cfg = {} # Allow empty config which will install the package
# TODO drop this when validate_cloudconfig_schema is strict=True
if not isinstance(ntp_cfg, (dict)):
raise RuntimeError(("'ntp' key existed in config,"
" but not a dictionary type,"
" is a %s %instead"), type_utils.obj_name(ntp_cfg))
validate_cloudconfig_schema(cfg, schema)
if ntp_installable():
service_name = 'ntp'
confpath = NTP_CONF
template_name = None
packages = ['ntp']
check_exe = 'ntpd'
else:
service_name = 'systemd-timesyncd'
confpath = TIMESYNCD_CONF
template_name = 'timesyncd.conf'
packages = []
check_exe = '/lib/systemd/systemd-timesyncd'
rename_ntp_conf()
# ensure when ntp is installed it has a configuration file
# to use instead of starting up with packaged defaults
write_ntp_config_template(ntp_cfg, cloud, confpath, template=template_name)
install_ntp(cloud.distro.install_packages,
packages=packages,
check_exe=check_exe)
try:
reload_ntp(service_name, systemd=cloud.distro.uses_systemd())
except util.ProcessExecutionError as e:
LOG.exception("Failed to reload/start ntp service: %s", e)
raise
def _fixup_modules(self, raw_mods):
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod["mod"]
freq = raw_mod.get("freq")
run_args = raw_mod.get("args") or []
mod_name = config.form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warn(("Config specified module %s" " has an unknown frequency %s"), raw_name, freq)
# Reset it so when ran it will get set to a known value
freq = None
mod_locs, looked_locs = importer.find_module(mod_name, ["", type_utils.obj_name(config)], ["handle"])
if not mod_locs:
LOG.warn("Could not find module named %s (searched %s)", mod_name, looked_locs)
continue
mod = config.fixup_module(importer.import_module(mod_locs[0]))
mostly_mods.append([mod, raw_name, freq, run_args])
return mostly_mods
class ExpectedDataSources(test_helpers.TestCase):
builtin_list = settings.CFG_BUILTIN['datasource_list']
deps_local = [sources.DEP_FILESYSTEM]
deps_network = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
pkg_list = [type_utils.obj_name(sources)]
def test_expected_default_local_sources_found(self):
found = sources.list_sources(self.builtin_list, self.deps_local,
self.pkg_list)
self.assertEqual(set(DEFAULT_LOCAL), set(found))
def test_expected_default_network_sources_found(self):
found = sources.list_sources(self.builtin_list, self.deps_network,
self.pkg_list)
self.assertEqual(set(DEFAULT_NETWORK), set(found))
def test_expected_nondefault_network_sources_found(self):
found = sources.list_sources(['AliYun'], self.deps_network,
self.pkg_list)
self.assertEqual(set([AliYun.DataSourceAliYun]), set(found))
def _fixup_modules(self, raw_mods):
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod['mod']
freq = raw_mod.get('freq')
run_args = raw_mod.get('args') or []
mod_name = config.form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warn(("Config specified module %s"
" has an unknown frequency %s"), raw_name, freq)
# Reset it so when ran it will get set to a known value
freq = None
mod_locs = importer.find_module(
mod_name, ['', type_utils.obj_name(config)], ['handle'])
if not mod_locs:
LOG.warn("Could not find module named %s", mod_name)
continue
mod = config.fixup_module(importer.import_module(mod_locs[0]))
mostly_mods.append([mod, raw_name, freq, run_args])
return mostly_mods
def __init__(self, sys_cfg, distro, paths, ud_proc=None):
self.sys_cfg = sys_cfg
self.distro = distro
self.paths = paths
self.userdata = None
self.metadata = None
self.userdata_raw = None
# find the datasource config name.
# remove 'DataSource' from classname on front, and remove 'Net' on end.
# Both Foo and FooNet sources expect config in cfg['sources']['Foo']
name = type_utils.obj_name(self)
if name.startswith(DS_PREFIX):
name = name[len(DS_PREFIX):]
if name.endswith('Net'):
name = name[0:-3]
self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", name),
{})
if not ud_proc:
self.ud_proc = ud.UserDataProcessor(self.paths)
else:
self.ud_proc = ud_proc
def write_sudo_rules(self, user, rules, sudo_file=None):
if not sudo_file:
sudo_file = self.ci_sudoers_fn
lines = [
'',
"# User rules for %s" % user,
]
if isinstance(rules, (list, tuple)):
for rule in rules:
lines.append("%s %s" % (user, rule))
elif isinstance(rules, str):
lines.append("%s %s" % (user, rules))
else:
msg = "Can not create sudoers rule addition with type %r"
raise TypeError(msg % (type_utils.obj_name(rules)))
content = "\n".join(lines)
content += "\n" # trailing newline
self.ensure_sudo_dir(os.path.dirname(sudo_file))
if not os.path.exists(sudo_file):
contents = [
util.make_header(),
content,
]
try:
util.write_file(sudo_file, "\n".join(contents), 0o440)
except IOError as e:
util.logexc(LOG, "Failed to write sudoers file %s", sudo_file)
raise e
else:
try:
util.append_file(sudo_file, content)
except IOError as e:
util.logexc(LOG, "Failed to append sudoers file %s", sudo_file)
raise e
def install_drivers(cfg, pkg_install_func):
if not isinstance(cfg, dict):
raise TypeError("'drivers' config expected dict, found '%s': %s" %
(type_utils.obj_name(cfg), cfg))
cfgpath = 'nvidia/license-accepted'
# Call translate_bool to ensure that we treat string values like "yes" as
# acceptance and _don't_ treat string values like "nah" as acceptance
# because they're True-ish
nv_acc = util.translate_bool(util.get_cfg_by_path(cfg, cfgpath))
if not nv_acc:
LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
return
if not util.which('ubuntu-drivers'):
LOG.debug("'ubuntu-drivers' command not available. "
"Installing ubuntu-drivers-common")
pkg_install_func(['ubuntu-drivers-common'])
driver_arg = 'nvidia'
version_cfg = util.get_cfg_by_path(cfg, 'nvidia/version')
if version_cfg:
driver_arg += ':{}'.format(version_cfg)
LOG.debug("Installing NVIDIA drivers (%s=%s, version=%s)", cfgpath, nv_acc,
version_cfg if version_cfg else 'latest')
try:
util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
except util.ProcessExecutionError as exc:
if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr:
LOG.warning('the available version of ubuntu-drivers is'
' too old to perform requested driver installation')
elif 'No drivers found for installation.' in exc.stdout:
LOG.warning('ubuntu-drivers found no drivers for installation')
raise
def _read_modules(self, name):
module_list = []
if name not in self.cfg:
return module_list
cfg_mods = self.cfg[name]
# Create 'module_list', an array of hashes
# Where hash['mod'] = module name
# hash['freq'] = frequency
# hash['args'] = arguments
for item in cfg_mods:
if not item:
continue
if isinstance(item, (str, basestring)):
module_list.append({"mod": item.strip()})
elif isinstance(item, (list)):
contents = {}
# Meant to fall through...
if len(item) >= 1:
contents["mod"] = item[0].strip()
if len(item) >= 2:
contents["freq"] = item[1].strip()
if len(item) >= 3:
contents["args"] = item[2:]
if contents:
module_list.append(contents)
elif isinstance(item, (dict)):
contents = {}
valid = False
if "name" in item:
contents["mod"] = item["name"].strip()
valid = True
if "frequency" in item:
contents["freq"] = item["frequency"].strip()
if "args" in item:
contents["args"] = item["args"] or []
if contents and valid:
module_list.append(contents)
else:
raise TypeError(
("Failed to read '%s' item in config," " unknown type %s") % (item, type_utils.obj_name(item))
)
return module_list
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], ["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
if "mounts" in cfg:
cfgmnt = cfg["mounts"]
for i in range(len(cfgmnt)):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warn("Mount option %s not a list, got a %s instead", (i + 1), type_utils.obj_name(cfgmnt[i]))
continue
startname = str(cfgmnt[i][0])
log.debug("Attempting to determine the real name of %s", startname)
# workaround, allow user to specify 'ephemeral'
# rather than more ec2 correct 'ephemeral0'
if startname == "ephemeral":
cfgmnt[i][0] = "ephemeral0"
log.debug(("Adjusted mount option %s " "name from ephemeral to ephemeral0"), (i + 1))
if is_mdname(startname):
newname = cloud.device_name_to_device(startname)
if not newname:
log.debug("Ignoring nonexistant named mount %s", startname)
cfgmnt[i][1] = None
else:
renamed = newname
if not newname.startswith("/"):
renamed = "/dev/%s" % newname
cfgmnt[i][0] = renamed
log.debug("Mapped metadata name %s to %s", startname, renamed)
else:
if SHORTNAME.match(startname):
renamed = "/dev/%s" % startname
log.debug("Mapped shortname name %s to %s", startname, renamed)
cfgmnt[i][0] = renamed
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
for j in range(len(cfgmnt[i])):
if cfgmnt[i][j] is None:
continue
else:
cfgmnt[i][j] = str(cfgmnt[i][j])
for i in range(len(cfgmnt)):
# fill in values with defaults from defvals above
for j in range(len(defvals)):
if len(cfgmnt[i]) <= j:
cfgmnt[i].append(defvals[j])
elif cfgmnt[i][j] is None:
cfgmnt[i][j] = defvals[j]
# if the second entry in the list is 'None' this
# clears all previous entries of that same 'fs_spec'
# (fs_spec is the first field in /etc/fstab, ie, that device)
if cfgmnt[i][1] is None:
for j in range(i):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
startname = defmnt[0]
devname = cloud.device_name_to_device(startname)
if devname is None:
log.debug("Ignoring nonexistant named default mount %s", startname)
continue
if devname.startswith("/"):
defmnt[0] = devname
else:
defmnt[0] = "/dev/%s" % devname
log.debug("Mapped default device %s to %s", startname, defmnt[0])
cfgmnt_has = False
for cfgm in cfgmnt:
if cfgm[0] == defmnt[0]:
cfgmnt_has = True
break
if cfgmnt_has:
log.debug(("Not including %s, already" " previously included"), startname)
continue
cfgmnt.append(defmnt)
# now, each entry in the cfgmnt list has all fstab values
# if the second field is None (not the string, the value) we skip it
actlist = []
for x in cfgmnt:
if x[1] is None:
log.debug("Skipping non-existent device named %s", x[0])
else:
actlist.append(x)
if len(actlist) == 0:
log.debug("No modifications to fstab needed.")
return
comment = "comment=cloudconfig"
cc_lines = []
needswap = False
dirs = []
for line in actlist:
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], comment)
if line[2] == "swap":
needswap = True
if line[1].startswith("/"):
dirs.append(line[1])
cc_lines.append("\t".join(line))
fstab_lines = []
for line in util.load_file(FSTAB_PATH).splitlines():
try:
toks = WS.split(line)
if toks[3].find(comment) != -1:
continue
except:
pass
fstab_lines.append(line)
fstab_lines.extend(cc_lines)
contents = "%s\n" % ("\n".join(fstab_lines))
util.write_file(FSTAB_PATH, contents)
if needswap:
try:
util.subp(("swapon", "-a"))
except:
util.logexc(log, "Activating swap via 'swapon -a' failed")
for d in dirs:
try:
util.ensure_dir(d)
except:
util.logexc(log, "Failed to make '%s' config-mount", d)
try:
util.subp(("mount", "-a"))
except:
util.logexc(log, "Activating mounts via 'mount -a' failed")
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
def_mnt_opts = "defaults,nobootwait"
if cloud.distro.uses_systemd():
def_mnt_opts = "defaults,nofail"
defvals = [None, None, "auto", def_mnt_opts, "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"],
["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
if "mounts" in cfg:
cfgmnt = cfg["mounts"]
for i in range(len(cfgmnt)):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warn("Mount option %s not a list, got a %s instead",
(i + 1), type_utils.obj_name(cfgmnt[i]))
continue
start = str(cfgmnt[i][0])
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized is None:
log.debug("Ignorming nonexistant named mount %s", start)
continue
if sanitized != start:
log.debug("changed %s => %s" % (start, sanitized))
cfgmnt[i][0] = sanitized
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
for j in range(len(cfgmnt[i])):
if cfgmnt[i][j] is None:
continue
else:
cfgmnt[i][j] = str(cfgmnt[i][j])
for i in range(len(cfgmnt)):
# fill in values with defaults from defvals above
for j in range(len(defvals)):
if len(cfgmnt[i]) <= j:
cfgmnt[i].append(defvals[j])
elif cfgmnt[i][j] is None:
cfgmnt[i][j] = defvals[j]
# if the second entry in the list is 'None' this
# clears all previous entries of that same 'fs_spec'
# (fs_spec is the first field in /etc/fstab, ie, that device)
if cfgmnt[i][1] is None:
for j in range(i):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
start = defmnt[0]
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized is None:
log.debug("Ignoring nonexistant default named mount %s", start)
continue
if sanitized != start:
log.debug("changed default device %s => %s" % (start, sanitized))
defmnt[0] = sanitized
cfgmnt_has = False
for cfgm in cfgmnt:
if cfgm[0] == defmnt[0]:
cfgmnt_has = True
break
if cfgmnt_has:
log.debug(("Not including %s, already"
" previously included"), start)
continue
cfgmnt.append(defmnt)
# now, each entry in the cfgmnt list has all fstab values
# if the second field is None (not the string, the value) we skip it
actlist = []
for x in cfgmnt:
if x[1] is None:
log.debug("Skipping non-existent device named %s", x[0])
else:
actlist.append(x)
swapret = handle_swapcfg(cfg.get('swap', {}))
if swapret:
actlist.append([swapret, "none", "swap", "sw", "0", "0"])
if len(actlist) == 0:
log.debug("No modifications to fstab needed.")
return
comment = "comment=cloudconfig"
cc_lines = []
needswap = False
dirs = []
for line in actlist:
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], comment)
if line[2] == "swap":
needswap = True
if line[1].startswith("/"):
dirs.append(line[1])
cc_lines.append('\t'.join(line))
fstab_lines = []
for line in util.load_file(FSTAB_PATH).splitlines():
try:
toks = WS.split(line)
if toks[3].find(comment) != -1:
continue
except:
pass
fstab_lines.append(line)
fstab_lines.extend(cc_lines)
contents = "%s\n" % ('\n'.join(fstab_lines))
util.write_file(FSTAB_PATH, contents)
if needswap:
try:
util.subp(("swapon", "-a"))
except:
util.logexc(log, "Activating swap via 'swapon -a' failed")
for d in dirs:
try:
util.ensure_dir(d)
except:
util.logexc(log, "Failed to make '%s' config-mount", d)
try:
util.subp(("mount", "-a"))
except:
util.logexc(log, "Activating mounts via 'mount -a' failed")
def __repr__(self):
return "%s: [%s]" % (type_utils.obj_name(self), self.list_types())
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"],
["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
if "mounts" in cfg:
cfgmnt = cfg["mounts"]
for i in range(len(cfgmnt)):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warn("Mount option %s not a list, got a %s instead", (i + 1),
type_utils.obj_name(cfgmnt[i]))
continue
startname = str(cfgmnt[i][0])
log.debug("Attempting to determine the real name of %s", startname)
# workaround, allow user to specify 'ephemeral'
# rather than more ec2 correct 'ephemeral0'
if startname == "ephemeral":
cfgmnt[i][0] = "ephemeral0"
log.debug(("Adjusted mount option %s "
"name from ephemeral to ephemeral0"), (i + 1))
if is_mdname(startname):
newname = cloud.device_name_to_device(startname)
if not newname:
log.debug("Ignoring nonexistant named mount %s", startname)
cfgmnt[i][1] = None
else:
renamed = newname
if not newname.startswith("/"):
renamed = "/dev/%s" % newname
cfgmnt[i][0] = renamed
log.debug("Mapped metadata name %s to %s", startname, renamed)
else:
if SHORTNAME.match(startname):
renamed = "/dev/%s" % startname
log.debug("Mapped shortname name %s to %s", startname, renamed)
cfgmnt[i][0] = renamed
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
for j in range(len(cfgmnt[i])):
if cfgmnt[i][j] is None:
continue
else:
cfgmnt[i][j] = str(cfgmnt[i][j])
for i in range(len(cfgmnt)):
# fill in values with defaults from defvals above
for j in range(len(defvals)):
if len(cfgmnt[i]) <= j:
cfgmnt[i].append(defvals[j])
elif cfgmnt[i][j] is None:
cfgmnt[i][j] = defvals[j]
# if the second entry in the list is 'None' this
# clears all previous entries of that same 'fs_spec'
# (fs_spec is the first field in /etc/fstab, ie, that device)
if cfgmnt[i][1] is None:
for j in range(i):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
startname = defmnt[0]
devname = cloud.device_name_to_device(startname)
if devname is None:
log.debug("Ignoring nonexistant named default mount %s", startname)
continue
if devname.startswith("/"):
defmnt[0] = devname
else:
defmnt[0] = "/dev/%s" % devname
log.debug("Mapped default device %s to %s", startname, defmnt[0])
cfgmnt_has = False
for cfgm in cfgmnt:
if cfgm[0] == defmnt[0]:
cfgmnt_has = True
break
if cfgmnt_has:
log.debug(("Not including %s, already"
" previously included"), startname)
continue
cfgmnt.append(defmnt)
# now, each entry in the cfgmnt list has all fstab values
# if the second field is None (not the string, the value) we skip it
actlist = []
for x in cfgmnt:
if x[1] is None:
log.debug("Skipping non-existent device named %s", x[0])
else:
actlist.append(x)
if len(actlist) == 0:
log.debug("No modifications to fstab needed.")
return
comment = "comment=cloudconfig"
cc_lines = []
needswap = False
dirs = []
for line in actlist:
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], comment)
if line[2] == "swap":
needswap = True
if line[1].startswith("/"):
dirs.append(line[1])
cc_lines.append('\t'.join(line))
fstab_lines = []
for line in util.load_file(FSTAB_PATH).splitlines():
try:
toks = WS.split(line)
if toks[3].find(comment) != -1:
continue
except:
pass
fstab_lines.append(line)
fstab_lines.extend(cc_lines)
contents = "%s\n" % ('\n'.join(fstab_lines))
util.write_file(FSTAB_PATH, contents)
if needswap:
try:
util.subp(("swapon", "-a"))
except:
util.logexc(log, "Activating swap via 'swapon -a' failed")
for d in dirs:
try:
util.ensure_dir(d)
except:
util.logexc(log, "Failed to make '%s' config-mount", d)
try:
util.subp(("mount", "-a"))
except:
util.logexc(log, "Activating mounts via 'mount -a' failed")
def _normalize_users(u_cfg, def_user_cfg=None):
if isinstance(u_cfg, dict):
ad_ucfg = []
for (k, v) in u_cfg.items():
if isinstance(v, (bool, int, float) + six.string_types):
if util.is_true(v):
ad_ucfg.append(str(k))
elif isinstance(v, dict):
v['name'] = k
ad_ucfg.append(v)
else:
raise TypeError(("Unmappable user value type %s"
" for key %s") % (type_utils.obj_name(v), k))
u_cfg = ad_ucfg
elif isinstance(u_cfg, six.string_types):
u_cfg = util.uniq_merge_sorted(u_cfg)
users = {}
for user_config in u_cfg:
if isinstance(user_config, (list,) + six.string_types):
for u in util.uniq_merge(user_config):
if u and u not in users:
users[u] = {}
elif isinstance(user_config, dict):
if 'name' in user_config:
n = user_config.pop('name')
prev_config = users.get(n) or {}
users[n] = util.mergemanydict([prev_config,
user_config])
else:
# Assume the default user then
prev_config = users.get('default') or {}
users['default'] = util.mergemanydict([prev_config,
user_config])
else:
raise TypeError(("User config must be dictionary/list "
" or string types only and not %s") %
type_utils.obj_name(user_config))
# Ensure user options are in the right python friendly format
if users:
c_users = {}
for (uname, uconfig) in users.items():
c_uconfig = {}
for (k, v) in uconfig.items():
k = k.replace('-', '_').strip()
if k:
c_uconfig[k] = v
c_users[uname] = c_uconfig
users = c_users
# Fixup the default user into the real
# default user name and replace it...
def_user = None
if users and 'default' in users:
def_config = users.pop('default')
if def_user_cfg:
# Pickup what the default 'real name' is
# and any groups that are provided by the
# default config
def_user_cfg = def_user_cfg.copy()
def_user = def_user_cfg.pop('name')
def_groups = def_user_cfg.pop('groups', [])
# Pickup any config + groups for that user name
# that we may have previously extracted
parsed_config = users.pop(def_user, {})
parsed_groups = parsed_config.get('groups', [])
# Now merge our extracted groups with
# anything the default config provided
users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
parsed_config['groups'] = ",".join(users_groups)
# The real config for the default user is the
# combination of the default user config provided
# by the distro, the default user config provided
# by the above merging for the user 'default' and
# then the parsed config from the user's 'real name'
# which does not have to be 'default' (but could be)
users[def_user] = util.mergemanydict([def_user_cfg,
def_config,
parsed_config])
# Ensure that only the default user that we
# found (if any) is actually marked as being
# the default user
if users:
for (uname, uconfig) in users.items():
if def_user and uname == def_user:
uconfig['default'] = True
else:
uconfig['default'] = False
return users
def handle(name, cfg, cloud, log, _args):
"""Enable and configure ntp."""
if 'ntp' not in cfg:
LOG.debug("Skipping module named %s, not present or disabled by cfg",
name)
return
ntp_cfg = cfg['ntp']
if ntp_cfg is None:
ntp_cfg = {} # Allow empty config which will install the package
# TODO drop this when validate_cloudconfig_schema is strict=True
if not isinstance(ntp_cfg, (dict)):
raise RuntimeError(
"'ntp' key existed in config, but not a dictionary type,"
" is a {_type} instead".format(_type=type_utils.obj_name(ntp_cfg)))
validate_cloudconfig_schema(cfg, schema)
# Allow users to explicitly enable/disable
enabled = ntp_cfg.get('enabled', True)
if util.is_false(enabled):
LOG.debug("Skipping module named %s, disabled by cfg", name)
return
# Select which client is going to be used and get the configuration
ntp_client_config = select_ntp_client(ntp_cfg.get('ntp_client'),
cloud.distro)
# Allow user ntp config to override distro configurations
ntp_client_config = util.mergemanydict(
[ntp_client_config, ntp_cfg.get('config', {})], reverse=True)
supplemental_schema_validation(ntp_client_config)
rename_ntp_conf(confpath=ntp_client_config.get('confpath'))
template_fn = None
if not ntp_client_config.get('template'):
template_name = (ntp_client_config.get('template_name').replace(
'{distro}', cloud.distro.name))
template_fn = cloud.get_template_filename(template_name)
if not template_fn:
msg = ('No template found, not rendering %s' %
ntp_client_config.get('template_name'))
raise RuntimeError(msg)
write_ntp_config_template(cloud.distro.name,
servers=ntp_cfg.get('servers', []),
pools=ntp_cfg.get('pools', []),
path=ntp_client_config.get('confpath'),
template_fn=template_fn,
template=ntp_client_config.get('template'))
install_ntp_client(cloud.distro.install_packages,
packages=ntp_client_config['packages'],
check_exe=ntp_client_config['check_exe'])
try:
reload_ntp(ntp_client_config['service_name'],
systemd=cloud.distro.uses_systemd())
except util.ProcessExecutionError as e:
LOG.exception("Failed to reload/start ntp service: %s", e)
raise
def _normalize_users(u_cfg, def_user_cfg=None):
if isinstance(u_cfg, dict):
ad_ucfg = []
for k, v in u_cfg.items():
if isinstance(v, (bool, int, float, str)):
if util.is_true(v):
ad_ucfg.append(str(k))
elif isinstance(v, dict):
v["name"] = k
ad_ucfg.append(v)
else:
raise TypeError("Unmappable user value type %s for key %s" %
(type_utils.obj_name(v), k))
u_cfg = ad_ucfg
elif isinstance(u_cfg, str):
u_cfg = util.uniq_merge_sorted(u_cfg)
users = {}
for user_config in u_cfg:
if isinstance(user_config, (list, str)):
for u in util.uniq_merge(user_config):
if u and u not in users:
users[u] = {}
elif isinstance(user_config, dict):
n = user_config.pop("name", "default")
prev_config = users.get(n) or {}
users[n] = util.mergemanydict([prev_config, user_config])
else:
raise TypeError("User config must be dictionary/list or string "
" types only and not %s" %
(type_utils.obj_name(user_config)))
# Ensure user options are in the right python friendly format
if users:
c_users = {}
for uname, uconfig in users.items():
c_uconfig = {}
for k, v in uconfig.items():
k = k.replace("-", "_").strip()
if k:
c_uconfig[k] = v
c_users[uname] = c_uconfig
users = c_users
# Fix the default user into the actual default user name and replace it.
def_user = None
if users and "default" in users:
def_config = users.pop("default")
if def_user_cfg:
# Pickup what the default 'real name' is and any groups that are
# provided by the default config
def_user_cfg = def_user_cfg.copy()
def_user = def_user_cfg.pop("name")
def_groups = def_user_cfg.pop("groups", [])
# Pick any config + groups for the user name that we may have
# extracted previously
parsed_config = users.pop(def_user, {})
parsed_groups = parsed_config.get("groups", [])
# Now merge the extracted groups with the default config provided
users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
parsed_config["groups"] = ",".join(users_groups)
# The real config for the default user is the combination of the
# default user config provided by the distro, the default user
# config provided by the above merging for the user 'default' and
# then the parsed config from the user's 'real name' which does not
# have to be 'default' (but could be)
users[def_user] = util.mergemanydict(
[def_user_cfg, def_config, parsed_config])
# Ensure that only the default user that we found (if any) is actually
# marked as the default user
for uname, uconfig in users.items():
uconfig["default"] = uname == def_user if def_user else False
return users
def _normalize_users(u_cfg, def_user_cfg=None):
if isinstance(u_cfg, dict):
ad_ucfg = []
for (k, v) in u_cfg.items():
if isinstance(v, (bool, int, float) + six.string_types):
if util.is_true(v):
ad_ucfg.append(str(k))
elif isinstance(v, dict):
v['name'] = k
ad_ucfg.append(v)
else:
raise TypeError(("Unmappable user value type %s"
" for key %s") % (type_utils.obj_name(v), k))
u_cfg = ad_ucfg
elif isinstance(u_cfg, six.string_types):
u_cfg = util.uniq_merge_sorted(u_cfg)
users = {}
for user_config in u_cfg:
if isinstance(user_config, (list, ) + six.string_types):
for u in util.uniq_merge(user_config):
if u and u not in users:
users[u] = {}
elif isinstance(user_config, dict):
if 'name' in user_config:
n = user_config.pop('name')
prev_config = users.get(n) or {}
users[n] = util.mergemanydict([prev_config, user_config])
else:
# Assume the default user then
prev_config = users.get('default') or {}
users['default'] = util.mergemanydict(
[prev_config, user_config])
else:
raise TypeError(("User config must be dictionary/list "
" or string types only and not %s") %
type_utils.obj_name(user_config))
# Ensure user options are in the right python friendly format
if users:
c_users = {}
for (uname, uconfig) in users.items():
c_uconfig = {}
for (k, v) in uconfig.items():
k = k.replace('-', '_').strip()
if k:
c_uconfig[k] = v
c_users[uname] = c_uconfig
users = c_users
# Fixup the default user into the real
# default user name and replace it...
def_user = None
if users and 'default' in users:
def_config = users.pop('default')
if def_user_cfg:
# Pickup what the default 'real name' is
# and any groups that are provided by the
# default config
def_user_cfg = def_user_cfg.copy()
def_user = def_user_cfg.pop('name')
def_groups = def_user_cfg.pop('groups', [])
# Pickup any config + groups for that user name
# that we may have previously extracted
parsed_config = users.pop(def_user, {})
parsed_groups = parsed_config.get('groups', [])
# Now merge our extracted groups with
# anything the default config provided
users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
parsed_config['groups'] = ",".join(users_groups)
# The real config for the default user is the
# combination of the default user config provided
# by the distro, the default user config provided
# by the above merging for the user 'default' and
# then the parsed config from the user's 'real name'
# which does not have to be 'default' (but could be)
users[def_user] = util.mergemanydict(
[def_user_cfg, def_config, parsed_config])
# Ensure that only the default user that we
# found (if any) is actually marked as being
# the default user
if users:
for (uname, uconfig) in users.items():
if def_user and uname == def_user:
uconfig['default'] = True
else:
uconfig['default'] = False
return users
def handle(name, cfg, cloud, log, _args):
"""Enable and configure ntp."""
if 'ntp' not in cfg:
LOG.debug(
"Skipping module named %s, not present or disabled by cfg", name)
return
ntp_cfg = cfg['ntp']
if ntp_cfg is None:
ntp_cfg = {} # Allow empty config which will install the package
# TODO drop this when validate_cloudconfig_schema is strict=True
if not isinstance(ntp_cfg, (dict)):
raise RuntimeError(
"'ntp' key existed in config, but not a dictionary type,"
" is a {_type} instead".format(_type=type_utils.obj_name(ntp_cfg)))
validate_cloudconfig_schema(cfg, schema)
# Allow users to explicitly enable/disable
enabled = ntp_cfg.get('enabled', True)
if util.is_false(enabled):
LOG.debug("Skipping module named %s, disabled by cfg", name)
return
# Select which client is going to be used and get the configuration
ntp_client_config = select_ntp_client(ntp_cfg.get('ntp_client'),
cloud.distro)
# Allow user ntp config to override distro configurations
ntp_client_config = util.mergemanydict(
[ntp_client_config, ntp_cfg.get('config', {})], reverse=True)
supplemental_schema_validation(ntp_client_config)
rename_ntp_conf(confpath=ntp_client_config.get('confpath'))
template_fn = None
if not ntp_client_config.get('template'):
template_name = (
ntp_client_config.get('template_name').replace('{distro}',
cloud.distro.name))
template_fn = cloud.get_template_filename(template_name)
if not template_fn:
msg = ('No template found, not rendering %s' %
ntp_client_config.get('template_name'))
raise RuntimeError(msg)
write_ntp_config_template(cloud.distro.name,
servers=ntp_cfg.get('servers', []),
pools=ntp_cfg.get('pools', []),
path=ntp_client_config.get('confpath'),
template_fn=template_fn,
template=ntp_client_config.get('template'))
install_ntp_client(cloud.distro.install_packages,
packages=ntp_client_config['packages'],
check_exe=ntp_client_config['check_exe'])
try:
reload_ntp(ntp_client_config['service_name'],
systemd=cloud.distro.uses_systemd())
except util.ProcessExecutionError as e:
LOG.exception("Failed to reload/start ntp service: %s", e)
raise
def __str__(self):
return type_utils.obj_name(self)
def normalize_users_groups(cfg, distro):
if not cfg:
cfg = {}
# Handle the previous style of doing this where the first user
# overrides the concept of the default user if provided in the user: XYZ
# format.
old_user = {}
if "user" in cfg and cfg["user"]:
old_user = cfg["user"]
# Translate it into a format that will be more useful going forward
if isinstance(old_user, str):
old_user = {"name": old_user}
LOG.warning(
"DEPRECATED: 'user' of type string is deprecated and will"
" be removed in a future release. Use 'users' list instead.")
elif not isinstance(old_user, dict):
LOG.warning(
"Format for 'user' key must be a string or dictionary"
" and not %s",
type_utils.obj_name(old_user),
)
old_user = {}
# If no old user format, then assume the distro provides what the 'default'
# user maps to, but notice that if this is provided, we won't automatically
# inject a 'default' user into the users list, while if an old user format
# is provided we will.
distro_user_config = {}
try:
distro_user_config = distro.get_default_user()
except NotImplementedError:
LOG.warning("Distro has not implemented default user access. No "
"distribution provided default user will be normalized.")
# Merge the old user (which may just be an empty dict when not present)
# with the distro provided default user configuration so that the old user
# style picks up all the distribution specific attributes (if any)
default_user_config = util.mergemanydict([old_user, distro_user_config])
base_users = cfg.get("users", [])
if isinstance(base_users, (dict, str)):
LOG.warning(
"DEPRECATED: 'users' of type %s is deprecated and will be removed"
" in a future release. Use 'users' as a list.",
type(base_users),
)
elif not isinstance(base_users, (list)):
LOG.warning(
"Format for 'users' key must be a comma-separated string"
" or a dictionary or a list but found %s",
type_utils.obj_name(base_users),
)
base_users = []
if old_user:
# When 'user:' is provided, it should be made as the default user
if isinstance(base_users, list):
base_users.append({"name": "default"})
elif isinstance(base_users, dict):
base_users["default"] = dict(base_users).get("default", True)
elif isinstance(base_users, str):
base_users += ",default"
groups = {}
if "groups" in cfg:
groups = _normalize_groups(cfg["groups"])
users = _normalize_users(base_users, default_user_config)
return (users, groups)
def __str__(self):
return "<%s using file %r>" % (type_utils.obj_name(self), self.fn)
def handle(name, cfg, cloud, log, _args):
"""Enable and configure ntp."""
if "ntp" not in cfg:
LOG.debug("Skipping module named %s, not present or disabled by cfg",
name)
return
ntp_cfg = cfg["ntp"]
if ntp_cfg is None:
ntp_cfg = {} # Allow empty config which will install the package
# TODO drop this when validate_cloudconfig_schema is strict=True
if not isinstance(ntp_cfg, (dict)):
raise RuntimeError(
"'ntp' key existed in config, but not a dictionary type,"
" is a {_type} instead".format(_type=type_utils.obj_name(ntp_cfg)))
# Allow users to explicitly enable/disable
enabled = ntp_cfg.get("enabled", True)
if util.is_false(enabled):
LOG.debug("Skipping module named %s, disabled by cfg", name)
return
# Select which client is going to be used and get the configuration
ntp_client_config = select_ntp_client(ntp_cfg.get("ntp_client"),
cloud.distro)
# Allow user ntp config to override distro configurations
ntp_client_config = util.mergemanydict(
[ntp_client_config, ntp_cfg.get("config", {})], reverse=True)
supplemental_schema_validation(ntp_client_config)
rename_ntp_conf(confpath=ntp_client_config.get("confpath"))
template_fn = None
if not ntp_client_config.get("template"):
template_name = ntp_client_config.get("template_name").replace(
"{distro}", cloud.distro.name)
template_fn = cloud.get_template_filename(template_name)
if not template_fn:
msg = ("No template found, not rendering %s" %
ntp_client_config.get("template_name"))
raise RuntimeError(msg)
write_ntp_config_template(
cloud.distro.name,
service_name=ntp_client_config.get("service_name"),
servers=ntp_cfg.get("servers", []),
pools=ntp_cfg.get("pools", []),
path=ntp_client_config.get("confpath"),
template_fn=template_fn,
template=ntp_client_config.get("template"),
)
install_ntp_client(
cloud.distro.install_packages,
packages=ntp_client_config["packages"],
check_exe=ntp_client_config["check_exe"],
)
try:
cloud.distro.manage_service("reload",
ntp_client_config.get("service_name"))
except subp.ProcessExecutionError as e:
LOG.exception("Failed to reload/start ntp service: %s", e)
raise
def normalize_users_groups(cfg, distro):
if not cfg:
cfg = {}
users = {}
groups = {}
if 'groups' in cfg:
groups = _normalize_groups(cfg['groups'])
# Handle the previous style of doing this where the first user
# overrides the concept of the default user if provided in the user: XYZ
# format.
old_user = {}
if 'user' in cfg and cfg['user']:
old_user = cfg['user']
# Translate it into the format that is more useful
# going forward
if isinstance(old_user, six.string_types):
old_user = {
'name': old_user,
}
if not isinstance(old_user, dict):
LOG.warn(("Format for 'user' key must be a string or "
"dictionary and not %s"), type_utils.obj_name(old_user))
old_user = {}
# If no old user format, then assume the distro
# provides what the 'default' user maps to, but notice
# that if this is provided, we won't automatically inject
# a 'default' user into the users list, while if a old user
# format is provided we will.
distro_user_config = {}
try:
distro_user_config = distro.get_default_user()
except NotImplementedError:
LOG.warn(("Distro has not implemented default user "
"access. No distribution provided default user"
" will be normalized."))
# Merge the old user (which may just be an empty dict when not
# present with the distro provided default user configuration so
# that the old user style picks up all the distribution specific
# attributes (if any)
default_user_config = util.mergemanydict([old_user, distro_user_config])
base_users = cfg.get('users', [])
if not isinstance(base_users, (list, dict) + six.string_types):
LOG.warn(("Format for 'users' key must be a comma separated string"
" or a dictionary or a list and not %s"),
type_utils.obj_name(base_users))
base_users = []
if old_user:
# Ensure that when user: is provided that this user
# always gets added (as the default user)
if isinstance(base_users, list):
# Just add it on at the end...
base_users.append({'name': 'default'})
elif isinstance(base_users, dict):
base_users['default'] = dict(base_users).get('default', True)
elif isinstance(base_users, six.string_types):
# Just append it on to be re-parsed later
base_users += ",default"
users = _normalize_users(base_users, default_user_config)
return (users, groups)
def __repr__(self):
return "%s: [%s]" % (type_utils.obj_name(self), self.list_types())
def __str__(self):
return "<%s using file %r>" % (type_utils.obj_name(self), self.fn)
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
def_mnt_opts = "defaults,nobootwait"
uses_systemd = cloud.distro.uses_systemd()
if uses_systemd:
def_mnt_opts = "defaults,nofail,x-systemd.requires=cloud-init.service"
defvals = [None, None, "auto", def_mnt_opts, "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"],
["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
if "mounts" in cfg:
cfgmnt = cfg["mounts"]
LOG.debug("mounts configuration is %s", cfgmnt)
fstab_lines = []
fstab_devs = {}
fstab_removed = []
for line in util.load_file(FSTAB_PATH).splitlines():
if MNT_COMMENT in line:
fstab_removed.append(line)
continue
try:
toks = WS.split(line)
except Exception:
pass
fstab_devs[toks[0]] = line
fstab_lines.append(line)
for i in range(len(cfgmnt)):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warn("Mount option %s not a list, got a %s instead",
(i + 1), type_utils.obj_name(cfgmnt[i]))
continue
start = str(cfgmnt[i][0])
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized != start:
log.debug("changed %s => %s" % (start, sanitized))
if sanitized is None:
log.debug("Ignoring nonexistent named mount %s", start)
continue
elif sanitized in fstab_devs:
log.info("Device %s already defined in fstab: %s",
sanitized, fstab_devs[sanitized])
continue
cfgmnt[i][0] = sanitized
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
for j in range(len(cfgmnt[i])):
if cfgmnt[i][j] is None:
continue
else:
cfgmnt[i][j] = str(cfgmnt[i][j])
for i in range(len(cfgmnt)):
# fill in values with defaults from defvals above
for j in range(len(defvals)):
if len(cfgmnt[i]) <= j:
cfgmnt[i].append(defvals[j])
elif cfgmnt[i][j] is None:
cfgmnt[i][j] = defvals[j]
# if the second entry in the list is 'None' this
# clears all previous entries of that same 'fs_spec'
# (fs_spec is the first field in /etc/fstab, ie, that device)
if cfgmnt[i][1] is None:
for j in range(i):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
start = defmnt[0]
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized != start:
log.debug("changed default device %s => %s" % (start, sanitized))
if sanitized is None:
log.debug("Ignoring nonexistent default named mount %s", start)
continue
elif sanitized in fstab_devs:
log.debug("Device %s already defined in fstab: %s",
sanitized, fstab_devs[sanitized])
continue
defmnt[0] = sanitized
cfgmnt_has = False
for cfgm in cfgmnt:
if cfgm[0] == defmnt[0]:
cfgmnt_has = True
break
if cfgmnt_has:
log.debug(("Not including %s, already"
" previously included"), start)
continue
cfgmnt.append(defmnt)
# now, each entry in the cfgmnt list has all fstab values
# if the second field is None (not the string, the value) we skip it
actlist = []
for x in cfgmnt:
if x[1] is None:
log.debug("Skipping nonexistent device named %s", x[0])
else:
actlist.append(x)
swapret = handle_swapcfg(cfg.get('swap', {}))
if swapret:
actlist.append([swapret, "none", "swap", "sw", "0", "0"])
if len(actlist) == 0:
log.debug("No modifications to fstab needed")
return
cc_lines = []
needswap = False
dirs = []
for line in actlist:
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], MNT_COMMENT)
if line[2] == "swap":
needswap = True
if line[1].startswith("/"):
dirs.append(line[1])
cc_lines.append('\t'.join(line))
for d in dirs:
try:
util.ensure_dir(d)
except Exception:
util.logexc(log, "Failed to make '%s' config-mount", d)
sadds = [WS.sub(" ", n) for n in cc_lines]
sdrops = [WS.sub(" ", n) for n in fstab_removed]
sops = (["- " + drop for drop in sdrops if drop not in sadds] +
["+ " + add for add in sadds if add not in sdrops])
fstab_lines.extend(cc_lines)
contents = "%s\n" % ('\n'.join(fstab_lines))
util.write_file(FSTAB_PATH, contents)
activate_cmds = []
if needswap:
activate_cmds.append(["swapon", "-a"])
if len(sops) == 0:
log.debug("No changes to /etc/fstab made.")
else:
log.debug("Changes to fstab: %s", sops)
activate_cmds.append(["mount", "-a"])
if uses_systemd:
activate_cmds.append(["systemctl", "daemon-reload"])
fmt = "Activating swap and mounts with: %s"
for cmd in activate_cmds:
fmt = "Activate mounts: %s:" + ' '.join(cmd)
try:
util.subp(cmd)
log.debug(fmt, "PASS")
except util.ProcessExecutionError:
log.warn(fmt, "FAIL")
util.logexc(log, fmt, "FAIL")
def __str__(self):
return type_utils.obj_name(self)
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
def_mnt_opts = "defaults,nobootwait"
if cloud.distro.uses_systemd():
def_mnt_opts = "defaults,nofail,x-systemd.requires=cloud-init.service"
defvals = [None, None, "auto", def_mnt_opts, "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"],
["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
if "mounts" in cfg:
cfgmnt = cfg["mounts"]
for i in range(len(cfgmnt)):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warn("Mount option %s not a list, got a %s instead", (i + 1),
type_utils.obj_name(cfgmnt[i]))
continue
start = str(cfgmnt[i][0])
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized is None:
log.debug("Ignorming nonexistant named mount %s", start)
continue
if sanitized != start:
log.debug("changed %s => %s" % (start, sanitized))
cfgmnt[i][0] = sanitized
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
for j in range(len(cfgmnt[i])):
if cfgmnt[i][j] is None:
continue
else:
cfgmnt[i][j] = str(cfgmnt[i][j])
for i in range(len(cfgmnt)):
# fill in values with defaults from defvals above
for j in range(len(defvals)):
if len(cfgmnt[i]) <= j:
cfgmnt[i].append(defvals[j])
elif cfgmnt[i][j] is None:
cfgmnt[i][j] = defvals[j]
# if the second entry in the list is 'None' this
# clears all previous entries of that same 'fs_spec'
# (fs_spec is the first field in /etc/fstab, ie, that device)
if cfgmnt[i][1] is None:
for j in range(i):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
start = defmnt[0]
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized is None:
log.debug("Ignoring nonexistant default named mount %s", start)
continue
if sanitized != start:
log.debug("changed default device %s => %s" % (start, sanitized))
defmnt[0] = sanitized
cfgmnt_has = False
for cfgm in cfgmnt:
if cfgm[0] == defmnt[0]:
cfgmnt_has = True
break
if cfgmnt_has:
log.debug(("Not including %s, already"
" previously included"), start)
continue
cfgmnt.append(defmnt)
# now, each entry in the cfgmnt list has all fstab values
# if the second field is None (not the string, the value) we skip it
actlist = []
for x in cfgmnt:
if x[1] is None:
log.debug("Skipping non-existent device named %s", x[0])
else:
actlist.append(x)
swapret = handle_swapcfg(cfg.get('swap', {}))
if swapret:
actlist.append([swapret, "none", "swap", "sw", "0", "0"])
if len(actlist) == 0:
log.debug("No modifications to fstab needed.")
return
comment = "comment=cloudconfig"
cc_lines = []
needswap = False
dirs = []
for line in actlist:
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], comment)
if line[2] == "swap":
needswap = True
if line[1].startswith("/"):
dirs.append(line[1])
cc_lines.append('\t'.join(line))
fstab_lines = []
for line in util.load_file(FSTAB_PATH).splitlines():
try:
toks = WS.split(line)
if toks[3].find(comment) != -1:
continue
except Exception:
pass
fstab_lines.append(line)
fstab_lines.extend(cc_lines)
contents = "%s\n" % ('\n'.join(fstab_lines))
util.write_file(FSTAB_PATH, contents)
if needswap:
try:
util.subp(("swapon", "-a"))
except Exception:
util.logexc(log, "Activating swap via 'swapon -a' failed")
for d in dirs:
try:
util.ensure_dir(d)
except Exception:
util.logexc(log, "Failed to make '%s' config-mount", d)
try:
util.subp(("mount", "-a"))
except util.ProcessExecutionError:
util.logexc(log, "Activating mounts via 'mount -a' failed")
def handle(_name, cfg, cloud, log, _args):
# fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
def_mnt_opts = "defaults,nobootwait"
uses_systemd = cloud.distro.uses_systemd()
if uses_systemd:
def_mnt_opts = "defaults,nofail,x-systemd.requires=cloud-init.service"
defvals = [None, None, "auto", def_mnt_opts, "0", "2"]
defvals = cfg.get("mount_default_fields", defvals)
# these are our default set of mounts
defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"],
["swap", "none", "swap", "sw", "0", "0"]]
cfgmnt = []
if "mounts" in cfg:
cfgmnt = cfg["mounts"]
LOG.debug("mounts configuration is %s", cfgmnt)
fstab_lines = []
fstab_devs = {}
fstab_removed = []
for line in util.load_file(FSTAB_PATH).splitlines():
if MNT_COMMENT in line:
fstab_removed.append(line)
continue
try:
toks = WS.split(line)
except Exception:
pass
fstab_devs[toks[0]] = line
fstab_lines.append(line)
for i in range(len(cfgmnt)):
# skip something that wasn't a list
if not isinstance(cfgmnt[i], list):
log.warning("Mount option %s not a list, got a %s instead",
(i + 1), type_utils.obj_name(cfgmnt[i]))
continue
start = str(cfgmnt[i][0])
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized != start:
log.debug("changed %s => %s" % (start, sanitized))
if sanitized is None:
log.debug("Ignoring nonexistent named mount %s", start)
continue
elif sanitized in fstab_devs:
log.info("Device %s already defined in fstab: %s", sanitized,
fstab_devs[sanitized])
continue
cfgmnt[i][0] = sanitized
# in case the user did not quote a field (likely fs-freq, fs_passno)
# but do not convert None to 'None' (LP: #898365)
for j in range(len(cfgmnt[i])):
if cfgmnt[i][j] is None:
continue
else:
cfgmnt[i][j] = str(cfgmnt[i][j])
for i in range(len(cfgmnt)):
# fill in values with defaults from defvals above
for j in range(len(defvals)):
if len(cfgmnt[i]) <= j:
cfgmnt[i].append(defvals[j])
elif cfgmnt[i][j] is None:
cfgmnt[i][j] = defvals[j]
# if the second entry in the list is 'None' this
# clears all previous entries of that same 'fs_spec'
# (fs_spec is the first field in /etc/fstab, ie, that device)
if cfgmnt[i][1] is None:
for j in range(i):
if cfgmnt[j][0] == cfgmnt[i][0]:
cfgmnt[j][1] = None
# for each of the "default" mounts, add them only if no other
# entry has the same device name
for defmnt in defmnts:
start = defmnt[0]
sanitized = sanitize_devname(start, cloud.device_name_to_device, log)
if sanitized != start:
log.debug("changed default device %s => %s" % (start, sanitized))
if sanitized is None:
log.debug("Ignoring nonexistent default named mount %s", start)
continue
elif sanitized in fstab_devs:
log.debug("Device %s already defined in fstab: %s", sanitized,
fstab_devs[sanitized])
continue
defmnt[0] = sanitized
cfgmnt_has = False
for cfgm in cfgmnt:
if cfgm[0] == defmnt[0]:
cfgmnt_has = True
break
if cfgmnt_has:
log.debug(("Not including %s, already"
" previously included"), start)
continue
cfgmnt.append(defmnt)
# now, each entry in the cfgmnt list has all fstab values
# if the second field is None (not the string, the value) we skip it
actlist = []
for x in cfgmnt:
if x[1] is None:
log.debug("Skipping nonexistent device named %s", x[0])
else:
actlist.append(x)
swapret = handle_swapcfg(cfg.get('swap', {}))
if swapret:
actlist.append([swapret, "none", "swap", "sw", "0", "0"])
if len(actlist) == 0:
log.debug("No modifications to fstab needed")
return
cc_lines = []
needswap = False
need_mount_all = False
dirs = []
for line in actlist:
# write 'comment' in the fs_mntops, entry, claiming this
line[3] = "%s,%s" % (line[3], MNT_COMMENT)
if line[2] == "swap":
needswap = True
if line[1].startswith("/"):
dirs.append(line[1])
cc_lines.append('\t'.join(line))
mount_points = [
v['mountpoint'] for k, v in util.mounts().items() if 'mountpoint' in v
]
for d in dirs:
try:
util.ensure_dir(d)
except Exception:
util.logexc(log, "Failed to make '%s' config-mount", d)
# dirs is list of directories on which a volume should be mounted.
# If any of them does not already show up in the list of current
# mount points, we will definitely need to do mount -a.
if not need_mount_all and d not in mount_points:
need_mount_all = True
sadds = [WS.sub(" ", n) for n in cc_lines]
sdrops = [WS.sub(" ", n) for n in fstab_removed]
sops = (["- " + drop for drop in sdrops if drop not in sadds] +
["+ " + add for add in sadds if add not in sdrops])
fstab_lines.extend(cc_lines)
contents = "%s\n" % ('\n'.join(fstab_lines))
util.write_file(FSTAB_PATH, contents)
activate_cmds = []
if needswap:
activate_cmds.append(["swapon", "-a"])
if len(sops) == 0:
log.debug("No changes to /etc/fstab made.")
else:
log.debug("Changes to fstab: %s", sops)
need_mount_all = True
if need_mount_all:
activate_cmds.append(["mount", "-a"])
if uses_systemd:
activate_cmds.append(["systemctl", "daemon-reload"])
fmt = "Activating swap and mounts with: %s"
for cmd in activate_cmds:
fmt = "Activate mounts: %s:" + ' '.join(cmd)
try:
util.subp(cmd)
log.debug(fmt, "PASS")
except util.ProcessExecutionError:
log.warning(fmt, "FAIL")
util.logexc(log, fmt, "FAIL")
def normalize_users_groups(cfg, distro):
if not cfg:
cfg = {}
users = {}
groups = {}
if 'groups' in cfg:
groups = _normalize_groups(cfg['groups'])
# Handle the previous style of doing this where the first user
# overrides the concept of the default user if provided in the user: XYZ
# format.
old_user = {}
if 'user' in cfg and cfg['user']:
old_user = cfg['user']
# Translate it into the format that is more useful
# going forward
if isinstance(old_user, six.string_types):
old_user = {
'name': old_user,
}
if not isinstance(old_user, dict):
LOG.warn(("Format for 'user' key must be a string or "
"dictionary and not %s"), type_utils.obj_name(old_user))
old_user = {}
# If no old user format, then assume the distro
# provides what the 'default' user maps to, but notice
# that if this is provided, we won't automatically inject
# a 'default' user into the users list, while if a old user
# format is provided we will.
distro_user_config = {}
try:
distro_user_config = distro.get_default_user()
except NotImplementedError:
LOG.warn(("Distro has not implemented default user "
"access. No distribution provided default user"
" will be normalized."))
# Merge the old user (which may just be an empty dict when not
# present with the distro provided default user configuration so
# that the old user style picks up all the distribution specific
# attributes (if any)
default_user_config = util.mergemanydict([old_user, distro_user_config])
base_users = cfg.get('users', [])
if not isinstance(base_users, (list, dict) + six.string_types):
LOG.warn(("Format for 'users' key must be a comma separated string"
" or a dictionary or a list and not %s"),
type_utils.obj_name(base_users))
base_users = []
if old_user:
# Ensure that when user: is provided that this user
# always gets added (as the default user)
if isinstance(base_users, list):
# Just add it on at the end...
base_users.append({'name': 'default'})
elif isinstance(base_users, dict):
base_users['default'] = dict(base_users).get('default', True)
elif isinstance(base_users, six.string_types):
# Just append it on to be re-parsed later
base_users += ",default"
users = _normalize_users(base_users, default_user_config)
return (users, groups)
