def test_nostrict_with_missing(self):
my_env = self.full_env.copy()
del my_env['OUTPUT_FSTAB']
try:
util.load_command_environment(my_env, strict=False)
except KeyError as e:
self.fail("unexpected key error raised: %s" % e)
def test_strict_with_missing(self):
my_env = self.full_env.copy()
del my_env['OUTPUT_FSTAB']
del my_env['WORKING_DIR']
exc = None
try:
util.load_command_environment(my_env, strict=True)
except KeyError as e:
self.assertIn("OUTPUT_FSTAB", str(e))
self.assertIn("WORKING_DIR", str(e))
exc = e
self.assertTrue(exc)
def in_target_main(args):
if args.target is not None:
target = args.target
else:
state = util.load_command_environment()
target = state['target']
if args.target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
daemons = args.allow_daemons
if paths.target_path(args.target) == "/":
sys.stderr.write("WARN: Target is /, daemons are allowed.\n")
daemons = True
cmd = args.command_args
with util.ChrootableTarget(target, allow_daemons=daemons) as chroot:
exit = 0
if not args.interactive:
try:
chroot.subp(cmd, capture=args.capture)
except util.ProcessExecutionError as e:
exit = e.exit_code
else:
if chroot.target != "/":
cmd = ["chroot", chroot.target] + args.command_args
# in python 3.4 pty.spawn started returning a value.
# There, it is the status from os.waitpid. From testing (py3.6)
# that seemse to be exit_code * 256.
ret = pty.spawn(cmd) # pylint: disable=E1111
if ret is not None:
exit = int(ret / 256)
sys.exit(exit)
def apt_command(args):
""" Main entry point for curtin apt-config standalone command
This does not read the global config as handled by curthooks, but
instead one can specify a different "target" and a new cfg via --config
"""
cfg = config.load_command_config(args, {})
if args.target is not None:
target = args.target
else:
state = util.load_command_environment()
target = state['target']
if target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
apt_cfg = cfg.get("apt")
# if no apt config section is available, do nothing
if apt_cfg is not None:
LOG.debug("Handling apt to target %s with config %s", target, apt_cfg)
try:
with util.ChrootableTarget(target, sys_resolvconf=True):
handle_apt(apt_cfg, target)
except (RuntimeError, TypeError, ValueError, IOError):
LOG.exception("Failed to configure apt features '%s'", apt_cfg)
sys.exit(1)
else:
LOG.info("No apt config provided, skipping")
sys.exit(0)
def main():
state = util.load_command_environment()
target = state['target']
if target is None:
print("Target was not provided in the environment.")
sys.exit(1)
fstab = state['fstab']
if fstab is None:
print("/etc/fstab output was not provided in the environment.")
sys.exit(1)
bootmac = get_boot_mac()
if bootmac is None:
print("Unable to determine boot interface.")
sys.exit(1)
devices = get_block_devices(target)
if not devices:
print("Unable to find block device for: %s" % target)
sys.exit(1)
write_fstab(target, fstab)
update_grub_default(target, extra=get_extra_kernel_parameters())
grub2_mkconfig(target)
if util.is_uefi_bootable():
uefi_part = get_uefi_partition()
if uefi_part is None:
print('Unable to determine UEFI parition.')
sys.exit(1)
install_efi(target, uefi_part['device_path'])
else:
for dev in devices:
grub2_install(target, dev)
set_autorelabel(target)
write_network_config(target, bootmac)
def main():
state = util.load_command_environment()
target = state['target']
if target is None:
print("Target was not provided in the environment.")
sys.exit(1)
fstab = state['fstab']
if fstab is None:
print("/etc/fstab output was not provided in the environment.")
sys.exit(1)
bootmac = get_boot_mac()
if bootmac is None:
print("Unable to determine boot interface.")
sys.exit(1)
devices = get_block_devices(target)
if not devices:
print("Unable to find block device for: %s" % target)
sys.exit(1)
write_fstab(target, fstab)
grub_root = get_grub_root(target)
write_grub_conf(target, grub_root, extra=get_extra_kernel_parameters())
grub_install(target, grub_root)
set_autorelabel(target)
if state.get('config'):
cfg = config.load_config(state['config'])
else:
cfg = {}
handle_cloudconfig(cfg, target)
apply_networking(cfg, target, bootmac)
def apply_net_main(args):
# curtin apply_net [--net-state=/config/netstate.yml] [--target=/]
# [--net-config=/config/maas_net.yml]
state = util.load_command_environment()
log.basicConfig(stream=args.log_file, verbosity=1)
if args.target is not None:
state['target'] = args.target
if args.net_state is not None:
state['network_state'] = args.net_state
if args.net_config is not None:
state['network_config'] = args.net_config
if state['target'] is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
if not state['network_config'] and not state['network_state']:
sys.stderr.write("Must provide at least config or state\n")
sys.exit(2)
LOG.info('Applying network configuration')
apply_net(target=state['target'],
network_state=state['network_state'],
network_config=state['network_config'])
LOG.info('Applied network configuration successfully')
sys.exit(0)
def curthooks():
state = util.load_command_environment()
config = load_command_config({}, state)
target = state['target']
cloudbaseinit = get_cloudbaseinit_dir(target)
if target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
context = get_oauth_data(config)
local_scripts = os.path.join(
cloudbaseinit,
"LocalScripts",
)
networking = config.get("network")
if networking:
curtin_dir = os.path.join(target, "curtin")
networking_file = os.path.join(target, "network.json")
if os.path.isdir(curtin_dir):
networking_file = os.path.join(curtin_dir, "network.json")
with open(networking_file, "wb") as fd:
fd.write(json.dumps(networking, indent=2).encode('utf-8'))
license_key = config.get("license_key")
if license_key and len(license_key) > 0:
try:
license_script = CHANGE_LICENSE_TPL.format(
{"license_key": license_key})
os.makedirs(local_scripts)
licensekey_path = os.path.join(local_scripts,
"ChangeLicenseKey.ps1")
with open(licensekey_path, "w") as script:
script.write(license_script)
except Exception as err:
sys.stderr.write("Failed to write LocalScripts: %r", err)
cloudbase_init_cfg = os.path.join(cloudbaseinit, "conf",
"cloudbase-init.conf")
cloudbase_init_unattended_cfg = os.path.join(
cloudbaseinit, "conf", "cloudbase-init-unattend.conf")
if os.path.isfile(cloudbase_init_cfg) is False:
sys.stderr.write("Unable to find cloudbase-init.cfg.\n")
sys.exit(2)
cloudbase_init_values = CLOUDBASE_INIT_TEMPLATE.format(**context)
fp = open(cloudbase_init_cfg, 'a')
fp_u = open(cloudbase_init_unattended_cfg, 'a')
for i in cloudbase_init_values.splitlines():
fp.write("%s\r\n" % i)
fp_u.write("%s\r\n" % i)
fp.close()
fp_u.close()
def interfaces_custom(args):
state = util.load_command_environment()
cfg = config.load_command_config(args, state)
network_config = cfg.get('network', [])
if not network_config:
raise Exception("network configuration is required by mode '%s' "
"but not provided in the config file" % 'custom')
return {'network': network_config}
def curthooks():
state = util.load_command_environment()
config = load_command_config({}, state)
target = state['target']
cloudbaseinit = get_cloudbaseinit_dir(target)
if target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
context = get_oauth_data(config)
local_scripts = os.path.join(
cloudbaseinit,
"LocalScripts",
)
networking = config.get("network")
if networking:
networking_file = os.path.join(target, "network.json")
with open(networking_file, "wb") as fd:
fd.write(json.dumps(networking, indent=2).encode('utf-8'))
license_key = config.get("license_key")
if license_key and len(license_key) > 0:
try:
license_script = CHANGE_LICENSE_TPL.format({"license_key": license_key})
os.makedirs(local_scripts)
licensekey_path = os.path.join(local_scripts, "ChangeLicenseKey.ps1")
with open(licensekey_path, "w") as script:
script.write(license_script)
except Exception as err:
sys.stderr.write("Failed to write LocalScripts: %r", err)
cloudbase_init_cfg = os.path.join(
cloudbaseinit,
"conf",
"cloudbase-init.conf")
cloudbase_init_unattended_cfg = os.path.join(
cloudbaseinit,
"conf",
"cloudbase-init-unattend.conf")
if os.path.isfile(cloudbase_init_cfg) is False:
sys.stderr.write("Unable to find cloudbase-init.cfg.\n")
sys.exit(2)
cloudbase_init_values = CLOUDBASE_INIT_TEMPLATE.format(**context)
fp = open(cloudbase_init_cfg, 'a')
fp_u = open(cloudbase_init_unattended_cfg, 'a')
for i in cloudbase_init_values.splitlines():
fp.write("%s\r\n" % i)
fp_u.write("%s\r\n" % i)
fp.close()
fp_u.close()
def save_iscsi_config(iscsi_disk):
state = util.load_command_environment()
# A nodes directory will be created in the same directory as the
# fstab in the configuration. This will then be copied onto the
# system later
if state['fstab']:
target_nodes_location = target_nodes_directory(state, iscsi_disk)
shutil.copy(iscsi_disk.etciscsi_nodefile, target_nodes_location)
else:
LOG.info("fstab configuration is not present in environment, "
"so cannot locate an appropriate directory to write "
"iSCSI node file in so not writing iSCSI node file")
def extract(args):
if not args.target:
raise ValueError("Target must be defined or set in environment")
state = util.load_command_environment()
cfg = curtin.config.load_command_config(args, state)
sources = args.sources
target = args.target
if not sources:
if not cfg.get('sources'):
raise ValueError("'sources' must be on cmdline or in config")
sources = cfg.get('sources')
if isinstance(sources, dict):
sources = [sources[k] for k in sorted(sources.keys())]
sources = [util.sanitize_source(s) for s in sources]
LOG.debug("Installing sources: %s to target at %s" % (sources, target))
stack_prefix = state.get('report_stack_prefix', '')
for source in sources:
with events.ReportEventStack(
name=stack_prefix,
reporting_enabled=True,
level="INFO",
description="acquiring and extracting image from %s" %
source['uri']):
if source['type'].startswith('dd-'):
continue
if source['uri'].startswith("cp://"):
copy_to_target(source['uri'], target)
elif source['type'] == "fsimage":
extract_root_fsimage_url(source['uri'], target=target)
elif source['type'] == "fsimage-layered":
extract_root_layered_fsimage_url(source['uri'], target=target)
else:
extract_root_tgz_url(source['uri'], target=target)
if cfg.get('write_files'):
LOG.info("Applying write_files from config.")
write_files(cfg['write_files'], target)
else:
LOG.info("No write_files in config.")
sys.exit(0)
def install_missing_packages(cfg, target):
''' describe which operation types will require specific packages
'custom_config_key': {
'pkg1': ['op_name_1', 'op_name_2', ...]
}
'''
installed_packages = util.get_installed_packages(target)
needed_packages = set([
pkg for pkg in detect_required_packages(cfg)
if pkg not in installed_packages
])
arch_packages = {
's390x': [('s390-tools', 'zipl')],
}
for pkg, cmd in arch_packages.get(platform.machine(), []):
if not util.which(cmd, target=target):
if pkg not in needed_packages:
needed_packages.add(pkg)
# Filter out ifupdown network packages on netplan enabled systems.
has_netplan = ('nplan' in installed_packages
or 'netplan.io' in installed_packages)
if 'ifupdown' not in installed_packages and has_netplan:
drops = set(['bridge-utils', 'ifenslave', 'vlan'])
if needed_packages.union(drops):
LOG.debug("Skipping install of %s. Not needed on netplan system.",
needed_packages.union(drops))
needed_packages = needed_packages.difference(drops)
if needed_packages:
to_add = list(sorted(needed_packages))
state = util.load_command_environment()
with events.ReportEventStack(
name=state.get('report_stack_prefix'),
reporting_enabled=True,
level="INFO",
description="Installing packages on target system: " +
str(to_add)):
util.install_packages(to_add, target=target)
def main():
state = util.load_command_environment()
target = state['target']
if target is None:
print("Target was not provided in the environment.")
sys.exit(1)
config_f = state['config']
if config_f is None:
print("Config was not provided in the environment.")
sys.exit(1)
config = load_config(config_f)
debconf = get_maas_debconf_selections(config)
if debconf is None:
print("Failed to get the debconf_selections.")
sys.exit(1)
params = extract_maas_parameters(debconf)
datasource = get_datasource(**params)
write_datasource(target, datasource)
def main():
state = util.load_command_environment()
target = state['target']
if target is None:
print("Target was not provided in the environment.")
sys.exit(1)
config_f = state['config']
if config_f is None:
print("Config was not provided in the environment.")
sys.exit(1)
config = load_config(config_f)
debconf = get_maas_debconf_selections(config)
if debconf is None:
print("Failed to get the debconf_selections.")
sys.exit(1)
params = extract_maas_parameters(debconf)
write_cloudbase_init(target, params)
license_key = get_license_key(config)
if license_key is not None:
write_license_key_script(target, license_key)
def swap_main(args):
# curtin swap [--size=4G] [--target=/] [--fstab=/etc/fstab] [swap]
state = util.load_command_environment()
if args.target is not None:
state['target'] = args.target
if args.fstab is not None:
state['fstab'] = args.fstab
if state['target'] is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
size = args.size
if size is not None and size.lower() == "auto":
size = None
if size is not None:
try:
size = util.human2bytes(size)
except ValueError as e:
sys.stderr.write("%s\n" % e)
sys.exit(2)
if args.maxsize is not None:
args.maxsize = util.human2bytes(args.maxsize)
swap.setup_swapfile(target=state['target'],
fstab=state['fstab'],
swapfile=args.swapfile,
size=size,
maxsize=args.maxsize,
force=args.force)
sys.exit(2)
def install_grub_main(args):
state = util.load_command_environment()
if args.target is not None:
target = args.target
else:
target = state['target']
if target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
cfg = config.load_command_config(args, state)
stack_prefix = state.get('report_stack_prefix', '')
uefi = util.is_uefi_bootable()
grubcfg = cfg.get('grub')
with events.ReportEventStack(
name=stack_prefix,
reporting_enabled=True,
level="INFO",
description="Installing grub to target devices"):
install_grub(args.devices, target, uefi=uefi, grubcfg=grubcfg)
sys.exit(0)
def net_meta(args):
# curtin net-meta --devices connected dhcp
# curtin net-meta --devices configured dhcp
# curtin net-meta --devices netboot dhcp
# curtin net-meta --devices connected custom
# if network-config hook exists in target,
# we do not run the builtin
if util.run_hook_if_exists(args.target, 'network-config'):
sys.exit(0)
state = util.load_command_environment()
cfg = config.load_command_config(args, state)
if cfg.get("network") is not None:
args.mode = "custom"
eni = "etc/network/interfaces"
if args.mode == "auto":
if not args.devices:
args.devices = ["connected"]
t_eni = None
if args.target:
t_eni = os.path.sep.join((
args.target,
eni,
))
if not os.path.isfile(t_eni):
t_eni = None
if t_eni:
args.mode = "copy"
else:
args.mode = "dhcp"
devices = []
if args.devices:
for dev in args.devices:
if dev in DEVNAME_ALIASES:
devices += resolve_alias(dev)
else:
devices.append(dev)
LOG.debug("net-meta mode is '%s'. devices=%s", args.mode, devices)
output_network_config = os.environ.get("OUTPUT_NETWORK_CONFIG", "")
if args.mode == "copy":
if not args.target:
raise argparse.ArgumentTypeError("mode 'copy' requires --target")
t_eni = os.path.sep.join((
args.target,
"etc/network/interfaces",
))
with open(t_eni, "r") as fp:
content = fp.read()
LOG.warn(
"net-meta mode is 'copy', static network interfaces files"
"can be brittle. Copied interfaces: %s", content)
target = args.output
elif args.mode == "dhcp":
target = output_network_config
content = config.dump_config(interfaces_basic_dhcp(devices))
elif args.mode == 'custom':
target = output_network_config
content = config.dump_config(interfaces_custom(args))
else:
raise Exception("Unexpected network config mode '%s'." % args.mode)
if not target:
raise Exception(
"No target given for mode = '%s'. No where to write content: %s" %
(args.mode, content))
LOG.debug("writing to file %s with network config: %s", target, content)
if target == "-":
sys.stdout.write(content)
else:
with open(target, "w") as fp:
fp.write(content)
sys.exit(0)
def curthooks(args):
state = util.load_command_environment()
if args.target is not None:
target = args.target
else:
target = state['target']
if target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
cfg = config.load_command_config(args, state)
stack_prefix = state.get('report_stack_prefix', '')
# if curtin-hooks hook exists in target we can defer to the in-target hooks
if util.run_hook_if_exists(target, 'curtin-hooks'):
# For vmtests to force execute centos_apply_network_config, uncomment
# the value in examples/tests/centos_defaults.yaml
if cfg.get('_ammend_centos_curthooks'):
if cfg.get('cloudconfig'):
handle_cloudconfig(cfg['cloudconfig'],
base_dir=util.target_path(
target, 'etc/cloud/cloud.cfg.d'))
if target_is_centos(target) or target_is_rhel(target):
LOG.info('Detected RHEL/CentOS image, running extra hooks')
with events.ReportEventStack(
name=stack_prefix,
reporting_enabled=True,
level="INFO",
description="Configuring CentOS for first boot"):
centos_apply_network_config(cfg.get('network', {}), target)
sys.exit(0)
if target_is_ubuntu_core(target):
LOG.info('Detected Ubuntu-Core image, running hooks')
with events.ReportEventStack(
name=stack_prefix,
reporting_enabled=True,
level="INFO",
description="Configuring Ubuntu-Core for first boot"):
ubuntu_core_curthooks(cfg, target)
sys.exit(0)
with events.ReportEventStack(
name=stack_prefix + '/writing-config',
reporting_enabled=True,
level="INFO",
description="configuring apt configuring apt"):
do_apt_config(cfg, target)
disable_overlayroot(cfg, target)
# LP: #1742560 prevent zfs-dkms from being installed (Xenial)
if util.lsb_release(target=target)['codename'] == 'xenial':
util.apt_update(target=target)
with util.ChrootableTarget(target) as in_chroot:
in_chroot.subp(['apt-mark', 'hold', 'zfs-dkms'])
# packages may be needed prior to installing kernel
with events.ReportEventStack(name=stack_prefix +
'/installing-missing-packages',
reporting_enabled=True,
level="INFO",
description="installing missing packages"):
install_missing_packages(cfg, target)
# If a /etc/iscsi/nodes/... file was created by block_meta then it
# needs to be copied onto the target system
nodes_location = os.path.join(os.path.split(state['fstab'])[0], "nodes")
if os.path.exists(nodes_location):
copy_iscsi_conf(nodes_location, target)
# do we need to reconfigure open-iscsi?
# If a mdadm.conf file was created by block_meta than it needs to be copied
# onto the target system
mdadm_location = os.path.join(
os.path.split(state['fstab'])[0], "mdadm.conf")
if os.path.exists(mdadm_location):
copy_mdadm_conf(mdadm_location, target)
# as per https://bugs.launchpad.net/ubuntu/+source/mdadm/+bug/964052
# reconfigure mdadm
util.subp(['dpkg-reconfigure', '--frontend=noninteractive', 'mdadm'],
data=None,
target=target)
with events.ReportEventStack(name=stack_prefix + '/installing-kernel',
reporting_enabled=True,
level="INFO",
description="installing kernel"):
setup_zipl(cfg, target)
install_kernel(cfg, target)
run_zipl(cfg, target)
restore_dist_interfaces(cfg, target)
with events.ReportEventStack(name=stack_prefix + '/setting-up-swap',
reporting_enabled=True,
level="INFO",
description="setting up swap"):
add_swap(cfg, target, state.get('fstab'))
with events.ReportEventStack(name=stack_prefix +
'/apply-networking-config',
reporting_enabled=True,
level="INFO",
description="apply networking config"):
apply_networking(target, state)
with events.ReportEventStack(name=stack_prefix + '/writing-etc-fstab',
reporting_enabled=True,
level="INFO",
description="writing etc/fstab"):
copy_fstab(state.get('fstab'), target)
with events.ReportEventStack(name=stack_prefix + '/configuring-multipath',
reporting_enabled=True,
level="INFO",
description="configuring multipath"):
detect_and_handle_multipath(cfg, target)
with events.ReportEventStack(
name=stack_prefix + '/system-upgrade',
reporting_enabled=True,
level="INFO",
description="updating packages on target system"):
system_upgrade(cfg, target)
with events.ReportEventStack(
name=stack_prefix + '/pollinate-user-agent',
reporting_enabled=True,
level="INFO",
description="configuring pollinate user-agent on target system"):
handle_pollinate_user_agent(cfg, target)
# If a crypttab file was created by block_meta than it needs to be copied
# onto the target system, and update_initramfs() needs to be run, so that
# the cryptsetup hooks are properly configured on the installed system and
# it will be able to open encrypted volumes at boot.
crypttab_location = os.path.join(
os.path.split(state['fstab'])[0], "crypttab")
if os.path.exists(crypttab_location):
copy_crypttab(crypttab_location, target)
update_initramfs(target)
# If udev dname rules were created, copy them to target
udev_rules_d = os.path.join(state['scratch'], "rules.d")
if os.path.isdir(udev_rules_d):
copy_dname_rules(udev_rules_d, target)
# As a rule, ARMv7 systems don't use grub. This may change some
# day, but for now, assume no. They do require the initramfs
# to be updated, and this also triggers boot loader setup via
# flash-kernel.
machine = platform.machine()
if (machine.startswith('armv7') or machine.startswith('s390x')
or machine.startswith('aarch64') and not util.is_uefi_bootable()):
update_initramfs(target)
else:
setup_grub(cfg, target)
sys.exit(0)
def test_full_and_strict(self):
try:
util.load_command_environment(self.full_env, strict=False)
except KeyError as e:
self.fail("unexpected key error raised: %s" % e)
