def reset_cleansers(confirm=True):
"""destroys all cleanser slaves and their rollback snapshots, as well as the initial master
snapshot - this allows re-running the jailhost deployment to recreate fresh cleansers."""
if value_asbool(confirm) and not yesno("""\nObacht!
This will destroy any existing and or currently running cleanser jails.
Are you sure that you want to continue?"""):
exit("Glad I asked...")
get_vars()
cleanser_count = AV['ploy_cleanser_count']
# make sure no workers interfere:
fab.run('ezjail-admin stop worker')
# stop and nuke the cleanser slaves
for cleanser_index in range(cleanser_count):
cindex = '{:02d}'.format(cleanser_index + 1)
fab.run('ezjail-admin stop cleanser_{cindex}'.format(cindex=cindex))
with fab.warn_only():
fab.run('zfs destroy tank/jails/cleanser_{cindex}@jdispatch_rollback'.format(cindex=cindex))
fab.run('ezjail-admin delete -fw cleanser_{cindex}'.format(cindex=cindex))
fab.run('umount -f /usr/jails/cleanser_{cindex}'.format(cindex=cindex))
fab.run('rm -rf /usr/jails/cleanser_{cindex}'.format(cindex=cindex))
with fab.warn_only():
# remove master snapshot
fab.run('zfs destroy -R tank/jails/cleanser@clonesource')
# restart worker and cleanser to prepare for subsequent ansible configuration runs
fab.run('ezjail-admin start worker')
fab.run('ezjail-admin stop cleanser')
fab.run('ezjail-admin start cleanser')
def cmd_import(self, args, src):
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
return
cmd = ['gpg', '--quiet', '--no-tty', '--decrypt']
cmd.extend(self.gpg_opts)
cmd.extend([args.file])
key = subprocess.check_output(cmd)
src.set(key)
def cmd_import(self, args, src):
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
return
cmd = ['gpg', '--quiet', '--no-tty', '--decrypt']
cmd.extend(self.gpg_opts)
cmd.extend([args.file])
key = subprocess.check_output(cmd)
src.set(key)
def cmd_generate(self, args, src):
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
sys.exit(1)
key = b2a_base64(os.urandom(32))
key = key.strip()
key = key.replace('+', '-')
key = key.replace('/', '_')
src.set(key)
def cmd_generate(self, args, src):
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
sys.exit(1)
key = b2a_base64(os.urandom(32))
key = key.strip()
key = key.replace('+', '-')
key = key.replace('/', '_')
src.set(key)
def __call__(self, argv, help):
"""Manage vault keys."""
parser = argparse.ArgumentParser(
prog="%s vault-key" % self.ctrl.progname,
description=help)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'-g', '--generate',
action="store_true",
help="generate a new random 32 byte vault key and store it")
group.add_argument(
'-s', '--set',
action="store_true",
help="set the vault key")
group.add_argument(
'-d', '--delete',
action="store_true",
help="delete the vault key")
parser.add_argument(
'-o', '--old',
action="store_true",
help="use 'vault-password-old-source'")
args = parser.parse_args(argv)
if args.old:
src = get_vault_password_source(self.ctrl.config, option='vault-password-old-source')
else:
src = get_vault_password_source(self.ctrl.config)
if args.generate:
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
sys.exit(1)
from binascii import b2a_base64
key = b2a_base64(os.urandom(32))
key = key.strip()
key = key.replace('+', '-')
key = key.replace('/', '_')
src.set(key)
elif args.set:
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
sys.exit(1)
import getpass
src.set(getpass.getpass("Password for '%s': " % src.id))
elif args.delete:
if yesno("Do you really want to delete the key for '%s'?" % src.id):
src.delete()
def test_yesno(default, all, question, answer, expected):
from ploy.common import yesno
raw_input_values = answer
def get_input_result(q):
assert q == question
a = raw_input_values.pop()
print(q, repr(a))
return a
with patch('ploy.common.get_input') as RawInput:
RawInput.side_effect = get_input_result
try:
assert yesno('Foo', default, all) == expected
except Exception as e:
assert type(e) == expected
def test_yesno(default, all, question, answer, expected):
from ploy.common import yesno
raw_input_values = answer
def get_input_result(q):
assert q == question
a = raw_input_values.pop()
print(q, repr(a))
return a
with patch('ploy.common.get_input') as RawInput:
RawInput.side_effect = get_input_result
try:
assert yesno('Foo', default, all) == expected
except Exception as e:
assert type(e) == expected
def ensure(self, instance):
dhcpservers = instance.vb.list('dhcpservers')
name = "HostInterfaceNetworking-%s" % self.name
kw = {}
for key in ('ip', 'netmask', 'lowerip', 'upperip'):
if key not in self.config:
log.error("The '%s' option is required for dhcpserver '%s'." % (key, self.name))
sys.exit(1)
kw[key] = self.config[key]
if name not in dhcpservers:
try:
instance.vb.dhcpserver('add', '--enable', netname=name, **kw)
except subprocess.CalledProcessError as e:
log.error("Failed to add dhcpserver '%s':\n%s" % (self.name, e))
sys.exit(1)
log.info("Added dhcpserver '%s'." % self.name)
dhcpserver = instance.vb.list('dhcpservers')[name]
matches = True
if 'ip' in self.config:
if dhcpserver['IP'] != self.config['ip']:
log.error("The host only interface '%s' has an IP '%s' that doesn't match the config '%s'." % (
self.name, dhcpserver['IP'], self.config['ip']))
matches = False
if 'netmask' in self.config:
if dhcpserver['NetworkMask'] != self.config['netmask']:
log.error("The host only interface '%s' has an netmask '%s' that doesn't match the config '%s'." % (
self.name, dhcpserver['NetworkMask'], self.config['netmask']))
matches = False
if 'lower-ip' in self.config:
if dhcpserver['lowerIPAddress'] != self.config['lower-ip']:
log.error("The host only interface '%s' has a lower IP '%s' that doesn't match the config '%s'." % (
self.name, dhcpserver['lowerIPAddress'], self.config['lower-ip']))
matches = False
if 'upper-ip' in self.config:
if dhcpserver['upperIPAddress'] != self.config['upper-ip']:
log.error("The host only interface '%s' has a upper IP '%s' that doesn't match the config '%s'." % (
self.name, dhcpserver['upperIPAddress'], self.config['upper-ip']))
matches = False
if not matches:
if not yesno("Should the dhcpserver '%s' be modified to match the config?" % self.name):
sys.exit(1)
try:
instance.vb.dhcpserver('modify', '--enable', netname=name, **kw)
except subprocess.CalledProcessError as e:
log.error("Failed to modify dhcpserver '%s':\n%s" % (self.name, e))
sys.exit(1)
def download_remote(self, url, sha_checksum=None):
def check(path, sha):
d = hashlib.sha1()
with open(path, 'rb') as f:
while 1:
buf = f.read(1024 * 1024)
if not len(buf):
break
d.update(buf)
return d.hexdigest() == sha
download_dir = os.path.expanduser(
self.master.main_config.get('global',
dict()).get('download_dir',
'~/.ploy/downloads'))
if not os.path.exists(download_dir):
os.makedirs(download_dir, mode=0o750)
path, filename = os.path.split(url.path)
local_path = os.path.join(download_dir, filename)
if sha_checksum is None:
if not yesno(
'No checksum provided! Are you sure you want to boot from an unverified image?'
):
sys.exit(1)
if os.path.exists(local_path):
if sha_checksum is None or check(local_path, sha_checksum):
return local_path
else:
log.error('Checksum mismatch for %s!' % local_path)
sys.exit(1)
log.info("Downloading remote disk image from %s to %s" %
(url.geturl(), local_path))
urllib.urlretrieve(url.geturl(), local_path)
log.info('Downloaded successfully to %s' % local_path)
if sha_checksum is not None and not check(local_path, sha_checksum):
log.error('Checksum mismatch!')
sys.exit(1)
return local_path
def cmd_terminate(self, argv, help):
"""Terminates the instance"""
from ploy.common import yesno
parser = argparse.ArgumentParser(
prog="%s terminate" % self.progname,
description=help,
)
instances = self.get_instances(command='terminate')
parser.add_argument("instance", nargs=1,
metavar="instance",
help="Name of the instance from the config.",
choices=sorted(instances))
args = parser.parse_args(argv)
instance = instances[args.instance[0]]
if not yesno("Are you sure you want to terminate '%s'?" % instance.config_id):
return
instance.hooks.before_terminate(instance)
instance.terminate()
instance.hooks.after_terminate(instance)
def cmd_terminate(self, argv, help):
"""Terminates the instance"""
from ploy.common import yesno
parser = argparse.ArgumentParser(
prog="%s terminate" % self.progname,
description=help,
)
instances = self.get_instances(command='terminate')
parser.add_argument("instance", nargs=1,
metavar="instance",
help="Name of the instance from the config.",
choices=sorted(instances))
args = parser.parse_args(argv)
instance = instances[args.instance[0]]
if not yesno("Are you sure you want to terminate '%s'?" % instance.config_id):
return
instance.hooks.before_terminate(instance)
instance.terminate()
instance.hooks.after_terminate(instance)
def reset_jails(confirm=True, keep_cleanser_master=True):
""" stops, deletes and re-creates all jails.
since the cleanser master is rather large, that one is omitted by default.
"""
if value_asbool(confirm) and not yesno("""\nObacht!
This will destroy all existing and or currently running jails on the host.
Are you sure that you want to continue?"""):
exit("Glad I asked...")
reset_cleansers(confirm=False)
jails = ['appserver', 'webserver', 'worker']
if not value_asbool(keep_cleanser_master):
jails.append('cleanser')
with fab.warn_only():
for jail in jails:
fab.run('ezjail-admin delete -fw {jail}'.format(jail=jail))
# remove authorized keys for no longer existing key (they are regenerated for each new worker)
fab.run('rm /usr/jails/cleanser/usr/home/cleanser/.ssh/authorized_keys')
def reset_jails(confirm=True, keep_cleanser_master=True):
""" stops, deletes and re-creates all jails.
since the cleanser master is rather large, that one is omitted by default.
"""
if value_asbool(confirm) and not yesno("""\nObacht!
This will destroy all existing and or currently running jails on the host.
Are you sure that you want to continue?"""):
exit("Glad I asked...")
reset_cleansers(confirm=False)
jails = ['appserver', 'webserver', 'worker']
if not value_asbool(keep_cleanser_master):
jails.append('cleanser')
with fab.warn_only():
for jail in jails:
fab.run('ezjail-admin delete -fw {jail}'.format(jail=jail))
# remove authorized keys for no longer existing key (they are regenerated for each new worker)
fab.run(
'rm /usr/jails/cleanser/usr/home/cleanser/.ssh/authorized_keys')
def download_remote(self, url, sha_checksum=None):
def check(path, sha):
d = hashlib.sha1()
with open(path, 'rb') as f:
while 1:
buf = f.read(1024 * 1024)
if not len(buf):
break
d.update(buf)
return d.hexdigest() == sha
download_dir = os.path.expanduser(self.master.main_config.get('global', dict()).get(
'download_dir', '~/.ploy/downloads'))
if not os.path.exists(download_dir):
os.makedirs(download_dir, mode=0o750)
path, filename = os.path.split(url.path)
local_path = os.path.join(download_dir, filename)
if sha_checksum is None:
if not yesno('No checksum provided! Are you sure you want to boot from an unverified image?'):
sys.exit(1)
if os.path.exists(local_path):
if sha_checksum is None or check(local_path, sha_checksum):
return local_path
else:
log.error('Checksum mismatch for %s!' % local_path)
sys.exit(1)
log.info("Downloading remote disk image from %s to %s" % (url.geturl(), local_path))
urllib.urlretrieve(url.geturl(), local_path)
log.info('Downloaded successfully to %s' % local_path)
if sha_checksum is not None and not check(local_path, sha_checksum):
log.error('Checksum mismatch!')
sys.exit(1)
return local_path
def reset_cleansers(confirm=True):
"""destroys all cleanser slaves and their rollback snapshots, as well as the initial master
snapshot - this allows re-running the jailhost deployment to recreate fresh cleansers."""
if value_asbool(confirm) and not yesno("""\nObacht!
This will destroy any existing and or currently running cleanser jails.
Are you sure that you want to continue?"""):
exit("Glad I asked...")
get_vars()
cleanser_count = AV['ploy_cleanser_count']
# make sure no workers interfere:
fab.run('ezjail-admin stop worker')
# stop and nuke the cleanser slaves
for cleanser_index in range(cleanser_count):
cindex = '{:02d}'.format(cleanser_index + 1)
fab.run('ezjail-admin stop cleanser_{cindex}'.format(cindex=cindex))
with fab.warn_only():
fab.run(
'zfs destroy tank/jails/cleanser_{cindex}@jdispatch_rollback'.
format(cindex=cindex))
fab.run('ezjail-admin delete -fw cleanser_{cindex}'.format(
cindex=cindex))
fab.run(
'umount -f /usr/jails/cleanser_{cindex}'.format(cindex=cindex))
fab.run(
'rm -rf /usr/jails/cleanser_{cindex}'.format(cindex=cindex))
with fab.warn_only():
# remove master snapshot
fab.run('zfs destroy -R tank/jails/cleanser@clonesource')
# restart worker and cleanser to prepare for subsequent ansible configuration runs
fab.run('ezjail-admin start worker')
fab.run('ezjail-admin stop cleanser')
fab.run('ezjail-admin start cleanser')
def cmd_set(self, args, src):
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
return
src.set(getpass.getpass("Password for '%s': " % src.id))
def bootstrap_files(self):
""" we need some files to bootstrap the FreeBSD installation.
Some...
- need to be provided by the user (i.e. authorized_keys)
- others have some (sensible) defaults (i.e. rc.conf)
- some can be downloaded via URL (i.e.) http://pkg.freebsd.org/freebsd:10:x86:64/latest/Latest/pkg.txz
For those which can be downloaded we check the downloads directory. if the file exists there
(and if the checksum matches TODO!) we will upload it to the host. If not, we will fetch the file
from the given URL from the host.
For files that cannot be downloaded (authorized_keys, rc.conf etc.) we allow the user to provide their
own version in a ``bootstrap-files`` folder. The location of this folder can either be explicitly provided
via the ``bootstrap-files`` key in the host definition of the config file or it defaults to ``deployment/bootstrap-files``.
User provided files can be rendered as Jinja2 templates, by providing ``use_jinja: True`` in the YAML file.
They will be rendered with the instance configuration dictionary as context.
If the file is not found there, we revert to the default
files that are part of bsdploy. If the file cannot be found there either we either error out or for authorized_keys
we look in ``~/.ssh/identity.pub``.
"""
bootstrap_file_yamls = [
abspath(join(self.default_template_path, self.bootstrap_files_yaml)),
abspath(join(self.custom_template_path, self.bootstrap_files_yaml)),
]
bootstrap_files = dict()
if self.upload_authorized_keys:
bootstrap_files["authorized_keys"] = BootstrapFile(
self,
"authorized_keys",
**{
"directory": "/mnt/root/.ssh",
"directory_mode": "0600",
"remote": "/mnt/root/.ssh/authorized_keys",
"fallback": [
"~/.ssh/identity.pub",
"~/.ssh/id_rsa.pub",
"~/.ssh/id_dsa.pub",
"~/.ssh/id_ecdsa.pub",
],
}
)
for bootstrap_file_yaml in bootstrap_file_yamls:
if not exists(bootstrap_file_yaml):
continue
with open(bootstrap_file_yaml) as f:
info = yaml.load(f, Loader=SafeLoader)
if info is None:
continue
for k, v in info.items():
bootstrap_files[k] = BootstrapFile(self, k, **v)
for bf in bootstrap_files.values():
if not exists(bf.local) and bf.raw_fallback:
if not bf.existing_fallback:
print(
"Found no public key in %s, you have to create '%s' manually" % (expanduser("~/.ssh"), bf.local)
)
sys.exit(1)
print("The '%s' file is missing." % bf.local)
for path in bf.existing_fallback:
yes = env.instance.config.get("bootstrap-yes", False)
if yes or yesno("Should we generate it using the key in '%s'?" % path):
if not exists(bf.expected_path):
os.mkdir(bf.expected_path)
with open(bf.local, "wb") as out:
with open(path, "rb") as f:
out.write(f.read())
break
else:
# answered no to all options
sys.exit(1)
if not bf.check():
print("Cannot find %s" % bf.local)
sys.exit(1)
packages_path = join(self.download_path, "packages")
if exists(packages_path):
for dirpath, dirnames, filenames in os.walk(packages_path):
path = dirpath.split(packages_path)[1][1:]
for filename in filenames:
if not filename.endswith(".txz"):
continue
bootstrap_files[join(path, filename)] = BootstrapFile(
self,
join(path, filename),
**dict(
local=join(packages_path, join(path, filename)),
remote=join("/mnt/var/cache/pkg/All", filename),
)
)
if self.ssh_keys is not None:
for ssh_key_name, ssh_key_options in list(self.ssh_keys):
ssh_key = join(self.custom_template_path, ssh_key_name)
if exists(ssh_key):
pub_key_name = "%s.pub" % ssh_key_name
pub_key = "%s.pub" % ssh_key
if not exists(pub_key):
print("Public key '%s' for '%s' missing." % (pub_key, ssh_key))
sys.exit(1)
bootstrap_files[ssh_key_name] = BootstrapFile(
self, ssh_key_name, **dict(local=ssh_key, remote="/mnt/etc/ssh/%s" % ssh_key_name, mode=0600)
)
bootstrap_files[pub_key_name] = BootstrapFile(
self, pub_key_name, **dict(local=pub_key, remote="/mnt/etc/ssh/%s" % pub_key_name, mode=0644)
)
return bootstrap_files
def bootstrap(**kwargs):
""" bootstrap an instance booted into mfsbsd (http://mfsbsd.vx.sk)
"""
env.shell = '/bin/sh -c'
# default ssh settings for mfsbsd with possible overwrite by bootstrap-fingerprint
fingerprint = env.instance.config.get(
'bootstrap-fingerprint',
'02:2e:b4:dd:c3:8a:b7:7b:ba:b2:4a:f0:ab:13:f4:2d')
env.instance.config['fingerprint'] = fingerprint
env.instance.config['password-fallback'] = True
env.instance.config['password'] = '******'
# allow overwrites from the commandline
env.instance.config.update(kwargs)
bu = BootstrapUtils()
bu.generate_ssh_keys()
bu.print_bootstrap_files()
# gather infos
if not bu.bsd_url:
print("Found no FreeBSD system to install, please specify bootstrap-bsd-url and make sure mfsbsd is running")
return
# get realmem here, because it may fail and we don't want that to happen
# in the middle of the bootstrap
realmem = bu.realmem
print("\nFound the following disk devices on the system:\n %s" % ' '.join(bu.sysctl_devices))
if bu.first_interface:
print("\nFound the following network interfaces, now is your chance to update your rc.conf accordingly!\n %s" % ' '.join(bu.phys_interfaces))
else:
print("\nWARNING! Found no suitable network interface!")
template_context = {}
# first the config, so we don't get something essential overwritten
template_context.update(env.instance.config)
template_context.update(
devices=bu.sysctl_devices,
interfaces=bu.phys_interfaces,
hostname=env.instance.id)
rc_conf = bu.bootstrap_files['rc.conf'].read(template_context)
if not rc_conf.endswith('\n'):
print("\nERROR! Your rc.conf doesn't end in a newline:\n==========\n%s<<<<<<<<<<\n" % rc_conf)
return
rc_conf_lines = rc_conf.split('\n')
for interface in [bu.first_interface, env.instance.config.get('ansible-dhcp_host_sshd_interface')]:
if interface is None:
continue
ifconfig = 'ifconfig_%s' % interface
for line in rc_conf_lines:
if line.strip().startswith(ifconfig):
break
else:
if not yesno("\nDidn't find an '%s' setting in rc.conf. You sure that you want to continue?" % ifconfig):
return
yes = env.instance.config.get('bootstrap-yes', False)
if not (yes or yesno("\nContinuing will destroy the existing data on the following devices:\n %s\n\nContinue?" % ' '.join(bu.devices))):
return
# install FreeBSD in ZFS root
devices_args = ' '.join('-d %s' % x for x in bu.devices)
system_pool_name = env.instance.config.get('bootstrap-system-pool-name', 'system')
data_pool_name = env.instance.config.get('bootstrap-data-pool-name', 'tank')
swap_arg = ''
swap_size = env.instance.config.get('bootstrap-swap-size', '%iM' % (realmem * 2))
if swap_size:
swap_arg = '-s %s' % swap_size
system_pool_arg = ''
system_pool_size = env.instance.config.get('bootstrap-system-pool-size', '20G')
if system_pool_size:
system_pool_arg = '-z %s' % system_pool_size
run('destroygeom {devices_args} -p {system_pool_name} -p {data_pool_name}'.format(
devices_args=devices_args,
system_pool_name=system_pool_name,
data_pool_name=data_pool_name))
run('{zfsinstall} {devices_args} -p {system_pool_name} -V 28 -u {bsd_url} {swap_arg} {system_pool_arg}'.format(
zfsinstall=bu.zfsinstall,
devices_args=devices_args,
system_pool_name=system_pool_name,
bsd_url=bu.bsd_url,
swap_arg=swap_arg,
system_pool_arg=system_pool_arg))
# create partitions for data pool, but only if the system pool doesn't use
# the whole disk anyway
if system_pool_arg:
for device in bu.devices:
run('gpart add -t freebsd-zfs -l {data_pool_name}_{device} {device}'.format(
data_pool_name=data_pool_name,
device=device))
# mount devfs inside the new system
if 'devfs on /rw/dev' not in bu.mounts:
run('mount -t devfs devfs /mnt/dev')
# setup bare essentials
run('cp /etc/resolv.conf /mnt/etc/resolv.conf')
bu.create_bootstrap_directories()
bu.upload_bootstrap_files(template_context)
# we need to install python here, because there is no way to install it via
# ansible playbooks
bu.install_pkg('/mnt', chroot=True, packages=['python27'])
# set autoboot delay
autoboot_delay = env.instance.config.get('bootstrap-autoboot-delay', '-1')
run('echo autoboot_delay=%s >> /mnt/boot/loader.conf' % autoboot_delay)
bu.generate_remote_ssh_keys()
# reboot
if value_asbool(env.instance.config.get('bootstrap-reboot', 'true')):
with settings(hide('warnings'), warn_only=True):
run('reboot')
def bootstrap_files(self):
""" we need some files to bootstrap the FreeBSD installation.
Some...
- need to be provided by the user (i.e. authorized_keys)
- others have some (sensible) defaults (i.e. rc.conf)
- some can be downloaded via URL (i.e.) http://pkg.freebsd.org/freebsd:10:x86:64/latest/Latest/pkg.txz
For those which can be downloaded we check the downloads directory. if the file exists there
(and if the checksum matches TODO!) we will upload it to the host. If not, we will fetch the file
from the given URL from the host.
For files that cannot be downloaded (authorized_keys, rc.conf etc.) we allow the user to provide their
own version in a ``bootstrap-files`` folder. The location of this folder can either be explicitly provided
via the ``bootstrap-files`` key in the host definition of the config file or it defaults to ``deployment/bootstrap-files``.
User provided files can be rendered as Jinja2 templates, by providing ``use_jinja: True`` in the YAML file.
They will be rendered with the instance configuration dictionary as context.
If the file is not found there, we revert to the default
files that are part of bsdploy. If the file cannot be found there either we either error out or for authorized_keys
we look in ``~/.ssh/identity.pub``.
"""
bootstrap_file_yamls = [
abspath(join(self.default_template_path,
self.bootstrap_files_yaml)),
abspath(join(self.custom_template_path, self.bootstrap_files_yaml))
]
bootstrap_files = dict()
if self.upload_authorized_keys:
bootstrap_files['authorized_keys'] = BootstrapFile(
self, 'authorized_keys', **{
'directory':
'/mnt/root/.ssh',
'directory_mode':
'0600',
'remote':
'/mnt/root/.ssh/authorized_keys',
'fallback': [
'~/.ssh/identity.pub', '~/.ssh/id_rsa.pub',
'~/.ssh/id_dsa.pub', '~/.ssh/id_ecdsa.pub'
]
})
for bootstrap_file_yaml in bootstrap_file_yamls:
if not exists(bootstrap_file_yaml):
continue
with open(bootstrap_file_yaml) as f:
info = yaml.load(f, Loader=SafeLoader)
if info is None:
continue
for k, v in info.items():
bootstrap_files[k] = BootstrapFile(self, k, **v)
for bf in bootstrap_files.values():
if not exists(bf.local) and bf.raw_fallback:
if not bf.existing_fallback:
print(
"Found no public key in %s, you have to create '%s' manually"
% (expanduser('~/.ssh'), bf.local))
sys.exit(1)
print("The '%s' file is missing." % bf.local)
for path in bf.existing_fallback:
yes = env.instance.config.get('bootstrap-yes', False)
if yes or yesno(
"Should we generate it using the key in '%s'?" %
path):
if not exists(bf.expected_path):
os.mkdir(bf.expected_path)
with open(bf.local, 'wb') as out:
with open(path, 'rb') as f:
out.write(f.read())
break
else:
# answered no to all options
sys.exit(1)
if not bf.check():
print('Cannot find %s' % bf.local)
sys.exit(1)
packages_path = join(self.download_path, 'packages')
if exists(packages_path):
for dirpath, dirnames, filenames in os.walk(packages_path):
path = dirpath.split(packages_path)[1][1:]
for filename in filenames:
if not filename.endswith('.txz'):
continue
bootstrap_files[join(path, filename)] = BootstrapFile(
self, join(path, filename),
**dict(local=join(packages_path, join(path, filename)),
remote=join('/mnt/var/cache/pkg/All', filename),
encrypted=False))
if self.ssh_keys is not None:
for ssh_key_name, ssh_key_options in list(self.ssh_keys):
ssh_key = join(self.custom_template_path, ssh_key_name)
if exists(ssh_key):
pub_key_name = '%s.pub' % ssh_key_name
pub_key = '%s.pub' % ssh_key
if not exists(pub_key):
print("Public key '%s' for '%s' missing." %
(pub_key, ssh_key))
sys.exit(1)
bootstrap_files[ssh_key_name] = BootstrapFile(
self, ssh_key_name,
**dict(local=ssh_key,
remote='/mnt/etc/ssh/%s' % ssh_key_name,
mode=0600))
bootstrap_files[pub_key_name] = BootstrapFile(
self, pub_key_name,
**dict(local=pub_key,
remote='/mnt/etc/ssh/%s' % pub_key_name,
mode=0644))
if hasattr(env.instance, 'get_vault_lib'):
vaultlib = env.instance.get_vault_lib()
for bf in bootstrap_files.values():
if bf.encrypted is None and exists(bf.local):
with open(bf.local) as f:
data = f.read()
bf.info['encrypted'] = vaultlib.is_encrypted(data)
return bootstrap_files
def _bootstrap():
bu = BootstrapUtils()
bu.generate_ssh_keys()
bu.print_bootstrap_files()
# gather infos
if not bu.bsd_url:
print("Found no FreeBSD system to install, please use 'special edition' or specify bootstrap-bsd-url and make sure mfsbsd is running")
return
# get realmem here, because it may fail and we don't want that to happen
# in the middle of the bootstrap
realmem = bu.realmem
print("\nFound the following disk devices on the system:\n %s" % ' '.join(bu.sysctl_devices))
if bu.first_interface:
print("\nFound the following network interfaces, now is your chance to update your rc.conf accordingly!\n %s" % ' '.join(bu.phys_interfaces))
else:
print("\nWARNING! Found no suitable network interface!")
template_context = {"ploy_jail_host_pkg_repository": "pkg+http://pkg.freeBSD.org/${ABI}/quarterly"}
# first the config, so we don't get something essential overwritten
template_context.update(env.instance.config)
template_context.update(
devices=bu.sysctl_devices,
interfaces=bu.phys_interfaces,
hostname=env.instance.id)
rc_conf = bu.bootstrap_files['rc.conf'].read(template_context)
if not rc_conf.endswith('\n'):
print("\nERROR! Your rc.conf doesn't end in a newline:\n==========\n%s<<<<<<<<<<\n" % rc_conf)
return
rc_conf_lines = rc_conf.split('\n')
for interface in [bu.first_interface, env.instance.config.get('ansible-dhcp_host_sshd_interface')]:
if interface is None:
continue
ifconfig = 'ifconfig_%s' % interface
for line in rc_conf_lines:
if line.strip().startswith(ifconfig):
break
else:
if not yesno("\nDidn't find an '%s' setting in rc.conf. You sure that you want to continue?" % ifconfig):
return
yes = env.instance.config.get('bootstrap-yes', False)
if not (yes or yesno("\nContinuing will destroy the existing data on the following devices:\n %s\n\nContinue?" % ' '.join(bu.devices))):
return
# install FreeBSD in ZFS root
devices_args = ' '.join('-d %s' % x for x in bu.devices)
system_pool_name = env.instance.config.get('bootstrap-system-pool-name', 'system')
data_pool_name = env.instance.config.get('bootstrap-data-pool-name', 'tank')
swap_arg = ''
swap_size = env.instance.config.get('bootstrap-swap-size', '%iM' % (realmem * 2))
if swap_size:
swap_arg = '-s %s' % swap_size
system_pool_arg = ''
system_pool_size = env.instance.config.get('bootstrap-system-pool-size', '20G')
if system_pool_size:
system_pool_arg = '-z %s' % system_pool_size
run('destroygeom {devices_args} -p {system_pool_name} -p {data_pool_name}'.format(
devices_args=devices_args,
system_pool_name=system_pool_name,
data_pool_name=data_pool_name))
run('{env_vars}{zfsinstall} {devices_args} -p {system_pool_name} -V 28 -u {bsd_url} {swap_arg} {system_pool_arg}'.format(
env_vars=bu.env_vars,
zfsinstall=bu.zfsinstall,
devices_args=devices_args,
system_pool_name=system_pool_name,
bsd_url=bu.bsd_url,
swap_arg=swap_arg,
system_pool_arg=system_pool_arg), shell=False)
# create partitions for data pool, but only if the system pool doesn't use
# the whole disk anyway
if system_pool_arg:
for device in bu.devices:
run('gpart add -t freebsd-zfs -l {data_pool_name}_{device} {device}'.format(
data_pool_name=data_pool_name,
device=device))
# mount devfs inside the new system
if 'devfs on /rw/dev' not in bu.mounts:
run('mount -t devfs devfs /mnt/dev')
# setup bare essentials
run('cp /etc/resolv.conf /mnt/etc/resolv.conf', warn_only=True)
bu.create_bootstrap_directories()
bu.upload_bootstrap_files(template_context)
bootstrap_packages = ['python27']
if value_asbool(env.instance.config.get('firstboot-update', 'false')):
bootstrap_packages.append('firstboot-freebsd-update')
run('''touch /mnt/firstboot''')
run('''sysrc -f /mnt/etc/rc.conf firstboot_freebsd_update_enable=YES''')
# we need to install python here, because there is no way to install it via
# ansible playbooks
bu.install_pkg('/mnt', chroot=True, packages=bootstrap_packages)
# set autoboot delay
autoboot_delay = env.instance.config.get('bootstrap-autoboot-delay', '-1')
run('echo autoboot_delay=%s >> /mnt/boot/loader.conf' % autoboot_delay)
bu.generate_remote_ssh_keys()
# reboot
if value_asbool(env.instance.config.get('bootstrap-reboot', 'true')):
with settings(hide('warnings'), warn_only=True):
run('reboot')
def ensure(self, instance):
dhcpservers = instance.vb.list('dhcpservers')
name = "HostInterfaceNetworking-%s" % self.name
kw = {}
for key in ('ip', 'netmask', 'lowerip', 'upperip'):
if key not in self.config:
log.error("The '%s' option is required for dhcpserver '%s'." %
(key, self.name))
sys.exit(1)
kw[key] = self.config[key]
if name not in dhcpservers:
try:
instance.vb.dhcpserver('add', '--enable', netname=name, **kw)
except subprocess.CalledProcessError as e:
log.error("Failed to add dhcpserver '%s':\n%s" %
(self.name, e))
sys.exit(1)
log.info("Added dhcpserver '%s'." % self.name)
dhcpserver = instance.vb.list('dhcpservers')[name]
matches = True
if 'ip' in self.config:
if dhcpserver['IP'] != self.config['ip']:
log.error(
"The host only interface '%s' has an IP '%s' that doesn't match the config '%s'."
% (self.name, dhcpserver['IP'], self.config['ip']))
matches = False
if 'netmask' in self.config:
if dhcpserver['NetworkMask'] != self.config['netmask']:
log.error(
"The host only interface '%s' has an netmask '%s' that doesn't match the config '%s'."
% (self.name, dhcpserver['NetworkMask'],
self.config['netmask']))
matches = False
if 'lower-ip' in self.config:
if dhcpserver['lowerIPAddress'] != self.config['lower-ip']:
log.error(
"The host only interface '%s' has a lower IP '%s' that doesn't match the config '%s'."
% (self.name, dhcpserver['lowerIPAddress'],
self.config['lower-ip']))
matches = False
if 'upper-ip' in self.config:
if dhcpserver['upperIPAddress'] != self.config['upper-ip']:
log.error(
"The host only interface '%s' has a upper IP '%s' that doesn't match the config '%s'."
% (self.name, dhcpserver['upperIPAddress'],
self.config['upper-ip']))
matches = False
if not matches:
if not yesno(
"Should the dhcpserver '%s' be modified to match the config?"
% self.name):
sys.exit(1)
try:
instance.vb.dhcpserver('modify',
'--enable',
netname=name,
**kw)
except subprocess.CalledProcessError as e:
log.error("Failed to modify dhcpserver '%s':\n%s" %
(self.name, e))
sys.exit(1)
def get_playbook(self, *args, **kwargs):
inject_ansible_paths()
import ansible.playbook
import ansible.callbacks
import ansible.errors
import ansible.utils
try:
from ansible.utils.vault import VaultLib
except ImportError:
VaultLib = None
from ploy_ansible.inventory import Inventory
host = self.uid
user = self.config.get('user', 'root')
sudo = self.config.get('sudo')
playbooks_directory = get_playbooks_directory(self.master.main_config)
class PlayBook(ansible.playbook.PlayBook):
def __init__(self, *args, **kwargs):
self.roles = kwargs.pop('roles', None)
if self.roles is not None:
if isinstance(self.roles, basestring):
self.roles = self.roles.split()
kwargs['playbook'] = '<dynamically generated from %s>' % self.roles
ansible.playbook.PlayBook.__init__(self, *args, **kwargs)
self.basedir = playbooks_directory
def _load_playbook_from_file(self, *args, **kwargs):
if self.roles is None:
return ansible.playbook.PlayBook._load_playbook_from_file(
self, *args, **kwargs)
settings = {
'hosts': [host],
'user': user,
'roles': self.roles}
if sudo is not None:
settings['sudo'] = sudo
return (
[settings],
[playbooks_directory])
patch_connect(self.master.ctrl)
playbook = kwargs.pop('playbook', None)
if playbook is None:
for instance_id in (self.uid, self.id):
playbook_path = os.path.join(playbooks_directory, '%s.yml' % instance_id)
if os.path.exists(playbook_path):
playbook = playbook_path
break
if 'playbook' in self.config:
if playbook is not None and playbook != self.config['playbook']:
log.warning("Instance '%s' has the 'playbook' option set, but there is also a playbook at the default location '%s', which differs from '%s'." % (self.config_id, playbook, self.config['playbook']))
playbook = self.config['playbook']
if playbook is not None:
log.info("Using playbook at '%s'." % playbook)
roles = kwargs.pop('roles', None)
if roles is None and 'roles' in self.config:
roles = self.config['roles']
if roles is not None and playbook is not None:
log.error("You can't use a playbook and the 'roles' options at the same time for instance '%s'." % self.config_id)
sys.exit(1)
stats = ansible.callbacks.AggregateStats()
callbacks = ansible.callbacks.PlaybookCallbacks(verbose=ansible.utils.VERBOSITY)
runner_callbacks = ansible.callbacks.PlaybookRunnerCallbacks(stats, verbose=ansible.utils.VERBOSITY)
skip_host_check = kwargs.pop('skip_host_check', False)
if roles is None:
kwargs['playbook'] = playbook
else:
kwargs['roles'] = roles
if VaultLib is not None:
kwargs['vault_password'] = get_vault_password_source(self.master.main_config).get()
inventory = Inventory(self.master.ctrl, vault_password=kwargs.get('vault_password'))
try:
pb = PlayBook(
*args,
callbacks=callbacks,
inventory=inventory,
runner_callbacks=runner_callbacks,
stats=stats,
**kwargs)
except ansible.errors.AnsibleError as e:
log.error("AnsibleError: %s" % e)
sys.exit(1)
for (play_ds, play_basedir) in zip(pb.playbook, pb.play_basedirs):
if 'user' not in play_ds:
play_ds['user'] = self.config.get('user', 'root')
if not skip_host_check:
hosts = play_ds.get('hosts', '')
if isinstance(hosts, basestring):
hosts = hosts.split(':')
if self.uid not in hosts:
log.warning("The host '%s' is not in the list of hosts (%s) of '%s'.", self.uid, ','.join(hosts), playbook)
if not yesno("Do you really want to apply '%s' to the host '%s'?" % (playbook, self.uid)):
sys.exit(1)
play_ds['hosts'] = [self.uid]
return pb
def cmd_set(self, args, src):
if src.get(fail_on_error=False) and not yesno("There is already a key stored, do you want to replace it?"):
return
src.set(getpass.getpass("Password for '%s': " % src.id))
def cmd_delete(self, args, src):
if yesno("Do you really want to delete the key for '%s'?" % src.id):
src.delete()
def get_playbook(self, *args, **kwargs):
inject_ansible_paths(self.master.ctrl)
from ansible.playbook import Play, Playbook
import ansible.errors
(options, loader, inventory, variable_manager) = self.get_ansible_variablemanager(**kwargs)
playbooks_directory = get_playbooks_directory(self.master.main_config)
playbook = kwargs.pop('playbook', None)
if playbook is None:
for instance_id in (self.uid, self.id):
playbook_path = os.path.join(playbooks_directory, '%s.yml' % instance_id)
if os.path.exists(playbook_path):
playbook = playbook_path
break
if 'playbook' in self.config:
if playbook is not None and playbook != self.config['playbook']:
log.warning("Instance '%s' has the 'playbook' option set, but there is also a playbook at the default location '%s', which differs from '%s'." % (self.config_id, playbook, self.config['playbook']))
playbook = self.config['playbook']
if playbook is not None:
log.info("Using playbook at '%s'." % playbook)
roles = kwargs.pop('roles', None)
if roles is None and 'roles' in self.config:
roles = self.config['roles']
if roles is not None and playbook is not None:
log.error("You can't use a playbook and the 'roles' options at the same time for instance '%s'." % self.config_id)
sys.exit(1)
if playbook is None and roles is None:
return None
skip_host_check = kwargs.pop('skip_host_check', False)
try:
if roles is None:
pb = Playbook.load(playbook, variable_manager=variable_manager, loader=loader)
plays = pb.get_plays()
else:
if isinstance(roles, basestring):
roles = roles.split()
data = {
'hosts': [self.uid],
'roles': roles}
plays = [Play.load(data, variable_manager=variable_manager, loader=loader)]
pb = Playbook(loader=loader)
pb._entries.extend(plays)
except ansible.errors.AnsibleError as e:
log.error("AnsibleError: %s" % e)
sys.exit(1)
for play in plays:
if play._attributes.get('remote_user') is None:
play._attributes['remote_user'] = self.config.get('user', 'root')
if self.config.get('sudo'):
play._attributes['sudo'] = self.config.get('sudo')
if not skip_host_check:
hosts = play._attributes.get('hosts', None)
if isinstance(hosts, basestring):
hosts = hosts.split(':')
if hosts is None:
hosts = {}
if self.uid not in hosts:
log.warning("The host '%s' is not in the list of hosts (%s) of '%s'.", self.uid, ','.join(hosts), playbook)
if not yesno("Do you really want to apply '%s' to the host '%s'?" % (playbook, self.uid)):
sys.exit(1)
play._attributes['hosts'] = [self.uid]
return pb
def _bootstrap():
bu = BootstrapUtils()
bu.generate_ssh_keys()
bu.print_bootstrap_files()
# gather infos
if not bu.bsd_url:
print(
"Found no FreeBSD system to install, please specify bootstrap-bsd-url and make sure mfsbsd is running"
)
return
# get realmem here, because it may fail and we don't want that to happen
# in the middle of the bootstrap
realmem = bu.realmem
print("\nFound the following disk devices on the system:\n %s" %
' '.join(bu.sysctl_devices))
if bu.first_interface:
print(
"\nFound the following network interfaces, now is your chance to update your rc.conf accordingly!\n %s"
% ' '.join(bu.phys_interfaces))
else:
print("\nWARNING! Found no suitable network interface!")
template_context = {
"ploy_jail_host_pkg_repository":
"pkg+http://pkg.freeBSD.org/${ABI}/quarterly"
}
# first the config, so we don't get something essential overwritten
template_context.update(env.instance.config)
template_context.update(devices=bu.sysctl_devices,
interfaces=bu.phys_interfaces,
hostname=env.instance.id)
rc_conf = bu.bootstrap_files['rc.conf'].read(template_context)
if not rc_conf.endswith('\n'):
print(
"\nERROR! Your rc.conf doesn't end in a newline:\n==========\n%s<<<<<<<<<<\n"
% rc_conf)
return
rc_conf_lines = rc_conf.split('\n')
for interface in [
bu.first_interface,
env.instance.config.get('ansible-dhcp_host_sshd_interface')
]:
if interface is None:
continue
ifconfig = 'ifconfig_%s' % interface
for line in rc_conf_lines:
if line.strip().startswith(ifconfig):
break
else:
if not yesno(
"\nDidn't find an '%s' setting in rc.conf. You sure that you want to continue?"
% ifconfig):
return
yes = env.instance.config.get('bootstrap-yes', False)
if not (yes or yesno(
"\nContinuing will destroy the existing data on the following devices:\n %s\n\nContinue?"
% ' '.join(bu.devices))):
return
# install FreeBSD in ZFS root
devices_args = ' '.join('-d %s' % x for x in bu.devices)
system_pool_name = env.instance.config.get('bootstrap-system-pool-name',
'system')
data_pool_name = env.instance.config.get('bootstrap-data-pool-name',
'tank')
swap_arg = ''
swap_size = env.instance.config.get('bootstrap-swap-size',
'%iM' % (realmem * 2))
if swap_size:
swap_arg = '-s %s' % swap_size
system_pool_arg = ''
system_pool_size = env.instance.config.get('bootstrap-system-pool-size',
'20G')
if system_pool_size:
system_pool_arg = '-z %s' % system_pool_size
run('destroygeom {devices_args} -p {system_pool_name} -p {data_pool_name}'.
format(devices_args=devices_args,
system_pool_name=system_pool_name,
data_pool_name=data_pool_name))
run('{env_vars}{zfsinstall} {devices_args} -p {system_pool_name} -V 28 -u {bsd_url} {swap_arg} {system_pool_arg}'
.format(env_vars=bu.env_vars,
zfsinstall=bu.zfsinstall,
devices_args=devices_args,
system_pool_name=system_pool_name,
bsd_url=bu.bsd_url,
swap_arg=swap_arg,
system_pool_arg=system_pool_arg),
shell=False)
# create partitions for data pool, but only if the system pool doesn't use
# the whole disk anyway
if system_pool_arg:
for device in bu.devices:
run('gpart add -t freebsd-zfs -l {data_pool_name}_{device} {device}'
.format(data_pool_name=data_pool_name, device=device))
# mount devfs inside the new system
if 'devfs on /rw/dev' not in bu.mounts:
run('mount -t devfs devfs /mnt/dev')
# setup bare essentials
run('cp /etc/resolv.conf /mnt/etc/resolv.conf', warn_only=True)
bu.create_bootstrap_directories()
bu.upload_bootstrap_files(template_context)
bootstrap_packages = ['python27']
if value_asbool(env.instance.config.get('firstboot-update', 'false')):
bootstrap_packages.append('firstboot-freebsd-update')
run('''touch /mnt/firstboot''')
run('''sysrc -f /mnt/etc/rc.conf firstboot_freebsd_update_enable=YES'''
)
# we need to install python here, because there is no way to install it via
# ansible playbooks
bu.install_pkg('/mnt', chroot=True, packages=bootstrap_packages)
# set autoboot delay
autoboot_delay = env.instance.config.get('bootstrap-autoboot-delay', '-1')
run('echo autoboot_delay=%s >> /mnt/boot/loader.conf' % autoboot_delay)
bu.generate_remote_ssh_keys()
# reboot
if value_asbool(env.instance.config.get('bootstrap-reboot', 'true')):
with settings(hide('warnings'), warn_only=True):
run('reboot')
def cmd_delete(self, args, src):
if yesno("Do you really want to delete the key for '%s'?" % src.id):
src.delete()