def symlink(src, dst):
if dst.endswith('/'):
raise NotImplementedError('Creating link inside directory not '
'implemented')
lst = remote.lstat(dst)
if lst:
if not S_ISLNK(lst.st_mode):
raise RemotePathIsNotALinkError('Already exists and not a link: '
'{}'.format(dst))
# remote is a link
rsrc = remote.readlink(dst)
if rsrc == src:
return Unchanged(msg='Unchanged link: {} -> {}'.format(dst, src))
# we need to update the link, unfortunately, this is often not possible
# atomically
remote.unlink(dst)
remote.symlink(src, dst)
return Changed(msg='Changed link: {} -> {} (previously -> {})'.format(
dst, src, rsrc))
remote.symlink(src, dst)
return Changed(msg='Created link: {} -> {}'.format(dst, src))
def touch(remote_path, mtime=None, atime=None):
"""Update mtime and atime of a path.
Similar to running ``touch remote_path``.
:param remote_path: Remote path whose times will get updated.
:param mtime: New mtime. If ``None``, uses the current time.
:param atime: New atime. Only used if ``mtime`` is not None. Defaults to
``mtime``.
:return: Since it always updates the current time, calling this function
will always result in a modification.
"""
# ensure the file exists
if not remote.lstat(remote_path):
with remote.file(remote_path, 'w') as out:
out.write('')
if mtime is None:
remote.utime(remote_path, None)
return Changed(msg=u'Touched {} to current time'.format(remote_path))
else:
atime = atime if atime is not None else mtime
remote.utime(remote_path, (atime, mtime))
return Changed(msg=u'Touched {} to mtime={}, atime={}'.format(
remote_path, mtime, atime))
def reload_unit(unit_name, only_if_running=False):
if only_if_running:
cmd = 'reload-or-try-restart'
else:
cmd = 'reload-or-restart'
proc.run([config['cmd_systemctl'], cmd, unit_name])
return Changed(msg='Reloaded {}'.format(unit_name))
def stop_unit(unit_name):
state = get_unit_state(unit_name)
if state['ActiveState'] == 'stopped':
return Unchanged(msg='{} already stopped'.format(unit_name))
proc.run([config['cmd_systemctl'], 'stop', unit_name])
return Changed(msg='Stopped {}'.format(unit_name))
def start_unit(unit_name):
state = get_unit_state(unit_name)
if 'ActiveState' in state and state['ActiveState'] == 'active' and state['SubState'] == 'running':
return Unchanged(msg='{} already running'.format(unit_name))
proc.run([config['cmd_systemctl'], 'start', unit_name])
return Changed(msg='Started {}'.format(unit_name))
def disable_unit(unit_name):
state = get_unit_state(unit_name)
if state.get('UnitFileState') == 'disabled':
return Unchanged(msg='{} already disabled'.format(unit_name))
proc.run([config['cmd_systemctl'], 'disable', unit_name])
return Changed(msg='Disabled {}'.format(unit_name))
def init_authorized_keys(user='******', fix_permissions=True):
ak_file = get_authorized_keys_file(user)
ak_dir = remote.path.dirname(ak_file)
changed = False
# ensure the directory exists
changed |= fs.create_dir(ak_dir, mode=AK_DIR_PERMS).changed
if fix_permissions:
changed |= fs.chmod(ak_dir, AK_DIR_PERMS).changed
changed |= fs.chown(ak_dir, uid=user).changed
# check if the authorized keys file exists
if not remote.lstat(ak_file):
changed |= fs.touch(ak_file).changed
if fix_permissions:
changed |= fs.chmod(ak_file, AK_FILE_PERMS).changed
changed |= fs.chown(ak_dir, uid=user).changed
# at this point, we have fixed permissions for file and dir, as well as
# ensured they exist. however, they might still be owned by root
if changed:
return Changed(ak_file,
msg='Changed permissions or owner on authorized keys')
return Unchanged(
ak_file, msg='authorized keys file has correct owner and permissions')
def remove_dir(remote_path, recursive=True):
"""Removes a remote directory.
If the directory does not exist, does nothing.
:param recursive: Makes ``remove_dir`` behave closer to ``rm -rf`` instead
of ``rmdir``.
"""
st = remote.lstat(remote_path)
if st is None:
return Unchanged(msg=u'Directory already gone: {}'.format(remote_path))
# if it is not a directory, don't touch it
if not S_ISDIR(st.st_mode):
raise RemotePathIsNotADirectoryError(remote_path)
if recursive:
for dirpath, dirnames, filenames in walk(remote_path, topdown=False):
for fn in filenames:
remote.unlink(remote.path.join(dirpath, fn))
remote.rmdir(dirpath)
else:
remote.rmdir(dirpath)
return Changed(msg=u'Removed directory: {}'.format(remote_path))
def upgrade(max_age=3600, force=False, dist_upgrade=False):
# FIXME: should allow upgrading selected packages
update(max_age)
args = [config['cmd_apt_get']]
# FIXME: check for upgrades first and output proper changed status
args.extend([
'upgrade' if not dist_upgrade else 'dist-upgrade',
'--quiet',
'--yes',
# FIXME: options below don't work. why?
# '--option', 'Dpkg::Options::="--force-confdef"',
# '--option', 'Dpkg::Options::="--force-confold"'
])
if force:
args.append('--force-yes')
proc.run(
args, extra_env={
'DEBIAN_FRONTEND': 'noninteractive',
})
info_installed_packages.invalidate_cache()
return Changed(msg='Upgraded all packages')
def extract(fn, remote_dest):
# FIXME: should check if remote_dest is an existing directory or create it
# FIXME: add a way to extract arbitrary archives locally and send?
if fn.endswith('.zip'):
with fs.remote_tmpdir() as tmp:
archive_name = remote.path.join(tmp, 'archive.zip')
fs.upload_file(fn, archive_name)
proc.run(['unzip', archive_name], cwd=remote_dest)
else:
# tar
for ext, flag in TAR_DECOMP_FLAGS.items():
if fn.endswith(ext):
decomp_flag = flag
break
else:
raise ValueError(
'Unsupported archive type (determined by file ending): {}'.
format(fn))
args = ['tar', decomp_flag + 'xf', '-', '-C', remote_dest]
with open(fn, 'rb') as inp:
proc.run(args, input=inp)
return Changed(msg='Extracted archive {} to {}'.format(fn, remote_dest))
def add_repo(distribution,
components=['main'],
site='http://deb.debian.org/debian',
src=False,
arch=[],
name=None):
comps = ' '.join(components)
options = ''
if arch:
options = ' [ arch={} ]'.format(','.join(arch))
line = '{}{} {} {} {}\n'.format(
'deb-src' if src else 'deb',
options,
site,
distribution,
comps,
)
if name is None:
name = '{}_{}{}'.format(distribution, '_'.join(components), ''
if not src else '-sources')
path = remote.path.join(config['apt_sources_list_d'], name + '.list')
upload = fs.upload_string(line, path, create_parent=True)
if upload.changed:
info_update_timestamp().mark_stale()
return Changed(msg='Added apt repository: {}'.format(line))
return Unchanged(msg='Already present: {}'.format(line))
def chmod(remote_path, mode, recursive=False, executable=False):
# FIXME: instead of executable, add parsing of rwxX-style modes
# FIXME: add speedup by using local chmod
xmode = mode if not executable else mode | 0o111
st = remote.lstat(remote_path)
if mode > 0o777:
raise ValueError('Modes above 0o777 are not supported')
changed = False
actual_mode = st.st_mode & 0o777
# if the target is a directory or already has at least one executable bit,
# we apply the executable mode (see chmod manpage for details)
correct_mode = (xmode
if S_ISDIR(st.st_mode) or actual_mode & 0o111 else mode)
if actual_mode != correct_mode:
remote.chmod(remote_path, correct_mode)
changed = True
if recursive and S_ISDIR(st.st_mode):
for rfn in remote.listdir(remote_path):
changed |= chmod(
remote.path.join(remote_path, rfn), mode, True,
executable).changed
if changed:
return Changed(msg='Changed mode of {} to {:o}'.format(
remote_path, mode))
return Unchanged(msg='Mode of {} already {:o}'.format(remote_path, mode))
def chown(remote_path, uid=None, gid=None, recursive=False):
new_owner = ':'
# no-op
if uid is None and gid is None:
return
if uid is not None:
new_owner = str(uid) + new_owner
if gid is not None:
new_owner += str(gid)
cmd = [config['cmd_chown']]
if recursive:
cmd.append('-R')
cmd.append('-c') # FIXME: on BSDs, we need -v here?
cmd.append(new_owner)
cmd.append(remote_path)
stdout, _, _ = proc.run(cmd)
if stdout.strip():
return Changed(msg='Changed ownership of {} to {}'.format(
remote_path, new_owner))
return Unchanged(msg='Ownership of {} already {}'.format(
remote_path, new_owner))
def install_private_key(key_file,
user='******',
key_type='rsa',
target_path=None):
if target_path is None:
# FIXME: auto-determine key type if None
if key_type not in ('rsa', ):
raise NotImplementedError('Key type {} not supported')
fn = 'id_' + key_type
target_path = remote.path.join(info['posix.users'][user].home, '.ssh',
fn)
changed = False
# blocked: SSH transport does not suppoort
# with remote.umasked(0o777 - KEY_FILE_PERMS):
changed |= fs.create_dir(remote.path.dirname(target_path),
mode=AK_DIR_PERMS).changed
changed |= fs.upload_file(key_file, target_path).changed
changed |= fs.chmod(target_path, mode=KEY_FILE_PERMS).changed
if changed:
return Changed(msg='Installed private key {}'.format(target_path))
return Unchanged(
msg='Private key {} already installed'.format(target_path))
def remove_packages(pkgs, check_first=True, purge=False, max_age=3600):
if check_first and not set(pkgs).intersection(
set(info_installed_packages().keys())):
return Unchanged(msg='Not installed: {}'.format(' '.join(pkgs)))
update(max_age)
args = [config['cmd_apt_get']]
args.extend([
'remove' if not purge else 'purge',
'--quiet',
'--yes', # FIXME: options below don't work. why?
# '--option', 'Dpkg::Options::="--force-confdef"',
# '--option', 'Dpkg::Options::="--force-confold"'
])
args.extend(pkgs)
proc.run(
args, extra_env={
'DEBIAN_FRONTEND': 'noninteractive',
})
info_installed_packages.invalidate_cache()
return Changed(msg='{} {}'.format('Removed' if not purge else 'Purged',
' '.join(pkgs)))
def dpkg_install(paths, check=True):
if not hasattr(paths, 'keys'):
pkgs = {}
# determine package names from filenames. ideally, we would open the
# package here and check
for p in paths:
fn = os.path.basename(p)
try:
name, version, tail = fn.split('_', 3)
pkgs[(name, version)] = p
except ValueError:
raise ValueError(
'Could not determine package version from '
'package filename {}. Please rename the .deb '
'to standard debian convention '
'(name_version_arch.deb) or supply a specific '
'version by passing a dictionary parameter.'.format(fn))
# log names
log.debug('Package names: ' + ', '.join('{} -> {}'.format(k, v)
for k, v in pkgs.items()))
if check:
missing = []
installed = info_installed_packages()
for name, version in pkgs:
if name not in installed or not installed[name].eq_version(version):
missing.append((name, version))
else:
missing = pkgs.keys()
log.debug('Installing packages: {}'.format(missing))
if not missing:
return Unchanged('Packages {!r} already installed'.format(pkgs.keys()))
# FIXME: see above
info_installed_packages.invalidate_cache()
with fs.remote_tmpdir() as rtmp:
# upload packages to be installed
pkg_files = []
for idx, key in enumerate(missing):
tmpdest = remote.path.join(rtmp, str(idx) + '.deb')
fs.upload_file(pkgs[key], tmpdest)
pkg_files.append(tmpdest)
# install in a single dpkg install line
# FIXME: add debconf default and such (same as apt)
args = [config['cmd_dpkg'], '-i']
args.extend(pkg_files)
proc.run(
args, extra_env={
'DEBIAN_FRONTEND': 'noninteractive',
})
return Changed(msg='Installed packages {!r}'.format(missing))
def remove_file(remote_path):
"""Removes a remote file, as long as it is a file or a symbolic link.
:param remote_path: Remote file to remote.
"""
try:
remote.unlink(remote_path)
except RemoteFileDoesNotExistError:
return Unchanged(msg=u'File already gone: {}'.format(remote_path))
return Changed(msg=u'Removed: {}'.format(remote_path))
def enable_unit(unit_name, check_first=False):
if check_first:
state = get_unit_state(unit_name)
# we use 'WantedBy' as a guess whether or not the service is enabled
# when UnitFileState is not available (SysV init or older systemd)
ufs = state.get('UnitFileState')
if ufs == 'enabled' or ufs is None and 'WantedBy' in state:
return Unchanged(msg='{} already enabled'.format(unit_name))
proc.run([config['cmd_systemctl'], 'enable', unit_name])
return Changed(msg='Enabled {}'.format(unit_name))
def reboot():
if config.get_bool('systemd'):
try:
proc.run([config['cmd_systemctl'], 'reboot'])
except RemoteFailureError:
# FIXME: should be more discerning; also verify reboot is taking
# place
pass # ignored, as the command will not finish - due to rebooting
elif config['remote_os'] in ('unix', 'posix'):
proc.run([config['cmd_shutdown'], '-r', 'now'])
return Changed(msg='Server rebooting')
def grant_me_root(my_key='~/.ssh/id_rsa.pub', unlock_root=True):
c = set_authorized_keys([os.path.expanduser(my_key)], user='******').changed
if unlock_root:
# we need to unlock the root user, otherwise a login is not possible
# via ssh
c |= posix.set_unlocked_no_password(['root']).changed
if c:
return Changed(msg='Granted root ssh login')
return Unchanged(msg='Already have root login')
def dpkg_add_architecture(arch):
archs = [info_dpkg_architecture()] + info_dpkg_foreign_architectures()
if arch in archs:
return Unchanged(msg='Architecture already enabled: {}'.format(arch))
proc.run([config['cmd_dpkg'], '--add-architecture', arch])
# invalidate caches
info_dpkg_foreign_architectures.invalidate_cache()
info_update_timestamp().mark_stale()
return Changed(msg='New architecture added: {}'.format(arch))
def useradd(name,
groups=[],
user_group=True,
comment=None,
home=None,
create_home=None,
system=False,
shell=None):
cmd = [config['cmd_useradd']]
gs = groups[:]
if gs:
if not user_group:
cmd.extend(('-g', gs.pop(0)))
if gs:
cmd.extend(('-G', ','.join(groups)))
if user_group is True:
cmd.append('-U')
elif user_group is False:
cmd.append('-N')
if comment is not None:
cmd.extend(('-c', comment))
if home is not None:
cmd.extend(('-d', home))
if create_home is False:
cmd.append('-M')
elif create_home is True:
cmd.append('-m')
if shell:
cmd.extend(('-s', shell))
if system:
cmd.append('-r')
cmd.append(name)
stdout, stderr, returncode = proc.run(cmd,
status_ok=(0, 9),
status_meaning=_USERADD_STATUS_CODES)
if returncode == 9:
# FIXME: should check if user is up-to-date (home, etc)
return Unchanged(msg='User {} already exists'.format(name))
info_users.invalidate_cache()
return Changed(msg='Created user {}'.format(name))
def expand_root_fs():
dev_size, _, _ = proc.run(['fdisk', '-s', '/dev/mmcblk0'])
p1_size, _, _ = proc.run(['fdisk', '-s', '/dev/mmcblk0p1'])
p2_size, _, _ = proc.run(['fdisk', '-s', '/dev/mmcblk0p2'])
free_space = (int(dev_size) - int(p1_size) - int(p2_size)) * 512
if free_space <= 4 * 1024 * 1024:
return Unchanged(
msg='Free space is <= 4M. Not expanding root filesystem')
else:
# fixme: run fdisk and resize2fs instead of raspi-config?
proc.run(['raspi-config', '--expand-rootfs'])
return Changed(msg='Expanded root filesystem')
def install_unit_file(unit_file, reload=True):
base, ext = os.path.splitext(unit_file)
if ext not in UNIT_EXTS:
raise ValueError('unit_file should be one of {}'.format(UNIT_EXTS))
remote_unit = os.path.join(config['systemd_unit_dir'],
os.path.basename(unit_file))
if fs.upload_file(unit_file, remote_unit).changed:
if reload:
daemon_reload()
return Changed(msg='Installed {}'.format(remote_unit))
return Unchanged(msg='{} already installed'.format(remote_unit))
def set_password(name, password=None, hashed=False, salt=None):
if salt is None:
salt = hashlib.sha1(os.urandom(64)).hexdigest()
if not hashed:
# hash using sha256
hash = crypt(password, "$6$" + salt)
else:
hash = password
# at this point, we have a hashed password
with fs.edit('/etc/shadow', create=False) as shadow:
new_lines = []
for line in shadow.lines():
entry = ShadowEntry.from_line(line)
if entry.name == name:
# we need to check if the password already matches (with a
# different salt)
pwc = entry.password_encrypted
need_update = True
if hashed:
# we were supplied a hashed password, simply compare the
# hashes
need_update = (pwc != password)
else:
# not hashed
if '$' in pwc:
lead = pwc[:pwc.rindex('$')]
need_update = (crypt(password, lead) != pwc)
if need_update:
entry = entry._replace(password_encrypted=hash)
new_lines.append(entry.to_line())
continue
new_lines.append(line)
shadow.set_lines(new_lines)
if shadow.changed:
return Changed(msg='Updated password for user {}'.format(name))
return Unchanged(msg='Password for {} already set'.format(name))
def install_network_file(network_file, reload=True):
base, ext = os.path.splitext(network_file)
if ext not in NETWORK_EXTS:
raise ValueError(
'network_file should be one of {}'.format(NETWORK_EXTS))
remote_network = os.path.join(config['systemd_network_dir'],
os.path.basename(network_file))
if fs.upload_file(network_file, remote_network).changed:
if reload:
daemon_reload()
return Changed(msg='Installed {}'.format(remote_network))
return Unchanged(msg='{} already installed'.format(remote_network))
def install_hostkeys(base_dir):
results = []
for key_type in ('ecdsa', 'ed25519', 'rsa'):
fn = 'ssh_host_' + key_type + '_key'
pub_fn = fn + '.pub'
sk = os.path.join(base_dir, fn)
pk = os.path.join(base_dir, pub_fn)
results.append(fs.upload_file(sk, '/etc/ssh/' + fn))
results.append(fs.upload_file(pk, '/etc/ssh/' + pub_fn))
if util.any_changed(*results):
systemd.reload_unit('ssh.service')
return Changed(msg='Installed SSH hostkeys from {}'.format(base_dir))
return Unchanged(msg='SSH hostkeys already installed')
def update(max_age=3600):
if max_age < 0:
return Unchanged(msg='apt update disabled (max_age < 0).')
ts = info_update_timestamp()
if max_age:
age = ts.get_age()
if age < max_age:
return Unchanged(
msg='apt cache is only {:.0f} minutes old, not updating'
.format(age / 60))
proc.run([config['cmd_apt_get'], 'update'])
# modify update stamp
ts.mark_current()
return Changed(msg='apt cache updated')
def _ensure_unit(service_name, upload_func, enable, auto_restart):
# FIXME: we also support sockets!
# assert service_name.endswith('.service')
changed = upload_func().changed
# FIXME: check if restart was successful?
if auto_restart and changed:
restart_unit(service_name)
if enable:
changed |= enable_unit(service_name).changed
if changed:
return Changed(
msg='Unit {} updated and restarted'.format(service_name))
return Unchanged(
msg='Unit {} already up to date and running'.format(service_name))
def enable_systemd():
changed = False
changed |= apt.install_packages(['systemd']).changed
with fs.edit('/boot/cmdline.txt', create=False) as e:
flag = 'init=/bin/systemd'
lines = e.lines()
assert len(lines) == 1
if flag not in lines[0]:
lines[0] += ' ' + flag
e.set_lines(lines)
changed |= e.changed
if changed:
return Changed(msg='Installed systemd')
return Unchanged(msg='systemd already active')