def copy(context: Context, src: str, dst: str, mode=None, owner=None, group=None):
"""
Copies the given src file to the remote host at dst.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
src : str
The local source file path relative to the project directory. Will be templated.
dst : str
The remote destination file path. Will be templated.
mode : int, optional
The new file mode. Defaults the current context file creation mode.
owner : str, optional
The new file owner. Defaults the current context owner.
group : str, optional
The new file group. Defaults the current context group.
Returns
-------
CompletedTransaction
The completed transaction
"""
src = template_str(context, src)
dst = template_str(context, dst)
check_valid_path(dst)
def get_content():
# Get source content
with open(os.path.join(context.host.manager.main_directory, src), 'r') as f:
return f.read()
return remote_upload(context, get_content, title="copy", name=dst, dst=dst, mode=mode, owner=owner, group=group)
def _resolve_variables(self, context):
"""
Resolve the templated variables for this tracked task
"""
url = template_str(context, self.tracking_repo_url)
dst = template_str(context, self.tracking_local_dst)
sub = template_str(context, self.tracking_subpath)
tracking_id = f"{url}:{dst}/{sub}"
check_valid_path(dst)
check_valid_relative_path(sub)
return (tracking_id, url, dst, sub)
def template(context: Context, dst: str, src: str = None, content: str = None, mode=None, owner=None, group=None):
"""
Templates the given src file or given content and copies the output to the remote host at dst.
Either content or src must be specified.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
src : str, optional
The local source file path relative to the project directory. Will be templated. Mutually exclusive with content.
content : str, optional
The content for the file. Will be templated. Mutually exclusive with src.
dst : str
The remote destination file path. Will be templated.
mode : int, optional
The new file mode. Defaults the current context file creation mode.
owner : str, optional
The new file owner. Defaults the current context owner.
group : str, optional
The new file group. Defaults the current context group.
Returns
-------
CompletedTransaction
The completed transaction
"""
if src is not None:
src = template_str(context, src)
dst = template_str(context, dst)
check_valid_path(dst)
if content is None and src is None:
raise LogicError("Either src or content must be given.")
if content is not None and src is not None:
raise LogicError("Exactly one of src or content must be given.")
def get_content():
if content is not None:
return template_str(context, content)
# Get templated content
try:
templ = context.host.manager.jinja2_env.get_template(src)
except TemplateNotFound as e:
raise LogicError("Template not found: " + str(e)) from e
try:
return templ.render(context.vars_dict)
except UndefinedError as e:
raise MessageError(f"Error while templating '{src}': " + str(e)) from e
return remote_upload(context, get_content, title="template", name=dst, dst=dst, mode=mode, owner=owner, group=group)
def package(context: Context, name: str, state="present", opts: list[str] = None):
"""
Installs or uninstalls the given package name (depending on state == "present" or "absent").
Additional options to apt-get can be passed via opts, and will be appended
before the package name. opts will be templated.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
name : str
The package name to be installed or uninstalled. Will be templated.
state : str, optional
The desired state, either "present" or "absent". Defaults to "present".
opts : list[str]
Additional options to pacman. Will be templated.
Returns
-------
CompletedTransaction
The completed transaction
"""
opts = [] if opts is None else [template_str(context, o) for o in opts]
def install(context, name):
apt_cmd = ["apt-get", "install"]
apt_cmd.extend(opts)
apt_cmd.append(name)
context.remote_exec(apt_cmd, checked=True)
def uninstall(context, name):
context.remote_exec(["apt-get", "remove"] + opts + [name], checked=True)
generic_package(context, name, state, is_installed, install, uninstall)
def directory(context: Context, path: str, mode=None, owner=None, group=None):
"""
Creates the given directory on the remote.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
path : str
The directory path to create (will be templated). Parent directory must exist.
mode : int, optional
The new directory mode. Defaults the current context directory creation mode.
owner : str, optional
The new directory owner. Defaults the current context owner.
group : str, optional
The new directory group. Defaults the current context group.
Returns
-------
CompletedTransaction
The completed transaction
"""
path = template_str(context, path)
check_valid_path(path)
with context.transaction(title="dir", name=path) as action:
mode, owner, group = resolve_mode_owner_group(context, mode, owner, group, context.dir_mode)
# Get current state
(cur_ft, cur_mode, cur_owner, cur_group) = remote_stat(context, path)
# Record this initial state
if cur_ft is None:
action.initial_state(exists=False, mode=None, owner=None, group=None)
elif cur_ft == "directory":
action.initial_state(exists=True, mode=cur_mode, owner=cur_owner, group=cur_group)
if mode == cur_mode and owner == cur_owner and group == cur_group:
return action.unchanged()
else:
raise LogicError("Cannot create directory on remote: Path already exists and is not a directory")
# Record the final state
action.final_state(exists=True, mode=mode, owner=owner, group=group)
# Apply actions to reach new state, if we aren't in pretend mode
if not context.pretend:
try:
# If stat failed, the directory doesn't exist and we need to create it.
if cur_ft is None:
context.remote_exec(["mkdir", path], checked=True)
# Set permissions
context.remote_exec(["chown", f"{owner}:{group}", path], checked=True)
context.remote_exec(["chmod", mode, path], checked=True)
except RemoteExecError as e:
return action.failure(e)
# Return success
return action.success()
def generic_package(context: Context, atom: str, state: str, is_installed,
install, uninstall):
"""
Installs or uninstalls (depending if state == "present" or "absent") the given
package atom. Additional options to emerge can be passed via opts, and will be appended
before the package atom. opts will be templated.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
atom : str
The package name to be installed or uninstalled. Will be templated.
state : str
The desired state, either "present" or "absent".
is_installed : Callable[[Context, str], bool]
Callback used to determine if a given package is installed.
install : Callable[[Context, str], None]
Callback used to install a package on the remote.
uninstall : Callable[[Context, str], None]
Callback used to uninstall a package on the remote.
Returns
-------
CompletedTransaction
The completed transaction
"""
# pylint: disable=R0801
if state not in ["present", "absent"]:
raise LogicError(f"Invalid package state '{state}'")
atom = template_str(context, atom)
with context.transaction(title="package", name=atom) as action:
# Query current state
installed = is_installed(context, atom)
# Record this initial state, and return early
# if there is nothing to do
action.initial_state(installed=installed)
should_install = (state == "present")
if installed == should_install:
return action.unchanged()
# Record the final state
action.final_state(installed=should_install)
# Apply actions to reach new state, if we aren't in pretend mode
if not context.pretend:
if should_install:
install(context, atom)
else:
uninstall(context, atom)
# Return success
return action.success()
def save_output(context: Context, command: list[str], dst: str, desc=None, mode=None, owner=None, group=None):
"""
Saves the stdout of the given command on the remote host at remote dst.
Using --pretend will still run the command, but won't save the output.
Changed status reflects if the file contents changed.
Optionally accepts file mode, owner and group, if not given, context defaults are used.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
command: list[str]
A list containing the command and its arguments. Each one will be templated.
dst : str
The remote destination file path. Will be templated.
desc : str
A description to be printed in the summary when executing.
mode : int, optional
The new file mode. Defaults the current context file creation mode.
owner : str, optional
The new file owner. Defaults the current context owner.
group : str, optional
The new file group. Defaults the current context group.
Returns
-------
CompletedTransaction
The completed transaction
"""
command = [template_str(context, c) for c in command]
dst = template_str(context, dst)
check_valid_path(dst)
def get_content():
# Get command output
return context.remote_exec(command, checked=True).stdout
name = f"{command}" if desc is None else desc
return remote_upload(context, get_content, title="save out", name=name, dst=dst, mode=mode, owner=owner, group=group)
def get_content():
if content is not None:
return template_str(context, content)
# Get templated content
try:
templ = context.host.manager.jinja2_env.get_template(src)
except TemplateNotFound as e:
raise LogicError("Template not found: " + str(e)) from e
try:
return templ.render(context.vars_dict)
except UndefinedError as e:
raise MessageError(f"Error while templating '{src}': " + str(e)) from e
def run(self, context):
with context.defaults(user=self.tracked_task.tracking_user,
owner=self.tracked_task.tracking_user,
group=self.tracked_task.tracking_group):
# Get tracking specific variables
(url, dst,
_) = context.cache["tracking"][self.tracked_task.tracking_id]
# Clone or update remote tracking repository
git.checkout(context, url, dst)
if not context.pretend:
# Set given git repo configs
for k, v in self.tracked_task.tracking_repo_configs.items(
):
v = template_str(context, v)
context.remote_exec(
["git", "-C", dst, "config", "--local", k, v],
checked=True)
def package(context: Context, atom: str, state="present", oneshot=False, opts: list[str] = None):
"""
Installs or uninstalls the given package atom (depending on state == "present" or "absent").
Additional options to emerge can be passed via opts, and will be appended
before the package atom. opts will be templated.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
atom : str
The package name to be installed or uninstalled. Will be templated.
state : str, optional
The desired state, either "present" or "absent". Defaults to "present".
oneshot : bool, optional
Use portage option --oneshot. Defaults to false.
opts : list[str]
Additional options to portage. Will be templated.
Returns
-------
CompletedTransaction
The completed transaction
"""
opts = [] if opts is None else [template_str(context, o) for o in opts]
def install(context, atom):
emerge_cmd = ["emerge", "--color=y", "--verbose"]
if oneshot:
emerge_cmd.append("--oneshot")
emerge_cmd.extend(opts)
emerge_cmd.append(atom)
context.remote_exec(emerge_cmd, checked=True)
def uninstall(context, atom):
context.remote_exec(["emerge", "--color=y", "--verbose", "--depclean"] + opts + [atom], checked=True)
generic_package(context, atom, state, is_installed, install, uninstall)
def user(context: Context,
name: str,
group: str = None,
groups: list[str] = None,
append_groups: bool = False,
state: str = "present",
system: bool = False,
shell: str = None,
password: str = None,
home: str = None,
create_home = True):
# pylint: disable=R0912,R0915
"""
Creates or modifies a unix user. Because we interally call ``userdel``, removing a user will also remove
it's associated primary group if no other user belongs to it.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
name : str
The name of the user to create or modify. Will be templated.
group: str, optional
The primary group of the user. If given, the group must already exists. Otherwise, a group will be created with the same name as the user,
if a user is created by this action. Will be templated.
groups: list[str], optional
Supplementary groups for the user.
append_groups: bool
If ``True``, the user will be added to all given supplementary groups. If ``False``, the user will be added to exactly the given supplementary groups and removed from other groups.
state: str
If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters.
system: bool
If ``True`` the user will be created as a system user. This has no effect on existing users.
shell: str
Specifies the shell for the user. Defaults to ``/sbin/nologin`` if not given but a user needs to be created. Will be templated.
password : str, optional
Will update the password hash to the given vaule of the user. Use ``!`` to lock the account.
Defaults to '!' if not given but a user needs to be created. Will be templated.
You can generate a password hash by using the following script:
>>> import crypt, getpass
>>> crypt.crypt(getpass.getpass(), crypt.mksalt(crypt.METHOD_SHA512))
'$6$rwn5z9MlYvcnE222$9wOP6Y6EcnF.cZ7BUjttWeSQNdOQWI...'
home: str, optional
The home directory for the user. Will be left empty if not given but a user needs to be created. Will be templated.
create_home: bool
If ``True`` and home was given, create the home directory of the user if it doesn't exist.
Returns
-------
CompletedTransaction
The completed transaction
"""
name = template_str(context, name)
group = template_str(context, group) if group is not None else None
home = template_str(context, home) if home is not None else None
password = template_str(context, password) if password is not None else None
shell = template_str(context, shell) if shell is not None else None
# pylint: disable=R0801
if state not in ["present", "absent"]:
raise LogicError(f"Invalid user state '{state}'")
if home is not None:
check_valid_path(home)
with context.transaction(title="user", name=name) as action:
pwinfo = context.remote_exec(["python", "-c", (
'import sys,grp,pwd,spwd\n'
'try:\n'
' p=pwd.getpwnam(sys.argv[1])\n'
'except KeyError:\n'
' print("0:")\n'
' sys.exit(0)\n'
'g=grp.getgrgid(p.pw_gid)\n'
's=spwd.getspnam(p.pw_name)\n'
'grps=\',\'.join([sg.gr_name for sg in grp.getgrall() if p.pw_name in sg.gr_mem])\n'
'print(f"1:{g.gr_name}:{grps}:{p.pw_dir}:{p.pw_shell}:{s.sp_pwdp}")')
, name], checked=True)
exists = pwinfo.stdout.strip().startswith('1:')
if exists:
( _
, cur_group
, cur_groups
, cur_home
, cur_shell
, cur_password
) = pwinfo.stdout.strip().split(':')
cur_groups = list(sorted(set([] if cur_groups == '' else cur_groups.split(','))))
else:
cur_group = None
cur_groups = []
cur_home = None
cur_shell = None
cur_password = None
if state == "absent":
action.initial_state(exists=exists)
if not exists:
return action.unchanged()
action.final_state(exists=False)
if not context.pretend:
try:
context.remote_exec(["userdel", name], checked=True)
except RemoteExecError as e:
return action.failure(e)
else:
action.initial_state(exists=exists, group=cur_group, groups=cur_groups, home=cur_home, shell=cur_shell, pw=cur_password)
fin_group = group or cur_group or name
fin_groups = cur_groups if groups is None else list(sorted(set(cur_groups + groups if append_groups else groups)))
fin_home = home or cur_home
fin_shell = shell or cur_shell or '/sbin/nologin'
fin_password = password or cur_password or '!'
action.final_state(exists=True, group=fin_group, groups=fin_groups, home=fin_home, shell=fin_shell, pw=fin_password)
if not context.pretend:
try:
if exists:
# Only apply changes to the existing user
if cur_group != fin_group:
context.remote_exec(["usermod", "--gid", fin_group, name], checked=True)
if cur_groups != fin_groups:
context.remote_exec(["usermod", "--groups", ','.join(fin_groups), name], checked=True)
if cur_home != fin_home:
context.remote_exec(["usermod", "--home", fin_home, name], checked=True)
if cur_shell != fin_shell:
context.remote_exec(["usermod", "--shell", fin_shell, name], checked=True)
if cur_password != fin_password:
context.remote_exec(["usermod", "--password", fin_password, name], checked=True)
else:
# Create a new user so that is results in the given final state
command = ["useradd"]
if system:
command.append("--system")
# Primary group
if group is None:
command.append("--user-group")
else:
command.extend(["--no-user-group", "--gid", group])
# Supplementary groups
if len(fin_groups) > 0:
command.extend(["--groups", ','.join(fin_groups)])
command.extend(["--no-create-home", "--home-dir", fin_home or ''])
command.extend(["--shell", fin_shell])
command.extend(["--password", fin_password])
command.append(name)
context.remote_exec(command, checked=True)
except RemoteExecError as e:
return action.failure(e)
# Remember result
action_result = action.success()
# Create home directory afterwards if necessary
if state == "present" and create_home and cur_home is None and fin_home:
directory(context, path=fin_home, mode=0o700, owner=name, group=fin_group)
return action_result
def group(context: Context,
name: str,
state: str = "present",
system: bool = False):
"""
Creates or deletes a unix group.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
name : str
The name of the user to create or modify. Will be templated.
state: str
If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters.
system: bool
If ``True`` the user will be created as a system user. This has no effect on existing users.
Returns
-------
CompletedTransaction
The completed transaction
"""
name = template_str(context, name)
# pylint: disable=R0801
if state not in ["present", "absent"]:
raise LogicError(f"Invalid user state '{state}'")
with context.transaction(title="group", name=name) as action:
grinfo = context.remote_exec(["python", "-c", (
'import sys,grp\n'
'try:\n'
' g=grp.getgrnam(sys.argv[1])\n'
'except KeyError:\n'
' print("0")\n'
' sys.exit(0)\n'
'print("1")')
, name], checked=True)
exists = grinfo.stdout.strip().startswith('1')
if state == "absent":
action.initial_state(exists=exists)
if not exists:
return action.unchanged()
action.final_state(exists=False)
if not context.pretend:
try:
context.remote_exec(["groupdel", name], checked=True)
except RemoteExecError as e:
return action.failure(e)
else:
action.initial_state(exists=exists)
if exists:
return action.unchanged()
action.final_state(exists=True)
if not context.pretend:
try:
command = ["groupadd"]
if system:
command.append("--system")
command.append(name)
context.remote_exec(command, checked=True)
except RemoteExecError as e:
return action.failure(e)
return action.success()
def checkout(context: Context,
url: str,
dst: str,
update: bool = True,
depth=None):
"""
Checkout (and optionally update) the given git repository to dst.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
url : str
The url of the git repository to checkout. Will be templated.
dst : str
The remote destination path for the repository. Will be templated.
update: bool, optional
Also tries to update the repository if it is already cloned. Defaults to true.
depth : str, optional
Restrict repository cloning depth. Beware that updates might not work correctly because of forced updates.
Returns
-------
CompletedTransaction
The completed transaction
"""
url = template_str(context, url)
dst = template_str(context, dst)
check_valid_path(dst)
with context.transaction(title="checkout", name=dst) as action:
# Add url as extra info
action.extra_info(url=url)
# Query current state
(cur_ft, _, _, _) = remote_stat(context, dst)
# Record this initial state
if cur_ft is None:
action.initial_state(cloned=False, commit=None)
cloned = False
elif cur_ft == "directory":
# Assert that it is a git directory
(cur_git_ft, _, _, _) = remote_stat(context, dst + "/.git")
if cur_git_ft != 'directory':
raise LogicError(
"Cannot checkout git repository on remote: Directory already exists and is not a git repository"
)
remote_commit = context.remote_exec(
["git", "-C", dst, "rev-parse", "HEAD"],
error_verbosity=0,
verbosity=2)
if remote_commit.return_code != 0:
raise LogicError(
"Cannot checkout git repository on remote: Directory already exists but 'git rev-parse HEAD' failed"
)
cur_commit = remote_commit.stdout.strip()
action.initial_state(cloned=True, commit=cur_commit)
cloned = True
else:
raise LogicError(
"Cannot checkout git repository on remote: Path already exists but isn't a directory"
)
# If the repository is already cloned but we shouldn't update,
# nothing will change and we are done.
if cloned and not update:
return action.unchanged()
# Check the newest available commit
remote_newest_commit = context.remote_exec(
["git", "ls-remote", "--exit-code", url, "HEAD"], checked=True)
newest_commit = remote_newest_commit.stdout.strip().split()[0]
# Record the final state
action.final_state(cloned=True, commit=newest_commit)
# Apply actions to reach new state, if we aren't in pretend mode
if not context.pretend:
try:
if not cloned:
clone_cmd = ["git", "clone"]
if depth is not None:
clone_cmd.append("--depth")
clone_cmd.append(str(depth))
clone_cmd.append(url)
clone_cmd.append(dst)
context.remote_exec(clone_cmd, checked=True)
else:
pull_cmd = ["git", "-C", dst, "pull", "--ff-only"]
if depth is not None:
pull_cmd.append("--depth")
pull_cmd.append(str(depth))
context.remote_exec(pull_cmd, checked=True)
except RemoteExecError as e:
return action.failure(e)
# Return success
return action.success()
def _track(self, context):
"""
Uses rsync to copy the tracking paths into the repository, and
creates and pushes a commit if there are any changes.
"""
with context.defaults(user=self.tracking_user,
owner=self.tracking_user,
group=self.tracking_group):
(_, dst, sub) = context.cache["tracking"][self.tracking_id]
# Check source paths
srcs = []
for src in self.tracking_paths:
src = template_str(context, src)
check_valid_path(src)
srcs.append(src)
# Begin transaction
with context.transaction(title="track", name=f"{srcs}") as action:
action.initial_state(added=None, modified=None, deleted=None)
if not context.pretend:
mode, owner, group = resolve_mode_owner_group(
context, None, None, None, context.dir_mode)
rsync_dst = f"{dst}/{sub}/"
base_parts = PurePosixPath(dst).parts
parts = PurePosixPath(rsync_dst).parts
# Create tracking destination subdirectories if they don't exist
cur = dst
for p in parts[len(base_parts):]:
cur = os.path.join(cur, p)
context.remote_exec(["mkdir", "-p", "--", cur],
checked=True)
context.remote_exec(["chown", f"{owner}:{group}", cur],
checked=True)
context.remote_exec(["chmod", mode, cur], checked=True)
# Use rsync to backup all paths into the repository
for src in srcs:
context.remote_exec([
"rsync", "--quiet", "--recursive",
"--one-file-system", "--links", "--times",
"--relative",
str(PurePosixPath(src)), rsync_dst
],
checked=True,
user="******",
umask=0o022)
# Make files accessible to the tracking user
context.remote_exec([
"chown", "--recursive", "--preserve-root",
f"{self.tracking_user}:{self.tracking_group}", dst
],
checked=True,
user="******")
# Add all changes
context.remote_exec(["git", "-C", dst, "add", "--all"],
checked=True)
# Query changes for message
remote_status = context.remote_exec(
["git", "-C", dst, "status", "--porcelain"],
checked=True)
if remote_status.stdout.strip() != "":
# We have changes
added = 0
modified = 0
deleted = 0
for line in remote_status.stdout.splitlines():
if line.startswith("A"):
added += 1
elif line.startswith("M"):
modified += 1
elif line.startswith("D"):
deleted += 1
action.final_state(added=added,
modified=modified,
deleted=deleted)
# Create commit
commit_opts = [
template_str(context, o)
for o in self.tracking_git_commit_opts
]
context.remote_exec([
"git", "-C", dst, "commit"
] + commit_opts + [
"--message",
f"task {self.identifier}: {added=}, {modified=}, {deleted=}"
],
checked=True)
# Push commit
context.remote_exec(
["git", "-C", dst, "push", "origin", "master"],
checked=True)
action.success()
else:
# Repo is still clean
action.unchanged()
else:
action.final_state(added="? (pretend)",
modified="? (pretend)",
deleted="? (pretend)")
action.success()