def finalize(self, context, transaction):
"""
Finalizes this transaction, which will verify that all states are set corectly, and print the transaction.
Parameters
----------
context : Context
The associated context.
transaction : Transaction
The transaction that should be finalized.
"""
if self.result is None:
raise LogicError(
"A transaction cannot be completed without a result status.")
if self.result.initial_state is None:
raise LogicError(
"A transaction cannot be completed without an initial state.")
if self.result.final_state is None:
raise LogicError(
"A transaction cannot be completed without a final state.")
if set(self.result.initial_state.keys()) != set(
self.result.final_state.keys()):
raise LogicError(
"Both initial and final transaction state must have the same keys."
)
self.result.title = transaction.title
self.result.name = transaction.name
print_transaction(context, self.result)
if not self.result.success:
raise TransactionError(self.result)
def template(context: Context, dst: str, src: str = None, content: str = None, mode=None, owner=None, group=None):
"""
Templates the given src file or given content and copies the output to the remote host at dst.
Either content or src must be specified.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
src : str, optional
The local source file path relative to the project directory. Will be templated. Mutually exclusive with content.
content : str, optional
The content for the file. Will be templated. Mutually exclusive with src.
dst : str
The remote destination file path. Will be templated.
mode : int, optional
The new file mode. Defaults the current context file creation mode.
owner : str, optional
The new file owner. Defaults the current context owner.
group : str, optional
The new file group. Defaults the current context group.
Returns
-------
CompletedTransaction
The completed transaction
"""
if src is not None:
src = template_str(context, src)
dst = template_str(context, dst)
check_valid_path(dst)
if content is None and src is None:
raise LogicError("Either src or content must be given.")
if content is not None and src is not None:
raise LogicError("Exactly one of src or content must be given.")
def get_content():
if content is not None:
return template_str(context, content)
# Get templated content
try:
templ = context.host.manager.jinja2_env.get_template(src)
except TemplateNotFound as e:
raise LogicError("Template not found: " + str(e)) from e
try:
return templ.render(context.vars_dict)
except UndefinedError as e:
raise MessageError(f"Error while templating '{src}': " + str(e)) from e
return remote_upload(context, get_content, title="template", name=dst, dst=dst, mode=mode, owner=owner, group=group)
def check_valid_relative_path(path):
"""
Asserts that a given path is non empty and relative.
Parameters
----------
path : str
The path to check.
"""
if not path:
raise LogicError("Path must be non-empty")
if path[0] == "/":
raise LogicError("Path must be relative")
def __init__(self, manager):
self.manager = manager
if self.identifier is None:
raise LogicError(
"A task must override the static variable 'identifier'")
if self.description is None:
raise LogicError(
"A task must override the static variable 'description'")
# Initialize variable defaults
self.var_enabled = f"tasks.{self.identifier}.enabled"
self.manager.set(self.var_enabled, True)
self.set_defaults()
def __init__(self, manager):
super().__init__(manager)
if self.tracking_repo_url is None:
raise LogicError(
"A tracked task must override the variable 'tracking_repo_url'"
)
if self.tracking_local_dst is None:
raise LogicError(
"A tracked task must override the variable 'tracking_local_dst'"
)
if self.tracking_paths == []:
raise LogicError(
"A tracked task must override the variable 'tracking_paths'")
self.tracking_id = None
def check_valid_key(key: str, msg: str = "Invalid key"):
"""
Asserts that a given key is a valid identifier.
Parameters
----------
key : str
The key to check.
msg : str, optional
The message to raise when the check fails.
"""
if not key:
raise LogicError("Invalid empty key")
if '..' in key or not all([k.isidentifier() for k in key.split('.')]):
raise LogicError(f"{msg}: '{key}' is not a valid identifier")
def set(self, key, value):
"""
Sets the given variable.
Parameters
----------
key : str
The key that should be read.
value : Any, optional
The value to be stored. Must be json (de-)serializable.
"""
check_valid_key(key)
d = self.vars
cs = []
keys = key.split('.')
for k in keys[:-1]:
if k not in d:
d[k] = {}
d = d[k]
cs.append(k)
if not isinstance(d, dict):
csname = '.'.join(cs)
raise LogicError(f"Cannot set variable '{key}' because existing variable '{csname}' is not a dictionary")
if self.warn_on_redefinition and keys[-1] in d:
print(f"[1;33mwarning:[m [1mRedefinition of variable '{key}':[m previous={d[keys[-1]]} new={value}")
d[keys[-1]] = value
def failure(self, reason, set_final_state=False, **kwargs):
"""
Completes the transaction, marking it as failed with the given reason.
If reason is a RemoteExecError, additional information will be printed.
Parameters
----------
reason : str
The reason for the failure.
set_final_state : bool
If true, the final transaction state will be set. Defaults to false.
**kwargs
Final state assocations
Returns
-------
CompletedTransaction
The completed transaction.
"""
if isinstance(reason, RemoteExecError):
e = reason
reason = f"{type(e).__name__}: {str(e)}\n"
reason += e.ret.stderr
if set_final_state:
self.final_state(**self.initial_state_dict)
if self.result is not None:
raise LogicError(
"A transaction cannot be completed multiple times.")
self.result = CompletedTransaction(self,
success=False,
failure_reason=reason,
store=kwargs)
return self.result
def __enter__(self):
"""
Begins a new transaction
"""
if self.transaction_context is not None:
raise LogicError("A transaction may only be started once.")
self.transaction_context = ActiveTransaction()
print_transaction_early(self)
return self.transaction_context
def generic_package(context: Context, atom: str, state: str, is_installed,
install, uninstall):
"""
Installs or uninstalls (depending if state == "present" or "absent") the given
package atom. Additional options to emerge can be passed via opts, and will be appended
before the package atom. opts will be templated.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
atom : str
The package name to be installed or uninstalled. Will be templated.
state : str
The desired state, either "present" or "absent".
is_installed : Callable[[Context, str], bool]
Callback used to determine if a given package is installed.
install : Callable[[Context, str], None]
Callback used to install a package on the remote.
uninstall : Callable[[Context, str], None]
Callback used to uninstall a package on the remote.
Returns
-------
CompletedTransaction
The completed transaction
"""
# pylint: disable=R0801
if state not in ["present", "absent"]:
raise LogicError(f"Invalid package state '{state}'")
atom = template_str(context, atom)
with context.transaction(title="package", name=atom) as action:
# Query current state
installed = is_installed(context, atom)
# Record this initial state, and return early
# if there is nothing to do
action.initial_state(installed=installed)
should_install = (state == "present")
if installed == should_install:
return action.unchanged()
# Record the final state
action.final_state(installed=should_install)
# Apply actions to reach new state, if we aren't in pretend mode
if not context.pretend:
if should_install:
install(context, atom)
else:
uninstall(context, atom)
# Return success
return action.success()
def directory(context: Context, path: str, mode=None, owner=None, group=None):
"""
Creates the given directory on the remote.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
path : str
The directory path to create (will be templated). Parent directory must exist.
mode : int, optional
The new directory mode. Defaults the current context directory creation mode.
owner : str, optional
The new directory owner. Defaults the current context owner.
group : str, optional
The new directory group. Defaults the current context group.
Returns
-------
CompletedTransaction
The completed transaction
"""
path = template_str(context, path)
check_valid_path(path)
with context.transaction(title="dir", name=path) as action:
mode, owner, group = resolve_mode_owner_group(context, mode, owner, group, context.dir_mode)
# Get current state
(cur_ft, cur_mode, cur_owner, cur_group) = remote_stat(context, path)
# Record this initial state
if cur_ft is None:
action.initial_state(exists=False, mode=None, owner=None, group=None)
elif cur_ft == "directory":
action.initial_state(exists=True, mode=cur_mode, owner=cur_owner, group=cur_group)
if mode == cur_mode and owner == cur_owner and group == cur_group:
return action.unchanged()
else:
raise LogicError("Cannot create directory on remote: Path already exists and is not a directory")
# Record the final state
action.final_state(exists=True, mode=mode, owner=owner, group=group)
# Apply actions to reach new state, if we aren't in pretend mode
if not context.pretend:
try:
# If stat failed, the directory doesn't exist and we need to create it.
if cur_ft is None:
context.remote_exec(["mkdir", path], checked=True)
# Set permissions
context.remote_exec(["chown", f"{owner}:{group}", path], checked=True)
context.remote_exec(["chmod", mode, path], checked=True)
except RemoteExecError as e:
return action.failure(e)
# Return success
return action.success()
def run_task(self, registered_task_class):
"""
Runs the registered instance (see Manager) for the given task class.
Parameters
----------
registered_task_class : class(Task)
The task class that should be executed. The registered instance is found first, and then called.
"""
instance = self.manager.tasks.get(registered_task_class.identifier,
None)
if not instance:
raise LogicError(
f"Cannot run unregistered task {registered_task_class}")
if not isinstance(instance, registered_task_class):
raise LogicError(
"Cannot run unrelated task with the same identifier as a registered task."
)
instance.exec(self)
def final_state(self, **kwargs):
"""
Records the (expected) final state of the remote.
Parameters
----------
**kwargs
Final state assocations
"""
if self.result is not None:
raise LogicError(
"A transaction cannot be altered after it is completed")
self.final_state_dict = dict(kwargs)
def get_content():
if content is not None:
return template_str(context, content)
# Get templated content
try:
templ = context.host.manager.jinja2_env.get_template(src)
except TemplateNotFound as e:
raise LogicError("Template not found: " + str(e)) from e
try:
return templ.render(context.vars_dict)
except UndefinedError as e:
raise MessageError(f"Error while templating '{src}': " + str(e)) from e
def get_or_throw(key):
"""
Retrieves a variable by the given key or raises a KeyError if no such key exists.
"""
check_valid_key(key)
d = self.vars
cs = []
for k in key.split('.'):
cs.append(k)
if not isinstance(d, dict):
csname = '.'.join(cs)
raise LogicError(f"Cannot access variable '{key}' because '{csname}' is not a dictionary")
if k not in d:
raise KeyError(f"Variable '{key}' does not exist")
d = d[k]
return d
def _assert_tracking_repo_clean(self, context):
"""
Asserts that the tracking repository is clean, so we will never
confuse different changes in a commit.
"""
(_, dst, _) = context.cache["tracking"][self.tracking_id]
with context.defaults(user=self.tracking_user,
owner=self.tracking_user,
group=self.tracking_group):
if not context.pretend:
# Assert that the repository is clean
remote_status = context.remote_exec(
["git", "-C", dst, "status", "--porcelain"], checked=True)
if remote_status.stdout.strip() != "":
raise LogicError(
"Refusing operation: Tracking repository is not clean!"
)
def success(self, **kwargs):
"""
Completes the transaction with successful status.
Parameters
----------
**kwargs
Final state assocations
Returns
-------
CompletedTransaction
The completed transaction.
"""
if self.result is not None:
raise LogicError(
"A transaction cannot be completed multiple times.")
self.result = CompletedTransaction(self, success=True, store=kwargs)
return self.result
def template_str(context: Context, content : str) -> str:
"""
Renders the given string template.
Parameters
----------
context : Context
The context providing the templating dictionary
content : str
The string to template
Returns
-------
str
The templated string
"""
templ = Template(content)
try:
return templ.render(context.vars_dict)
except UndefinedError as e:
raise LogicError(f"Error while templating string '{content}': " + str(e)) from e
def encrypt_content(self, plaintext: bytes) -> bytes:
"""
Encrypts the given plaintext.
Parameters
----------
plaintext : bytes
The bytes to encrypt.
Returns
-------
bytes
The ciphertext
"""
if self.recipient is None:
raise LogicError("GpgVault encryption requires a recipient")
return subprocess.run(
["gpg", "--quiet", "--encrypt", "--recipient", self.recipient],
input=plaintext,
capture_output=True,
check=True).stdout
def user(context: Context,
name: str,
group: str = None,
groups: list[str] = None,
append_groups: bool = False,
state: str = "present",
system: bool = False,
shell: str = None,
password: str = None,
home: str = None,
create_home = True):
# pylint: disable=R0912,R0915
"""
Creates or modifies a unix user. Because we interally call ``userdel``, removing a user will also remove
it's associated primary group if no other user belongs to it.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
name : str
The name of the user to create or modify. Will be templated.
group: str, optional
The primary group of the user. If given, the group must already exists. Otherwise, a group will be created with the same name as the user,
if a user is created by this action. Will be templated.
groups: list[str], optional
Supplementary groups for the user.
append_groups: bool
If ``True``, the user will be added to all given supplementary groups. If ``False``, the user will be added to exactly the given supplementary groups and removed from other groups.
state: str
If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters.
system: bool
If ``True`` the user will be created as a system user. This has no effect on existing users.
shell: str
Specifies the shell for the user. Defaults to ``/sbin/nologin`` if not given but a user needs to be created. Will be templated.
password : str, optional
Will update the password hash to the given vaule of the user. Use ``!`` to lock the account.
Defaults to '!' if not given but a user needs to be created. Will be templated.
You can generate a password hash by using the following script:
>>> import crypt, getpass
>>> crypt.crypt(getpass.getpass(), crypt.mksalt(crypt.METHOD_SHA512))
'$6$rwn5z9MlYvcnE222$9wOP6Y6EcnF.cZ7BUjttWeSQNdOQWI...'
home: str, optional
The home directory for the user. Will be left empty if not given but a user needs to be created. Will be templated.
create_home: bool
If ``True`` and home was given, create the home directory of the user if it doesn't exist.
Returns
-------
CompletedTransaction
The completed transaction
"""
name = template_str(context, name)
group = template_str(context, group) if group is not None else None
home = template_str(context, home) if home is not None else None
password = template_str(context, password) if password is not None else None
shell = template_str(context, shell) if shell is not None else None
# pylint: disable=R0801
if state not in ["present", "absent"]:
raise LogicError(f"Invalid user state '{state}'")
if home is not None:
check_valid_path(home)
with context.transaction(title="user", name=name) as action:
pwinfo = context.remote_exec(["python", "-c", (
'import sys,grp,pwd,spwd\n'
'try:\n'
' p=pwd.getpwnam(sys.argv[1])\n'
'except KeyError:\n'
' print("0:")\n'
' sys.exit(0)\n'
'g=grp.getgrgid(p.pw_gid)\n'
's=spwd.getspnam(p.pw_name)\n'
'grps=\',\'.join([sg.gr_name for sg in grp.getgrall() if p.pw_name in sg.gr_mem])\n'
'print(f"1:{g.gr_name}:{grps}:{p.pw_dir}:{p.pw_shell}:{s.sp_pwdp}")')
, name], checked=True)
exists = pwinfo.stdout.strip().startswith('1:')
if exists:
( _
, cur_group
, cur_groups
, cur_home
, cur_shell
, cur_password
) = pwinfo.stdout.strip().split(':')
cur_groups = list(sorted(set([] if cur_groups == '' else cur_groups.split(','))))
else:
cur_group = None
cur_groups = []
cur_home = None
cur_shell = None
cur_password = None
if state == "absent":
action.initial_state(exists=exists)
if not exists:
return action.unchanged()
action.final_state(exists=False)
if not context.pretend:
try:
context.remote_exec(["userdel", name], checked=True)
except RemoteExecError as e:
return action.failure(e)
else:
action.initial_state(exists=exists, group=cur_group, groups=cur_groups, home=cur_home, shell=cur_shell, pw=cur_password)
fin_group = group or cur_group or name
fin_groups = cur_groups if groups is None else list(sorted(set(cur_groups + groups if append_groups else groups)))
fin_home = home or cur_home
fin_shell = shell or cur_shell or '/sbin/nologin'
fin_password = password or cur_password or '!'
action.final_state(exists=True, group=fin_group, groups=fin_groups, home=fin_home, shell=fin_shell, pw=fin_password)
if not context.pretend:
try:
if exists:
# Only apply changes to the existing user
if cur_group != fin_group:
context.remote_exec(["usermod", "--gid", fin_group, name], checked=True)
if cur_groups != fin_groups:
context.remote_exec(["usermod", "--groups", ','.join(fin_groups), name], checked=True)
if cur_home != fin_home:
context.remote_exec(["usermod", "--home", fin_home, name], checked=True)
if cur_shell != fin_shell:
context.remote_exec(["usermod", "--shell", fin_shell, name], checked=True)
if cur_password != fin_password:
context.remote_exec(["usermod", "--password", fin_password, name], checked=True)
else:
# Create a new user so that is results in the given final state
command = ["useradd"]
if system:
command.append("--system")
# Primary group
if group is None:
command.append("--user-group")
else:
command.extend(["--no-user-group", "--gid", group])
# Supplementary groups
if len(fin_groups) > 0:
command.extend(["--groups", ','.join(fin_groups)])
command.extend(["--no-create-home", "--home-dir", fin_home or ''])
command.extend(["--shell", fin_shell])
command.extend(["--password", fin_password])
command.append(name)
context.remote_exec(command, checked=True)
except RemoteExecError as e:
return action.failure(e)
# Remember result
action_result = action.success()
# Create home directory afterwards if necessary
if state == "present" and create_home and cur_home is None and fin_home:
directory(context, path=fin_home, mode=0o700, owner=name, group=fin_group)
return action_result
def group(context: Context,
name: str,
state: str = "present",
system: bool = False):
"""
Creates or deletes a unix group.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
name : str
The name of the user to create or modify. Will be templated.
state: str
If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters.
system: bool
If ``True`` the user will be created as a system user. This has no effect on existing users.
Returns
-------
CompletedTransaction
The completed transaction
"""
name = template_str(context, name)
# pylint: disable=R0801
if state not in ["present", "absent"]:
raise LogicError(f"Invalid user state '{state}'")
with context.transaction(title="group", name=name) as action:
grinfo = context.remote_exec(["python", "-c", (
'import sys,grp\n'
'try:\n'
' g=grp.getgrnam(sys.argv[1])\n'
'except KeyError:\n'
' print("0")\n'
' sys.exit(0)\n'
'print("1")')
, name], checked=True)
exists = grinfo.stdout.strip().startswith('1')
if state == "absent":
action.initial_state(exists=exists)
if not exists:
return action.unchanged()
action.final_state(exists=False)
if not context.pretend:
try:
context.remote_exec(["groupdel", name], checked=True)
except RemoteExecError as e:
return action.failure(e)
else:
action.initial_state(exists=exists)
if exists:
return action.unchanged()
action.final_state(exists=True)
if not context.pretend:
try:
command = ["groupadd"]
if system:
command.append("--system")
command.append(name)
context.remote_exec(command, checked=True)
except RemoteExecError as e:
return action.failure(e)
return action.success()
def resolve_mode_owner_group(context: Context, mode, owner, group, fallback_mode):
"""
Canonicalize mode, owner and group. If any of them is None, the respective
variable will be replaced with the context default.
Parameters
----------
context : Context
The host execution context
mode : int
The mode to canonicalize. May be None.
owner : str
User id or name for the owner. User will be verified as existing on the remote.
group : str
Group id or name for the group. Group will be verified as existing on the remote.
fallback_mode : int
The fallback_mode to canonicalize in case mode = None. Usually set to either context.file_mode or context.dir_mode.
Returns
-------
(str, str, str)
A tuple of (resolved_mode, resolved_owner, resolved_group)
"""
# Resolve mode to string
resolved_mode = _mode_to_str(fallback_mode if mode is None else mode)
# Resolve owner name/uid to name
owner = context.owner if owner is None else owner
remote_cmd_owner_id = context.remote_exec(["python", "-c", (
'import sys,pwd\n'
'try:\n'
' p=pwd.getpwnam(sys.argv[1])\n'
'except KeyError:\n'
' try:\n'
' p=pwd.getpwuid(int(sys.argv[1]))\n'
' except (KeyError, ValueError):\n'
' sys.exit(1)\n'
'print(p.pw_name)')
, owner])
if remote_cmd_owner_id.return_code != 0:
raise LogicError(f"Could not resolve remote user '{owner}'")
resolved_owner = remote_cmd_owner_id.stdout.strip()
# TODO use python because id -ng is wrng
# Resolve group name/gid to name
group = context.group if group is None else group
remote_cmd_group_id = context.remote_exec(["python", "-c", (
'import sys,grp\n'
'try:\n'
' g=grp.getgrnam(sys.argv[1])\n'
'except KeyError:\n'
' try:\n'
' g=grp.getgrgid(int(sys.argv[1]))\n'
' except (KeyError, ValueError):\n'
' sys.exit(1)\n'
'print(g.gr_name)')
, group])
if remote_cmd_group_id.return_code != 0:
raise LogicError(f"Could not resolve remote group '{group}'")
resolved_group = remote_cmd_group_id.stdout.strip()
# Return resolved tuple
return (resolved_mode, resolved_owner, resolved_group)
def checkout(context: Context,
url: str,
dst: str,
update: bool = True,
depth=None):
"""
Checkout (and optionally update) the given git repository to dst.
Parameters
----------
context : Context
The context providing the execution context and templating dictionary.
url : str
The url of the git repository to checkout. Will be templated.
dst : str
The remote destination path for the repository. Will be templated.
update: bool, optional
Also tries to update the repository if it is already cloned. Defaults to true.
depth : str, optional
Restrict repository cloning depth. Beware that updates might not work correctly because of forced updates.
Returns
-------
CompletedTransaction
The completed transaction
"""
url = template_str(context, url)
dst = template_str(context, dst)
check_valid_path(dst)
with context.transaction(title="checkout", name=dst) as action:
# Add url as extra info
action.extra_info(url=url)
# Query current state
(cur_ft, _, _, _) = remote_stat(context, dst)
# Record this initial state
if cur_ft is None:
action.initial_state(cloned=False, commit=None)
cloned = False
elif cur_ft == "directory":
# Assert that it is a git directory
(cur_git_ft, _, _, _) = remote_stat(context, dst + "/.git")
if cur_git_ft != 'directory':
raise LogicError(
"Cannot checkout git repository on remote: Directory already exists and is not a git repository"
)
remote_commit = context.remote_exec(
["git", "-C", dst, "rev-parse", "HEAD"],
error_verbosity=0,
verbosity=2)
if remote_commit.return_code != 0:
raise LogicError(
"Cannot checkout git repository on remote: Directory already exists but 'git rev-parse HEAD' failed"
)
cur_commit = remote_commit.stdout.strip()
action.initial_state(cloned=True, commit=cur_commit)
cloned = True
else:
raise LogicError(
"Cannot checkout git repository on remote: Path already exists but isn't a directory"
)
# If the repository is already cloned but we shouldn't update,
# nothing will change and we are done.
if cloned and not update:
return action.unchanged()
# Check the newest available commit
remote_newest_commit = context.remote_exec(
["git", "ls-remote", "--exit-code", url, "HEAD"], checked=True)
newest_commit = remote_newest_commit.stdout.strip().split()[0]
# Record the final state
action.final_state(cloned=True, commit=newest_commit)
# Apply actions to reach new state, if we aren't in pretend mode
if not context.pretend:
try:
if not cloned:
clone_cmd = ["git", "clone"]
if depth is not None:
clone_cmd.append("--depth")
clone_cmd.append(str(depth))
clone_cmd.append(url)
clone_cmd.append(dst)
context.remote_exec(clone_cmd, checked=True)
else:
pull_cmd = ["git", "-C", dst, "pull", "--ff-only"]
if depth is not None:
pull_cmd.append("--depth")
pull_cmd.append(str(depth))
context.remote_exec(pull_cmd, checked=True)
except RemoteExecError as e:
return action.failure(e)
# Return success
return action.success()