def finalize(self, context, transaction): """ Finalizes this transaction, which will verify that all states are set corectly, and print the transaction. Parameters ---------- context : Context The associated context. transaction : Transaction The transaction that should be finalized. """ if self.result is None: raise LogicError( "A transaction cannot be completed without a result status.") if self.result.initial_state is None: raise LogicError( "A transaction cannot be completed without an initial state.") if self.result.final_state is None: raise LogicError( "A transaction cannot be completed without a final state.") if set(self.result.initial_state.keys()) != set( self.result.final_state.keys()): raise LogicError( "Both initial and final transaction state must have the same keys." ) self.result.title = transaction.title self.result.name = transaction.name print_transaction(context, self.result) if not self.result.success: raise TransactionError(self.result)
def template(context: Context, dst: str, src: str = None, content: str = None, mode=None, owner=None, group=None): """ Templates the given src file or given content and copies the output to the remote host at dst. Either content or src must be specified. Parameters ---------- context : Context The context providing the execution context and templating dictionary. src : str, optional The local source file path relative to the project directory. Will be templated. Mutually exclusive with content. content : str, optional The content for the file. Will be templated. Mutually exclusive with src. dst : str The remote destination file path. Will be templated. mode : int, optional The new file mode. Defaults the current context file creation mode. owner : str, optional The new file owner. Defaults the current context owner. group : str, optional The new file group. Defaults the current context group. Returns ------- CompletedTransaction The completed transaction """ if src is not None: src = template_str(context, src) dst = template_str(context, dst) check_valid_path(dst) if content is None and src is None: raise LogicError("Either src or content must be given.") if content is not None and src is not None: raise LogicError("Exactly one of src or content must be given.") def get_content(): if content is not None: return template_str(context, content) # Get templated content try: templ = context.host.manager.jinja2_env.get_template(src) except TemplateNotFound as e: raise LogicError("Template not found: " + str(e)) from e try: return templ.render(context.vars_dict) except UndefinedError as e: raise MessageError(f"Error while templating '{src}': " + str(e)) from e return remote_upload(context, get_content, title="template", name=dst, dst=dst, mode=mode, owner=owner, group=group)
def check_valid_relative_path(path): """ Asserts that a given path is non empty and relative. Parameters ---------- path : str The path to check. """ if not path: raise LogicError("Path must be non-empty") if path[0] == "/": raise LogicError("Path must be relative")
def __init__(self, manager): self.manager = manager if self.identifier is None: raise LogicError( "A task must override the static variable 'identifier'") if self.description is None: raise LogicError( "A task must override the static variable 'description'") # Initialize variable defaults self.var_enabled = f"tasks.{self.identifier}.enabled" self.manager.set(self.var_enabled, True) self.set_defaults()
def __init__(self, manager): super().__init__(manager) if self.tracking_repo_url is None: raise LogicError( "A tracked task must override the variable 'tracking_repo_url'" ) if self.tracking_local_dst is None: raise LogicError( "A tracked task must override the variable 'tracking_local_dst'" ) if self.tracking_paths == []: raise LogicError( "A tracked task must override the variable 'tracking_paths'") self.tracking_id = None
def check_valid_key(key: str, msg: str = "Invalid key"): """ Asserts that a given key is a valid identifier. Parameters ---------- key : str The key to check. msg : str, optional The message to raise when the check fails. """ if not key: raise LogicError("Invalid empty key") if '..' in key or not all([k.isidentifier() for k in key.split('.')]): raise LogicError(f"{msg}: '{key}' is not a valid identifier")
def set(self, key, value): """ Sets the given variable. Parameters ---------- key : str The key that should be read. value : Any, optional The value to be stored. Must be json (de-)serializable. """ check_valid_key(key) d = self.vars cs = [] keys = key.split('.') for k in keys[:-1]: if k not in d: d[k] = {} d = d[k] cs.append(k) if not isinstance(d, dict): csname = '.'.join(cs) raise LogicError(f"Cannot set variable '{key}' because existing variable '{csname}' is not a dictionary") if self.warn_on_redefinition and keys[-1] in d: print(f"[1;33mwarning:[m [1mRedefinition of variable '{key}':[m previous={d[keys[-1]]} new={value}") d[keys[-1]] = value
def failure(self, reason, set_final_state=False, **kwargs): """ Completes the transaction, marking it as failed with the given reason. If reason is a RemoteExecError, additional information will be printed. Parameters ---------- reason : str The reason for the failure. set_final_state : bool If true, the final transaction state will be set. Defaults to false. **kwargs Final state assocations Returns ------- CompletedTransaction The completed transaction. """ if isinstance(reason, RemoteExecError): e = reason reason = f"{type(e).__name__}: {str(e)}\n" reason += e.ret.stderr if set_final_state: self.final_state(**self.initial_state_dict) if self.result is not None: raise LogicError( "A transaction cannot be completed multiple times.") self.result = CompletedTransaction(self, success=False, failure_reason=reason, store=kwargs) return self.result
def __enter__(self): """ Begins a new transaction """ if self.transaction_context is not None: raise LogicError("A transaction may only be started once.") self.transaction_context = ActiveTransaction() print_transaction_early(self) return self.transaction_context
def generic_package(context: Context, atom: str, state: str, is_installed, install, uninstall): """ Installs or uninstalls (depending if state == "present" or "absent") the given package atom. Additional options to emerge can be passed via opts, and will be appended before the package atom. opts will be templated. Parameters ---------- context : Context The context providing the execution context and templating dictionary. atom : str The package name to be installed or uninstalled. Will be templated. state : str The desired state, either "present" or "absent". is_installed : Callable[[Context, str], bool] Callback used to determine if a given package is installed. install : Callable[[Context, str], None] Callback used to install a package on the remote. uninstall : Callable[[Context, str], None] Callback used to uninstall a package on the remote. Returns ------- CompletedTransaction The completed transaction """ # pylint: disable=R0801 if state not in ["present", "absent"]: raise LogicError(f"Invalid package state '{state}'") atom = template_str(context, atom) with context.transaction(title="package", name=atom) as action: # Query current state installed = is_installed(context, atom) # Record this initial state, and return early # if there is nothing to do action.initial_state(installed=installed) should_install = (state == "present") if installed == should_install: return action.unchanged() # Record the final state action.final_state(installed=should_install) # Apply actions to reach new state, if we aren't in pretend mode if not context.pretend: if should_install: install(context, atom) else: uninstall(context, atom) # Return success return action.success()
def directory(context: Context, path: str, mode=None, owner=None, group=None): """ Creates the given directory on the remote. Parameters ---------- context : Context The context providing the execution context and templating dictionary. path : str The directory path to create (will be templated). Parent directory must exist. mode : int, optional The new directory mode. Defaults the current context directory creation mode. owner : str, optional The new directory owner. Defaults the current context owner. group : str, optional The new directory group. Defaults the current context group. Returns ------- CompletedTransaction The completed transaction """ path = template_str(context, path) check_valid_path(path) with context.transaction(title="dir", name=path) as action: mode, owner, group = resolve_mode_owner_group(context, mode, owner, group, context.dir_mode) # Get current state (cur_ft, cur_mode, cur_owner, cur_group) = remote_stat(context, path) # Record this initial state if cur_ft is None: action.initial_state(exists=False, mode=None, owner=None, group=None) elif cur_ft == "directory": action.initial_state(exists=True, mode=cur_mode, owner=cur_owner, group=cur_group) if mode == cur_mode and owner == cur_owner and group == cur_group: return action.unchanged() else: raise LogicError("Cannot create directory on remote: Path already exists and is not a directory") # Record the final state action.final_state(exists=True, mode=mode, owner=owner, group=group) # Apply actions to reach new state, if we aren't in pretend mode if not context.pretend: try: # If stat failed, the directory doesn't exist and we need to create it. if cur_ft is None: context.remote_exec(["mkdir", path], checked=True) # Set permissions context.remote_exec(["chown", f"{owner}:{group}", path], checked=True) context.remote_exec(["chmod", mode, path], checked=True) except RemoteExecError as e: return action.failure(e) # Return success return action.success()
def run_task(self, registered_task_class): """ Runs the registered instance (see Manager) for the given task class. Parameters ---------- registered_task_class : class(Task) The task class that should be executed. The registered instance is found first, and then called. """ instance = self.manager.tasks.get(registered_task_class.identifier, None) if not instance: raise LogicError( f"Cannot run unregistered task {registered_task_class}") if not isinstance(instance, registered_task_class): raise LogicError( "Cannot run unrelated task with the same identifier as a registered task." ) instance.exec(self)
def final_state(self, **kwargs): """ Records the (expected) final state of the remote. Parameters ---------- **kwargs Final state assocations """ if self.result is not None: raise LogicError( "A transaction cannot be altered after it is completed") self.final_state_dict = dict(kwargs)
def get_content(): if content is not None: return template_str(context, content) # Get templated content try: templ = context.host.manager.jinja2_env.get_template(src) except TemplateNotFound as e: raise LogicError("Template not found: " + str(e)) from e try: return templ.render(context.vars_dict) except UndefinedError as e: raise MessageError(f"Error while templating '{src}': " + str(e)) from e
def get_or_throw(key): """ Retrieves a variable by the given key or raises a KeyError if no such key exists. """ check_valid_key(key) d = self.vars cs = [] for k in key.split('.'): cs.append(k) if not isinstance(d, dict): csname = '.'.join(cs) raise LogicError(f"Cannot access variable '{key}' because '{csname}' is not a dictionary") if k not in d: raise KeyError(f"Variable '{key}' does not exist") d = d[k] return d
def _assert_tracking_repo_clean(self, context): """ Asserts that the tracking repository is clean, so we will never confuse different changes in a commit. """ (_, dst, _) = context.cache["tracking"][self.tracking_id] with context.defaults(user=self.tracking_user, owner=self.tracking_user, group=self.tracking_group): if not context.pretend: # Assert that the repository is clean remote_status = context.remote_exec( ["git", "-C", dst, "status", "--porcelain"], checked=True) if remote_status.stdout.strip() != "": raise LogicError( "Refusing operation: Tracking repository is not clean!" )
def success(self, **kwargs): """ Completes the transaction with successful status. Parameters ---------- **kwargs Final state assocations Returns ------- CompletedTransaction The completed transaction. """ if self.result is not None: raise LogicError( "A transaction cannot be completed multiple times.") self.result = CompletedTransaction(self, success=True, store=kwargs) return self.result
def template_str(context: Context, content : str) -> str: """ Renders the given string template. Parameters ---------- context : Context The context providing the templating dictionary content : str The string to template Returns ------- str The templated string """ templ = Template(content) try: return templ.render(context.vars_dict) except UndefinedError as e: raise LogicError(f"Error while templating string '{content}': " + str(e)) from e
def encrypt_content(self, plaintext: bytes) -> bytes: """ Encrypts the given plaintext. Parameters ---------- plaintext : bytes The bytes to encrypt. Returns ------- bytes The ciphertext """ if self.recipient is None: raise LogicError("GpgVault encryption requires a recipient") return subprocess.run( ["gpg", "--quiet", "--encrypt", "--recipient", self.recipient], input=plaintext, capture_output=True, check=True).stdout
def user(context: Context, name: str, group: str = None, groups: list[str] = None, append_groups: bool = False, state: str = "present", system: bool = False, shell: str = None, password: str = None, home: str = None, create_home = True): # pylint: disable=R0912,R0915 """ Creates or modifies a unix user. Because we interally call ``userdel``, removing a user will also remove it's associated primary group if no other user belongs to it. Parameters ---------- context : Context The context providing the execution context and templating dictionary. name : str The name of the user to create or modify. Will be templated. group: str, optional The primary group of the user. If given, the group must already exists. Otherwise, a group will be created with the same name as the user, if a user is created by this action. Will be templated. groups: list[str], optional Supplementary groups for the user. append_groups: bool If ``True``, the user will be added to all given supplementary groups. If ``False``, the user will be added to exactly the given supplementary groups and removed from other groups. state: str If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters. system: bool If ``True`` the user will be created as a system user. This has no effect on existing users. shell: str Specifies the shell for the user. Defaults to ``/sbin/nologin`` if not given but a user needs to be created. Will be templated. password : str, optional Will update the password hash to the given vaule of the user. Use ``!`` to lock the account. Defaults to '!' if not given but a user needs to be created. Will be templated. You can generate a password hash by using the following script: >>> import crypt, getpass >>> crypt.crypt(getpass.getpass(), crypt.mksalt(crypt.METHOD_SHA512)) '$6$rwn5z9MlYvcnE222$9wOP6Y6EcnF.cZ7BUjttWeSQNdOQWI...' home: str, optional The home directory for the user. Will be left empty if not given but a user needs to be created. Will be templated. create_home: bool If ``True`` and home was given, create the home directory of the user if it doesn't exist. Returns ------- CompletedTransaction The completed transaction """ name = template_str(context, name) group = template_str(context, group) if group is not None else None home = template_str(context, home) if home is not None else None password = template_str(context, password) if password is not None else None shell = template_str(context, shell) if shell is not None else None # pylint: disable=R0801 if state not in ["present", "absent"]: raise LogicError(f"Invalid user state '{state}'") if home is not None: check_valid_path(home) with context.transaction(title="user", name=name) as action: pwinfo = context.remote_exec(["python", "-c", ( 'import sys,grp,pwd,spwd\n' 'try:\n' ' p=pwd.getpwnam(sys.argv[1])\n' 'except KeyError:\n' ' print("0:")\n' ' sys.exit(0)\n' 'g=grp.getgrgid(p.pw_gid)\n' 's=spwd.getspnam(p.pw_name)\n' 'grps=\',\'.join([sg.gr_name for sg in grp.getgrall() if p.pw_name in sg.gr_mem])\n' 'print(f"1:{g.gr_name}:{grps}:{p.pw_dir}:{p.pw_shell}:{s.sp_pwdp}")') , name], checked=True) exists = pwinfo.stdout.strip().startswith('1:') if exists: ( _ , cur_group , cur_groups , cur_home , cur_shell , cur_password ) = pwinfo.stdout.strip().split(':') cur_groups = list(sorted(set([] if cur_groups == '' else cur_groups.split(',')))) else: cur_group = None cur_groups = [] cur_home = None cur_shell = None cur_password = None if state == "absent": action.initial_state(exists=exists) if not exists: return action.unchanged() action.final_state(exists=False) if not context.pretend: try: context.remote_exec(["userdel", name], checked=True) except RemoteExecError as e: return action.failure(e) else: action.initial_state(exists=exists, group=cur_group, groups=cur_groups, home=cur_home, shell=cur_shell, pw=cur_password) fin_group = group or cur_group or name fin_groups = cur_groups if groups is None else list(sorted(set(cur_groups + groups if append_groups else groups))) fin_home = home or cur_home fin_shell = shell or cur_shell or '/sbin/nologin' fin_password = password or cur_password or '!' action.final_state(exists=True, group=fin_group, groups=fin_groups, home=fin_home, shell=fin_shell, pw=fin_password) if not context.pretend: try: if exists: # Only apply changes to the existing user if cur_group != fin_group: context.remote_exec(["usermod", "--gid", fin_group, name], checked=True) if cur_groups != fin_groups: context.remote_exec(["usermod", "--groups", ','.join(fin_groups), name], checked=True) if cur_home != fin_home: context.remote_exec(["usermod", "--home", fin_home, name], checked=True) if cur_shell != fin_shell: context.remote_exec(["usermod", "--shell", fin_shell, name], checked=True) if cur_password != fin_password: context.remote_exec(["usermod", "--password", fin_password, name], checked=True) else: # Create a new user so that is results in the given final state command = ["useradd"] if system: command.append("--system") # Primary group if group is None: command.append("--user-group") else: command.extend(["--no-user-group", "--gid", group]) # Supplementary groups if len(fin_groups) > 0: command.extend(["--groups", ','.join(fin_groups)]) command.extend(["--no-create-home", "--home-dir", fin_home or '']) command.extend(["--shell", fin_shell]) command.extend(["--password", fin_password]) command.append(name) context.remote_exec(command, checked=True) except RemoteExecError as e: return action.failure(e) # Remember result action_result = action.success() # Create home directory afterwards if necessary if state == "present" and create_home and cur_home is None and fin_home: directory(context, path=fin_home, mode=0o700, owner=name, group=fin_group) return action_result
def group(context: Context, name: str, state: str = "present", system: bool = False): """ Creates or deletes a unix group. Parameters ---------- context : Context The context providing the execution context and templating dictionary. name : str The name of the user to create or modify. Will be templated. state: str If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters. system: bool If ``True`` the user will be created as a system user. This has no effect on existing users. Returns ------- CompletedTransaction The completed transaction """ name = template_str(context, name) # pylint: disable=R0801 if state not in ["present", "absent"]: raise LogicError(f"Invalid user state '{state}'") with context.transaction(title="group", name=name) as action: grinfo = context.remote_exec(["python", "-c", ( 'import sys,grp\n' 'try:\n' ' g=grp.getgrnam(sys.argv[1])\n' 'except KeyError:\n' ' print("0")\n' ' sys.exit(0)\n' 'print("1")') , name], checked=True) exists = grinfo.stdout.strip().startswith('1') if state == "absent": action.initial_state(exists=exists) if not exists: return action.unchanged() action.final_state(exists=False) if not context.pretend: try: context.remote_exec(["groupdel", name], checked=True) except RemoteExecError as e: return action.failure(e) else: action.initial_state(exists=exists) if exists: return action.unchanged() action.final_state(exists=True) if not context.pretend: try: command = ["groupadd"] if system: command.append("--system") command.append(name) context.remote_exec(command, checked=True) except RemoteExecError as e: return action.failure(e) return action.success()
def resolve_mode_owner_group(context: Context, mode, owner, group, fallback_mode): """ Canonicalize mode, owner and group. If any of them is None, the respective variable will be replaced with the context default. Parameters ---------- context : Context The host execution context mode : int The mode to canonicalize. May be None. owner : str User id or name for the owner. User will be verified as existing on the remote. group : str Group id or name for the group. Group will be verified as existing on the remote. fallback_mode : int The fallback_mode to canonicalize in case mode = None. Usually set to either context.file_mode or context.dir_mode. Returns ------- (str, str, str) A tuple of (resolved_mode, resolved_owner, resolved_group) """ # Resolve mode to string resolved_mode = _mode_to_str(fallback_mode if mode is None else mode) # Resolve owner name/uid to name owner = context.owner if owner is None else owner remote_cmd_owner_id = context.remote_exec(["python", "-c", ( 'import sys,pwd\n' 'try:\n' ' p=pwd.getpwnam(sys.argv[1])\n' 'except KeyError:\n' ' try:\n' ' p=pwd.getpwuid(int(sys.argv[1]))\n' ' except (KeyError, ValueError):\n' ' sys.exit(1)\n' 'print(p.pw_name)') , owner]) if remote_cmd_owner_id.return_code != 0: raise LogicError(f"Could not resolve remote user '{owner}'") resolved_owner = remote_cmd_owner_id.stdout.strip() # TODO use python because id -ng is wrng # Resolve group name/gid to name group = context.group if group is None else group remote_cmd_group_id = context.remote_exec(["python", "-c", ( 'import sys,grp\n' 'try:\n' ' g=grp.getgrnam(sys.argv[1])\n' 'except KeyError:\n' ' try:\n' ' g=grp.getgrgid(int(sys.argv[1]))\n' ' except (KeyError, ValueError):\n' ' sys.exit(1)\n' 'print(g.gr_name)') , group]) if remote_cmd_group_id.return_code != 0: raise LogicError(f"Could not resolve remote group '{group}'") resolved_group = remote_cmd_group_id.stdout.strip() # Return resolved tuple return (resolved_mode, resolved_owner, resolved_group)
def checkout(context: Context, url: str, dst: str, update: bool = True, depth=None): """ Checkout (and optionally update) the given git repository to dst. Parameters ---------- context : Context The context providing the execution context and templating dictionary. url : str The url of the git repository to checkout. Will be templated. dst : str The remote destination path for the repository. Will be templated. update: bool, optional Also tries to update the repository if it is already cloned. Defaults to true. depth : str, optional Restrict repository cloning depth. Beware that updates might not work correctly because of forced updates. Returns ------- CompletedTransaction The completed transaction """ url = template_str(context, url) dst = template_str(context, dst) check_valid_path(dst) with context.transaction(title="checkout", name=dst) as action: # Add url as extra info action.extra_info(url=url) # Query current state (cur_ft, _, _, _) = remote_stat(context, dst) # Record this initial state if cur_ft is None: action.initial_state(cloned=False, commit=None) cloned = False elif cur_ft == "directory": # Assert that it is a git directory (cur_git_ft, _, _, _) = remote_stat(context, dst + "/.git") if cur_git_ft != 'directory': raise LogicError( "Cannot checkout git repository on remote: Directory already exists and is not a git repository" ) remote_commit = context.remote_exec( ["git", "-C", dst, "rev-parse", "HEAD"], error_verbosity=0, verbosity=2) if remote_commit.return_code != 0: raise LogicError( "Cannot checkout git repository on remote: Directory already exists but 'git rev-parse HEAD' failed" ) cur_commit = remote_commit.stdout.strip() action.initial_state(cloned=True, commit=cur_commit) cloned = True else: raise LogicError( "Cannot checkout git repository on remote: Path already exists but isn't a directory" ) # If the repository is already cloned but we shouldn't update, # nothing will change and we are done. if cloned and not update: return action.unchanged() # Check the newest available commit remote_newest_commit = context.remote_exec( ["git", "ls-remote", "--exit-code", url, "HEAD"], checked=True) newest_commit = remote_newest_commit.stdout.strip().split()[0] # Record the final state action.final_state(cloned=True, commit=newest_commit) # Apply actions to reach new state, if we aren't in pretend mode if not context.pretend: try: if not cloned: clone_cmd = ["git", "clone"] if depth is not None: clone_cmd.append("--depth") clone_cmd.append(str(depth)) clone_cmd.append(url) clone_cmd.append(dst) context.remote_exec(clone_cmd, checked=True) else: pull_cmd = ["git", "-C", dst, "pull", "--ff-only"] if depth is not None: pull_cmd.append("--depth") pull_cmd.append(str(depth)) context.remote_exec(pull_cmd, checked=True) except RemoteExecError as e: return action.failure(e) # Return success return action.success()