def copy(context: Context, src: str, dst: str, mode=None, owner=None, group=None): """ Copies the given src file to the remote host at dst. Parameters ---------- context : Context The context providing the execution context and templating dictionary. src : str The local source file path relative to the project directory. Will be templated. dst : str The remote destination file path. Will be templated. mode : int, optional The new file mode. Defaults the current context file creation mode. owner : str, optional The new file owner. Defaults the current context owner. group : str, optional The new file group. Defaults the current context group. Returns ------- CompletedTransaction The completed transaction """ src = template_str(context, src) dst = template_str(context, dst) check_valid_path(dst) def get_content(): # Get source content with open(os.path.join(context.host.manager.main_directory, src), 'r') as f: return f.read() return remote_upload(context, get_content, title="copy", name=dst, dst=dst, mode=mode, owner=owner, group=group)
def _resolve_variables(self, context): """ Resolve the templated variables for this tracked task """ url = template_str(context, self.tracking_repo_url) dst = template_str(context, self.tracking_local_dst) sub = template_str(context, self.tracking_subpath) tracking_id = f"{url}:{dst}/{sub}" check_valid_path(dst) check_valid_relative_path(sub) return (tracking_id, url, dst, sub)
def template(context: Context, dst: str, src: str = None, content: str = None, mode=None, owner=None, group=None): """ Templates the given src file or given content and copies the output to the remote host at dst. Either content or src must be specified. Parameters ---------- context : Context The context providing the execution context and templating dictionary. src : str, optional The local source file path relative to the project directory. Will be templated. Mutually exclusive with content. content : str, optional The content for the file. Will be templated. Mutually exclusive with src. dst : str The remote destination file path. Will be templated. mode : int, optional The new file mode. Defaults the current context file creation mode. owner : str, optional The new file owner. Defaults the current context owner. group : str, optional The new file group. Defaults the current context group. Returns ------- CompletedTransaction The completed transaction """ if src is not None: src = template_str(context, src) dst = template_str(context, dst) check_valid_path(dst) if content is None and src is None: raise LogicError("Either src or content must be given.") if content is not None and src is not None: raise LogicError("Exactly one of src or content must be given.") def get_content(): if content is not None: return template_str(context, content) # Get templated content try: templ = context.host.manager.jinja2_env.get_template(src) except TemplateNotFound as e: raise LogicError("Template not found: " + str(e)) from e try: return templ.render(context.vars_dict) except UndefinedError as e: raise MessageError(f"Error while templating '{src}': " + str(e)) from e return remote_upload(context, get_content, title="template", name=dst, dst=dst, mode=mode, owner=owner, group=group)
def package(context: Context, name: str, state="present", opts: list[str] = None): """ Installs or uninstalls the given package name (depending on state == "present" or "absent"). Additional options to apt-get can be passed via opts, and will be appended before the package name. opts will be templated. Parameters ---------- context : Context The context providing the execution context and templating dictionary. name : str The package name to be installed or uninstalled. Will be templated. state : str, optional The desired state, either "present" or "absent". Defaults to "present". opts : list[str] Additional options to pacman. Will be templated. Returns ------- CompletedTransaction The completed transaction """ opts = [] if opts is None else [template_str(context, o) for o in opts] def install(context, name): apt_cmd = ["apt-get", "install"] apt_cmd.extend(opts) apt_cmd.append(name) context.remote_exec(apt_cmd, checked=True) def uninstall(context, name): context.remote_exec(["apt-get", "remove"] + opts + [name], checked=True) generic_package(context, name, state, is_installed, install, uninstall)
def directory(context: Context, path: str, mode=None, owner=None, group=None): """ Creates the given directory on the remote. Parameters ---------- context : Context The context providing the execution context and templating dictionary. path : str The directory path to create (will be templated). Parent directory must exist. mode : int, optional The new directory mode. Defaults the current context directory creation mode. owner : str, optional The new directory owner. Defaults the current context owner. group : str, optional The new directory group. Defaults the current context group. Returns ------- CompletedTransaction The completed transaction """ path = template_str(context, path) check_valid_path(path) with context.transaction(title="dir", name=path) as action: mode, owner, group = resolve_mode_owner_group(context, mode, owner, group, context.dir_mode) # Get current state (cur_ft, cur_mode, cur_owner, cur_group) = remote_stat(context, path) # Record this initial state if cur_ft is None: action.initial_state(exists=False, mode=None, owner=None, group=None) elif cur_ft == "directory": action.initial_state(exists=True, mode=cur_mode, owner=cur_owner, group=cur_group) if mode == cur_mode and owner == cur_owner and group == cur_group: return action.unchanged() else: raise LogicError("Cannot create directory on remote: Path already exists and is not a directory") # Record the final state action.final_state(exists=True, mode=mode, owner=owner, group=group) # Apply actions to reach new state, if we aren't in pretend mode if not context.pretend: try: # If stat failed, the directory doesn't exist and we need to create it. if cur_ft is None: context.remote_exec(["mkdir", path], checked=True) # Set permissions context.remote_exec(["chown", f"{owner}:{group}", path], checked=True) context.remote_exec(["chmod", mode, path], checked=True) except RemoteExecError as e: return action.failure(e) # Return success return action.success()
def generic_package(context: Context, atom: str, state: str, is_installed, install, uninstall): """ Installs or uninstalls (depending if state == "present" or "absent") the given package atom. Additional options to emerge can be passed via opts, and will be appended before the package atom. opts will be templated. Parameters ---------- context : Context The context providing the execution context and templating dictionary. atom : str The package name to be installed or uninstalled. Will be templated. state : str The desired state, either "present" or "absent". is_installed : Callable[[Context, str], bool] Callback used to determine if a given package is installed. install : Callable[[Context, str], None] Callback used to install a package on the remote. uninstall : Callable[[Context, str], None] Callback used to uninstall a package on the remote. Returns ------- CompletedTransaction The completed transaction """ # pylint: disable=R0801 if state not in ["present", "absent"]: raise LogicError(f"Invalid package state '{state}'") atom = template_str(context, atom) with context.transaction(title="package", name=atom) as action: # Query current state installed = is_installed(context, atom) # Record this initial state, and return early # if there is nothing to do action.initial_state(installed=installed) should_install = (state == "present") if installed == should_install: return action.unchanged() # Record the final state action.final_state(installed=should_install) # Apply actions to reach new state, if we aren't in pretend mode if not context.pretend: if should_install: install(context, atom) else: uninstall(context, atom) # Return success return action.success()
def save_output(context: Context, command: list[str], dst: str, desc=None, mode=None, owner=None, group=None): """ Saves the stdout of the given command on the remote host at remote dst. Using --pretend will still run the command, but won't save the output. Changed status reflects if the file contents changed. Optionally accepts file mode, owner and group, if not given, context defaults are used. Parameters ---------- context : Context The context providing the execution context and templating dictionary. command: list[str] A list containing the command and its arguments. Each one will be templated. dst : str The remote destination file path. Will be templated. desc : str A description to be printed in the summary when executing. mode : int, optional The new file mode. Defaults the current context file creation mode. owner : str, optional The new file owner. Defaults the current context owner. group : str, optional The new file group. Defaults the current context group. Returns ------- CompletedTransaction The completed transaction """ command = [template_str(context, c) for c in command] dst = template_str(context, dst) check_valid_path(dst) def get_content(): # Get command output return context.remote_exec(command, checked=True).stdout name = f"{command}" if desc is None else desc return remote_upload(context, get_content, title="save out", name=name, dst=dst, mode=mode, owner=owner, group=group)
def get_content(): if content is not None: return template_str(context, content) # Get templated content try: templ = context.host.manager.jinja2_env.get_template(src) except TemplateNotFound as e: raise LogicError("Template not found: " + str(e)) from e try: return templ.render(context.vars_dict) except UndefinedError as e: raise MessageError(f"Error while templating '{src}': " + str(e)) from e
def run(self, context): with context.defaults(user=self.tracked_task.tracking_user, owner=self.tracked_task.tracking_user, group=self.tracked_task.tracking_group): # Get tracking specific variables (url, dst, _) = context.cache["tracking"][self.tracked_task.tracking_id] # Clone or update remote tracking repository git.checkout(context, url, dst) if not context.pretend: # Set given git repo configs for k, v in self.tracked_task.tracking_repo_configs.items( ): v = template_str(context, v) context.remote_exec( ["git", "-C", dst, "config", "--local", k, v], checked=True)
def package(context: Context, atom: str, state="present", oneshot=False, opts: list[str] = None): """ Installs or uninstalls the given package atom (depending on state == "present" or "absent"). Additional options to emerge can be passed via opts, and will be appended before the package atom. opts will be templated. Parameters ---------- context : Context The context providing the execution context and templating dictionary. atom : str The package name to be installed or uninstalled. Will be templated. state : str, optional The desired state, either "present" or "absent". Defaults to "present". oneshot : bool, optional Use portage option --oneshot. Defaults to false. opts : list[str] Additional options to portage. Will be templated. Returns ------- CompletedTransaction The completed transaction """ opts = [] if opts is None else [template_str(context, o) for o in opts] def install(context, atom): emerge_cmd = ["emerge", "--color=y", "--verbose"] if oneshot: emerge_cmd.append("--oneshot") emerge_cmd.extend(opts) emerge_cmd.append(atom) context.remote_exec(emerge_cmd, checked=True) def uninstall(context, atom): context.remote_exec(["emerge", "--color=y", "--verbose", "--depclean"] + opts + [atom], checked=True) generic_package(context, atom, state, is_installed, install, uninstall)
def user(context: Context, name: str, group: str = None, groups: list[str] = None, append_groups: bool = False, state: str = "present", system: bool = False, shell: str = None, password: str = None, home: str = None, create_home = True): # pylint: disable=R0912,R0915 """ Creates or modifies a unix user. Because we interally call ``userdel``, removing a user will also remove it's associated primary group if no other user belongs to it. Parameters ---------- context : Context The context providing the execution context and templating dictionary. name : str The name of the user to create or modify. Will be templated. group: str, optional The primary group of the user. If given, the group must already exists. Otherwise, a group will be created with the same name as the user, if a user is created by this action. Will be templated. groups: list[str], optional Supplementary groups for the user. append_groups: bool If ``True``, the user will be added to all given supplementary groups. If ``False``, the user will be added to exactly the given supplementary groups and removed from other groups. state: str If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters. system: bool If ``True`` the user will be created as a system user. This has no effect on existing users. shell: str Specifies the shell for the user. Defaults to ``/sbin/nologin`` if not given but a user needs to be created. Will be templated. password : str, optional Will update the password hash to the given vaule of the user. Use ``!`` to lock the account. Defaults to '!' if not given but a user needs to be created. Will be templated. You can generate a password hash by using the following script: >>> import crypt, getpass >>> crypt.crypt(getpass.getpass(), crypt.mksalt(crypt.METHOD_SHA512)) '$6$rwn5z9MlYvcnE222$9wOP6Y6EcnF.cZ7BUjttWeSQNdOQWI...' home: str, optional The home directory for the user. Will be left empty if not given but a user needs to be created. Will be templated. create_home: bool If ``True`` and home was given, create the home directory of the user if it doesn't exist. Returns ------- CompletedTransaction The completed transaction """ name = template_str(context, name) group = template_str(context, group) if group is not None else None home = template_str(context, home) if home is not None else None password = template_str(context, password) if password is not None else None shell = template_str(context, shell) if shell is not None else None # pylint: disable=R0801 if state not in ["present", "absent"]: raise LogicError(f"Invalid user state '{state}'") if home is not None: check_valid_path(home) with context.transaction(title="user", name=name) as action: pwinfo = context.remote_exec(["python", "-c", ( 'import sys,grp,pwd,spwd\n' 'try:\n' ' p=pwd.getpwnam(sys.argv[1])\n' 'except KeyError:\n' ' print("0:")\n' ' sys.exit(0)\n' 'g=grp.getgrgid(p.pw_gid)\n' 's=spwd.getspnam(p.pw_name)\n' 'grps=\',\'.join([sg.gr_name for sg in grp.getgrall() if p.pw_name in sg.gr_mem])\n' 'print(f"1:{g.gr_name}:{grps}:{p.pw_dir}:{p.pw_shell}:{s.sp_pwdp}")') , name], checked=True) exists = pwinfo.stdout.strip().startswith('1:') if exists: ( _ , cur_group , cur_groups , cur_home , cur_shell , cur_password ) = pwinfo.stdout.strip().split(':') cur_groups = list(sorted(set([] if cur_groups == '' else cur_groups.split(',')))) else: cur_group = None cur_groups = [] cur_home = None cur_shell = None cur_password = None if state == "absent": action.initial_state(exists=exists) if not exists: return action.unchanged() action.final_state(exists=False) if not context.pretend: try: context.remote_exec(["userdel", name], checked=True) except RemoteExecError as e: return action.failure(e) else: action.initial_state(exists=exists, group=cur_group, groups=cur_groups, home=cur_home, shell=cur_shell, pw=cur_password) fin_group = group or cur_group or name fin_groups = cur_groups if groups is None else list(sorted(set(cur_groups + groups if append_groups else groups))) fin_home = home or cur_home fin_shell = shell or cur_shell or '/sbin/nologin' fin_password = password or cur_password or '!' action.final_state(exists=True, group=fin_group, groups=fin_groups, home=fin_home, shell=fin_shell, pw=fin_password) if not context.pretend: try: if exists: # Only apply changes to the existing user if cur_group != fin_group: context.remote_exec(["usermod", "--gid", fin_group, name], checked=True) if cur_groups != fin_groups: context.remote_exec(["usermod", "--groups", ','.join(fin_groups), name], checked=True) if cur_home != fin_home: context.remote_exec(["usermod", "--home", fin_home, name], checked=True) if cur_shell != fin_shell: context.remote_exec(["usermod", "--shell", fin_shell, name], checked=True) if cur_password != fin_password: context.remote_exec(["usermod", "--password", fin_password, name], checked=True) else: # Create a new user so that is results in the given final state command = ["useradd"] if system: command.append("--system") # Primary group if group is None: command.append("--user-group") else: command.extend(["--no-user-group", "--gid", group]) # Supplementary groups if len(fin_groups) > 0: command.extend(["--groups", ','.join(fin_groups)]) command.extend(["--no-create-home", "--home-dir", fin_home or '']) command.extend(["--shell", fin_shell]) command.extend(["--password", fin_password]) command.append(name) context.remote_exec(command, checked=True) except RemoteExecError as e: return action.failure(e) # Remember result action_result = action.success() # Create home directory afterwards if necessary if state == "present" and create_home and cur_home is None and fin_home: directory(context, path=fin_home, mode=0o700, owner=name, group=fin_group) return action_result
def group(context: Context, name: str, state: str = "present", system: bool = False): """ Creates or deletes a unix group. Parameters ---------- context : Context The context providing the execution context and templating dictionary. name : str The name of the user to create or modify. Will be templated. state: str If "present" the user will be added / modified, if "absent" the user will be deleted ignoring all other parameters. system: bool If ``True`` the user will be created as a system user. This has no effect on existing users. Returns ------- CompletedTransaction The completed transaction """ name = template_str(context, name) # pylint: disable=R0801 if state not in ["present", "absent"]: raise LogicError(f"Invalid user state '{state}'") with context.transaction(title="group", name=name) as action: grinfo = context.remote_exec(["python", "-c", ( 'import sys,grp\n' 'try:\n' ' g=grp.getgrnam(sys.argv[1])\n' 'except KeyError:\n' ' print("0")\n' ' sys.exit(0)\n' 'print("1")') , name], checked=True) exists = grinfo.stdout.strip().startswith('1') if state == "absent": action.initial_state(exists=exists) if not exists: return action.unchanged() action.final_state(exists=False) if not context.pretend: try: context.remote_exec(["groupdel", name], checked=True) except RemoteExecError as e: return action.failure(e) else: action.initial_state(exists=exists) if exists: return action.unchanged() action.final_state(exists=True) if not context.pretend: try: command = ["groupadd"] if system: command.append("--system") command.append(name) context.remote_exec(command, checked=True) except RemoteExecError as e: return action.failure(e) return action.success()
def checkout(context: Context, url: str, dst: str, update: bool = True, depth=None): """ Checkout (and optionally update) the given git repository to dst. Parameters ---------- context : Context The context providing the execution context and templating dictionary. url : str The url of the git repository to checkout. Will be templated. dst : str The remote destination path for the repository. Will be templated. update: bool, optional Also tries to update the repository if it is already cloned. Defaults to true. depth : str, optional Restrict repository cloning depth. Beware that updates might not work correctly because of forced updates. Returns ------- CompletedTransaction The completed transaction """ url = template_str(context, url) dst = template_str(context, dst) check_valid_path(dst) with context.transaction(title="checkout", name=dst) as action: # Add url as extra info action.extra_info(url=url) # Query current state (cur_ft, _, _, _) = remote_stat(context, dst) # Record this initial state if cur_ft is None: action.initial_state(cloned=False, commit=None) cloned = False elif cur_ft == "directory": # Assert that it is a git directory (cur_git_ft, _, _, _) = remote_stat(context, dst + "/.git") if cur_git_ft != 'directory': raise LogicError( "Cannot checkout git repository on remote: Directory already exists and is not a git repository" ) remote_commit = context.remote_exec( ["git", "-C", dst, "rev-parse", "HEAD"], error_verbosity=0, verbosity=2) if remote_commit.return_code != 0: raise LogicError( "Cannot checkout git repository on remote: Directory already exists but 'git rev-parse HEAD' failed" ) cur_commit = remote_commit.stdout.strip() action.initial_state(cloned=True, commit=cur_commit) cloned = True else: raise LogicError( "Cannot checkout git repository on remote: Path already exists but isn't a directory" ) # If the repository is already cloned but we shouldn't update, # nothing will change and we are done. if cloned and not update: return action.unchanged() # Check the newest available commit remote_newest_commit = context.remote_exec( ["git", "ls-remote", "--exit-code", url, "HEAD"], checked=True) newest_commit = remote_newest_commit.stdout.strip().split()[0] # Record the final state action.final_state(cloned=True, commit=newest_commit) # Apply actions to reach new state, if we aren't in pretend mode if not context.pretend: try: if not cloned: clone_cmd = ["git", "clone"] if depth is not None: clone_cmd.append("--depth") clone_cmd.append(str(depth)) clone_cmd.append(url) clone_cmd.append(dst) context.remote_exec(clone_cmd, checked=True) else: pull_cmd = ["git", "-C", dst, "pull", "--ff-only"] if depth is not None: pull_cmd.append("--depth") pull_cmd.append(str(depth)) context.remote_exec(pull_cmd, checked=True) except RemoteExecError as e: return action.failure(e) # Return success return action.success()
def _track(self, context): """ Uses rsync to copy the tracking paths into the repository, and creates and pushes a commit if there are any changes. """ with context.defaults(user=self.tracking_user, owner=self.tracking_user, group=self.tracking_group): (_, dst, sub) = context.cache["tracking"][self.tracking_id] # Check source paths srcs = [] for src in self.tracking_paths: src = template_str(context, src) check_valid_path(src) srcs.append(src) # Begin transaction with context.transaction(title="track", name=f"{srcs}") as action: action.initial_state(added=None, modified=None, deleted=None) if not context.pretend: mode, owner, group = resolve_mode_owner_group( context, None, None, None, context.dir_mode) rsync_dst = f"{dst}/{sub}/" base_parts = PurePosixPath(dst).parts parts = PurePosixPath(rsync_dst).parts # Create tracking destination subdirectories if they don't exist cur = dst for p in parts[len(base_parts):]: cur = os.path.join(cur, p) context.remote_exec(["mkdir", "-p", "--", cur], checked=True) context.remote_exec(["chown", f"{owner}:{group}", cur], checked=True) context.remote_exec(["chmod", mode, cur], checked=True) # Use rsync to backup all paths into the repository for src in srcs: context.remote_exec([ "rsync", "--quiet", "--recursive", "--one-file-system", "--links", "--times", "--relative", str(PurePosixPath(src)), rsync_dst ], checked=True, user="******", umask=0o022) # Make files accessible to the tracking user context.remote_exec([ "chown", "--recursive", "--preserve-root", f"{self.tracking_user}:{self.tracking_group}", dst ], checked=True, user="******") # Add all changes context.remote_exec(["git", "-C", dst, "add", "--all"], checked=True) # Query changes for message remote_status = context.remote_exec( ["git", "-C", dst, "status", "--porcelain"], checked=True) if remote_status.stdout.strip() != "": # We have changes added = 0 modified = 0 deleted = 0 for line in remote_status.stdout.splitlines(): if line.startswith("A"): added += 1 elif line.startswith("M"): modified += 1 elif line.startswith("D"): deleted += 1 action.final_state(added=added, modified=modified, deleted=deleted) # Create commit commit_opts = [ template_str(context, o) for o in self.tracking_git_commit_opts ] context.remote_exec([ "git", "-C", dst, "commit" ] + commit_opts + [ "--message", f"task {self.identifier}: {added=}, {modified=}, {deleted=}" ], checked=True) # Push commit context.remote_exec( ["git", "-C", dst, "push", "origin", "master"], checked=True) action.success() else: # Repo is still clean action.unchanged() else: action.final_state(added="? (pretend)", modified="? (pretend)", deleted="? (pretend)") action.success()