def keyscan(hostname, force=False, port=22): """ Check/add hosts to the ``~/.ssh/known_hosts`` file. + hostname: hostname that should have a key in ``known_hosts`` + force: if the key already exists, remove and rescan **Example:** .. code:: python ssh.keyscan( name="Set add server two to known_hosts on one", hostname="two.example.com", ) """ homedir = host.get_fact(Home) yield from files.directory( "{0}/.ssh".format(homedir), mode=700, ) hostname_present = host.get_fact( FindInFile, path="{0}/.ssh/known_hosts".format(homedir), pattern=hostname, ) did_keyscan = False keyscan_command = "ssh-keyscan -p {0} {1} >> {2}/.ssh/known_hosts".format( port, hostname, homedir, ) if not hostname_present: yield keyscan_command did_keyscan = True elif force: yield "ssh-keygen -R {0}".format(hostname) yield keyscan_command did_keyscan = True else: host.noop("host key for {0} already exists".format(hostname)) if did_keyscan: host.create_fact( FindInFile, kwargs={ "path": "{0}/.ssh/known_hosts".format(homedir), "pattern": hostname }, data=["{0} unknown unknown".format(hostname)], )
def hostname(hostname, hostname_file=None): """ Set the system hostname using ``hostnamectl`` or ``hostname`` on older systems. + hostname: the hostname that should be set + hostname_file: the file that permanently sets the hostname Hostname file: The hostname file only matters no systems that do not have ``hostnamectl``, which is part of ``systemd``. By default pyinfra will auto detect this by targeting ``/etc/hostname`` on Linux and ``/etc/myname`` on OpenBSD. To completely disable writing the hostname file, set ``hostname_file=False``. **Example:** .. code:: python server.hostname( name="Set the hostname", hostname="server1.example.com", ) """ current_hostname = host.get_fact(Hostname) if host.get_fact(Which, command="hostnamectl"): if current_hostname != hostname: yield "hostnamectl set-hostname {0}".format(hostname) host.create_fact(Hostname, data=hostname) else: host.noop("hostname is set") return if hostname_file is None: os = host.get_fact(Os) if os == "Linux": hostname_file = "/etc/hostname" elif os == "OpenBSD": hostname_file = "/etc/myname" if current_hostname != hostname: yield "hostname {0}".format(hostname) host.create_fact(Hostname, data=hostname) else: host.noop("hostname is set") if hostname_file: # Create a whole new hostname file file = StringIO("{0}\n".format(hostname)) # And ensure it exists yield from files.put(file, hostname_file)
def bare_repo( path, user=None, group=None, present=True, ): """ Create bare git repositories. + path: path to the folder + present: whether the bare repository should exist + user: chown files to this user after + group: chown files to this group after **Example:** .. code:: python git.bare_repo( name="Create bare repo", path="/home/git/test.git", ) """ yield from files.directory(path, present=present) if present: head_filename = unix_path_join(path, "HEAD") head_file = host.get_fact(File, path=head_filename) if not head_file: yield "git init --bare {0}".format(path) if user or group: yield chown(path, user, group, recursive=True) else: if (user and head_file["user"] != user) or ( group and head_file["group"] != group): yield chown(path, user, group, recursive=True) host.create_fact( File, kwargs={"path": head_filename}, data={ "user": user, "group": group, "mode": None }, )
def update(cache_time=None): """ Updates apt repositories. + cache_time: cache updates for this many seconds **Example:** .. code:: python apt.update( name="Update apt repositories", cache_time=3600, ) """ # If cache_time check when apt was last updated, prevent updates if within time if cache_time: # Ubuntu provides this handy file cache_info = host.get_fact(File, path=APT_UPDATE_FILENAME) # Time on files is not tz-aware, and will be the same tz as the server's time, # so we can safely remove the tzinfo from the Date fact before comparison. host_cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta( seconds=cache_time) if cache_info and cache_info[ "mtime"] and cache_info["mtime"] > host_cache_time: host.noop("apt is already up to date") return yield "apt-get update" # Some apt systems (Debian) have the /var/lib/apt/periodic directory, but # don't bother touching anything in there - so pyinfra does it, enabling # cache_time to work. if cache_time: yield "touch {0}".format(APT_UPDATE_FILENAME) if cache_info is None: host.create_fact( File, kwargs={"path": APT_UPDATE_FILENAME}, data={"mtime": datetime.utcnow()}, ) else: cache_info["mtime"] = datetime.utcnow()
def put( src, dest, user=None, group=None, mode=None, add_deploy_dir=True, create_remote_dir=True, force=False, assume_exists=False, ): """ Upload a local file, or file-like object, to the remote system. + src: filename or IO-like object to upload + dest: remote filename to upload to + user: user to own the files + group: group to own the files + mode: permissions of the files, use ``True`` to copy the local file + add_deploy_dir: src is relative to the deploy directory + create_remote_dir: create the remote directory if it doesn't exist + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists ``dest``: If this is a directory that already exists on the remote side, the local file will be uploaded to that directory with the same filename. ``mode``: When set to ``True`` the permissions of the local file are applied to the remote file after the upload is complete. ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Note: This operation is not suitable for large files as it may involve copying the file before uploading it. **Examples:** .. code:: python files.put( name="Update the message of the day file", src="files/motd", dest="/etc/motd", mode="644", ) files.put( name="Upload a StringIO object", src=StringIO("file contents"), dest="/etc/motd", ) """ # Upload IO objects as-is if hasattr(src, "read"): local_file = src local_sum = get_file_sha1(src) # Assume string filename else: # Add deploy directory? if add_deploy_dir and state.cwd: src = os.path.join(state.cwd, src) local_file = src if os.path.isfile(local_file): local_sum = get_file_sha1(local_file) elif assume_exists: local_sum = None else: raise IOError("No such file: {0}".format(local_file)) if mode is True: if os.path.isfile(local_file): mode = get_path_permissions_mode(local_file) else: logger.warning(( "No local file exists to get permissions from with `mode=True` ({0})" ).format(get_call_location(), ), ) else: mode = ensure_mode_int(mode) remote_file = host.get_fact(File, path=dest) if not remote_file and host.get_fact(Directory, path=dest): dest = unix_path_join(dest, os.path.basename(src)) remote_file = host.get_fact(File, path=dest) if create_remote_dir: yield from _create_remote_dir(state, host, dest, user, group) # No remote file, always upload and user/group/mode if supplied if not remote_file or force: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) # File exists, check sum and check user/group/mode if supplied else: remote_sum = host.get_fact(Sha1File, path=dest) # Check sha1sum, upload if needed if local_sum != remote_sum: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) else: changed = False # Check mode if mode and remote_file["mode"] != mode: yield file_utils.chmod(dest, mode) changed = True # Check user/group if (user and remote_file["user"] != user) or ( group and remote_file["group"] != group): yield file_utils.chown(dest, user, group) changed = True if not changed: host.noop("file {0} is already uploaded".format(dest)) # Now we've uploaded the file and ensured user/group/mode, update the relevant fact data host.create_fact(Sha1File, kwargs={"path": dest}, data=local_sum) host.create_fact( File, kwargs={"path": dest}, data={ "user": user, "group": group, "mode": mode }, )
def download( src, dest, user=None, group=None, mode=None, cache_time=None, force=False, sha256sum=None, sha1sum=None, md5sum=None, headers=None, insecure=False, ): """ Download files from remote locations using ``curl`` or ``wget``. + src: source URL of the file + dest: where to save the file + user: user to own the files + group: group to own the files + mode: permissions of the files + cache_time: if the file exists already, re-download after this time (in seconds) + force: always download the file, even if it already exists + sha256sum: sha256 hash to checksum the downloaded file against + sha1sum: sha1 hash to checksum the downloaded file against + md5sum: md5 hash to checksum the downloaded file against + headers: optional dictionary of headers to set for the HTTP request + insecure: disable SSL verification for the HTTP request **Example:** .. code:: python files.download( name="Download the Docker repo file", src="https://download.docker.com/linux/centos/docker-ce.repo", dest="/etc/yum.repos.d/docker-ce.repo", ) """ info = host.get_fact(File, path=dest) host_datetime = host.get_fact(Date).replace(tzinfo=None) # Destination is a directory? if info is False: raise OperationError( "Destination {0} already exists and is not a file".format(dest), ) # Do we download the file? Force by default download = force # Doesn't exist, lets download it if info is None: download = True # Destination file exists & cache_time: check when the file was last modified, # download if old else: if cache_time: # Time on files is not tz-aware, and will be the same tz as the server's time, # so we can safely remove the tzinfo from the Date fact before comparison. cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta( seconds=cache_time) if info["mtime"] and info["mtime"] < cache_time: download = True if sha1sum: if sha1sum != host.get_fact(Sha1File, path=dest): download = True if sha256sum: if sha256sum != host.get_fact(Sha256File, path=dest): download = True if md5sum: if md5sum != host.get_fact(Md5File, path=dest): download = True # If we download, always do user/group/mode as SSH user may be different if download: temp_file = state.get_temp_filename(dest) curl_args = ["-sSLf"] wget_args = ["-q"] if insecure: curl_args.append("--insecure") wget_args.append("--no-check-certificate") if headers: for key, value in headers.items(): header_arg = StringCommand("--header", QuoteString(f"{key}: {value}")) curl_args.append(header_arg) wget_args.append(header_arg) curl_command = make_formatted_string_command( "curl {0} {1} -o {2}", StringCommand(*curl_args), QuoteString(src), QuoteString(temp_file), ) wget_command = make_formatted_string_command( "wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )", StringCommand(*wget_args), QuoteString(src), QuoteString(temp_file), ) if host.get_fact(Which, command="curl"): yield curl_command elif host.get_fact(Which, command="wget"): yield wget_command else: yield "( {0} ) || ( {1} )".format(curl_command, wget_command) yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest)) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) if sha1sum: yield make_formatted_string_command( ("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) " "|| ( echo {2} && exit 1 )"), QuoteString(dest), sha1sum, QuoteString("SHA1 did not match!"), ) if sha256sum: yield make_formatted_string_command( ("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) " "| grep {1}) || ( echo {2} && exit 1 )"), QuoteString(dest), sha256sum, QuoteString("SHA256 did not match!"), ) if md5sum: yield make_formatted_string_command( ("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) " "|| ( echo {2} && exit 1 )"), QuoteString(dest), md5sum, QuoteString("MD5 did not match!"), ) host.create_fact( File, kwargs={"path": dest}, data={ "mode": mode, "group": group, "user": user, "mtime": host_datetime }, ) # Remove any checksum facts as we don't know the correct values for value, fact_cls in ( (sha1sum, Sha1File), (sha256sum, Sha256File), (md5sum, Md5File), ): fact_kwargs = {"path": dest} if value: host.create_fact(fact_cls, kwargs=fact_kwargs, data=value) else: host.delete_fact(fact_cls, kwargs=fact_kwargs) else: host.noop("file {0} has already been downloaded".format(dest))
def replace( path, text=None, replace=None, flags=None, backup=False, interpolate_variables=False, match=None, # deprecated ): """ Replace contents of a file using ``sed``. + path: target remote file to edit + text: text/regex to match against + replace: text to replace with + flags: list of flags to pass to sed + backup: whether to backup the file (see below) + interpolate_variables: whether to interpolate variables in ``replace`` Backup: If set to ``True``, any editing of the file will place an old copy with the ISO date (taken from the machine running ``pyinfra``) appended as the extension. If you pass a string value this will be used as the extension of the backed up file. **Example:** .. code:: python files.replace( name="Change part of a line in a file", path="/etc/motd", text="verboten", replace="forbidden", ) """ if text is None and match: text = match logger.warning( ("The `match` argument has been replaced by " "`text` in the `files.replace` operation ({0})").format( get_call_location()), ) if text is None: raise TypeError( "Missing argument `text` required in `files.replace` operation") if replace is None: raise TypeError( "Missing argument `replace` required in `files.replace` operation") existing_lines = host.get_fact( FindInFile, path=path, pattern=text, interpolate_variables=interpolate_variables, ) # Only do the replacement if the file does not exist (it may be created earlier) # or we have matching lines. if existing_lines is None or existing_lines: yield sed_replace( path, text, replace, flags=flags, backup=backup, interpolate_variables=interpolate_variables, ) host.create_fact( FindInFile, kwargs={ "path": path, "pattern": text, "interpolate_variables": interpolate_variables, }, data=[], ) else: host.noop('string "{0}" does not exist in {1}'.format(text, path))
def line( path, line, present=True, replace=None, flags=None, backup=False, interpolate_variables=False, escape_regex_characters=False, assume_present=False, ): """ Ensure lines in files using grep to locate and sed to replace. + path: target remote file to edit + line: string or regex matching the target line + present: whether the line should be in the file + replace: text to replace entire matching lines when ``present=True`` + flags: list of flags to pass to sed when replacing/deleting + backup: whether to backup the file (see below) + interpolate_variables: whether to interpolate variables in ``replace`` + assume_present: whether to assume a matching line already exists in the file + escape_regex_characters: whether to escape regex characters from the matching line Regex line matching: Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To change bits of lines, see ``files.replace``. Regex line escaping: If matching special characters (eg a crontab line containing *), remember to escape it first using Python's ``re.escape``. Backup: If set to ``True``, any editing of the file will place an old copy with the ISO date (taken from the machine running ``pyinfra``) appended as the extension. If you pass a string value this will be used as the extension of the backed up file. Append: If ``line`` is not in the file but we want it (``present`` set to ``True``), then it will be append to the end of the file. **Examples:** .. code:: python # prepare to do some maintenance maintenance_line = "SYSTEM IS DOWN FOR MAINTENANCE" files.line( name="Add the down-for-maintence line in /etc/motd", path="/etc/motd", line=maintenance_line, ) # Then, after the mantenance is done, remove the maintenance line files.line( name="Remove the down-for-maintenance line in /etc/motd", path="/etc/motd", line=maintenance_line, replace="", present=False, ) # example where there is '*' in the line files.line( name="Ensure /netboot/nfs is in /etc/exports", path="/etc/exports", line=r"/netboot/nfs .*", replace="/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash," "insecure,no_subtree_check)", ) files.line( name="Ensure myweb can run /usr/bin/python3 without password", path="/etc/sudoers", line=r"myweb .*", replace="myweb ALL=(ALL) NOPASSWD: /usr/bin/python3", ) # example when there are double quotes (") line = 'QUOTAUSER=""' files.line( name="Example with double quotes (")", path="/etc/adduser.conf", line="^{}$".format(line), replace=line, ) """ match_line = line if escape_regex_characters: match_line = re.sub(r"([\.\\\+\*\?\[\^\]\$\(\)\{\}\-])", r"\\\1", match_line) # Ensure we're matching a whole line, note: match may be a partial line so we # put any matches on either side. if not match_line.startswith("^"): match_line = "^.*{0}".format(match_line) if not match_line.endswith("$"): match_line = "{0}.*$".format(match_line) # Is there a matching line in this file? if assume_present: present_lines = [line] else: present_lines = host.get_fact( FindInFile, path=path, pattern=match_line, interpolate_variables=interpolate_variables, ) # If replace present, use that over the matching line if replace: line = replace # We must provide some kind of replace to sed_replace_command below else: replace = "" # Save commands for re-use in dynamic script when file not present at fact stage echo_command = make_formatted_string_command( "echo {0} >> {1}", '"{0}"'.format(line) if interpolate_variables else QuoteString(line), QuoteString(path), ) if backup: backup_filename = "{0}.{1}".format(path, get_timestamp()) echo_command = StringCommand( make_formatted_string_command( "cp {0} {1} && ", QuoteString(path), QuoteString(backup_filename), ), echo_command, ) sed_replace_command = sed_replace( path, match_line, replace, flags=flags, backup=backup, interpolate_variables=interpolate_variables, ) # No line and we want it, append it if not present_lines and present: # If the file does not exist - it *might* be created, so we handle it # dynamically with a little script. if present_lines is None: yield make_formatted_string_command( """ if [ -f '{target}' ]; then ( grep {match_line} '{target}' && \ {sed_replace_command}) 2> /dev/null || \ {echo_command} ; else {echo_command} ; fi """, target=QuoteString(path), match_line=QuoteString(match_line), echo_command=echo_command, sed_replace_command=sed_replace_command, ) # File exists but has no matching lines - append it. else: # If we're doing replacement, only append if the *replacement* line # does not exist (as we are appending the replacement). if replace: # Ensure replace explicitly matches a whole line replace_line = replace if not replace_line.startswith("^"): replace_line = f"^{replace_line}" if not replace_line.endswith("$"): replace_line = f"{replace_line}$" present_lines = host.get_fact( FindInFile, path=path, pattern=replace_line, interpolate_variables=interpolate_variables, ) if not present_lines: yield echo_command else: host.noop('line "{0}" exists in {1}'.format( replace or line, path)) host.create_fact( FindInFile, kwargs={ "path": path, "pattern": match_line, "interpolate_variables": interpolate_variables, }, data=[replace or line], ) # Line(s) exists and we want to remove them, replace with nothing elif present_lines and not present: yield sed_replace( path, match_line, "", flags=flags, backup=backup, interpolate_variables=interpolate_variables, ) host.delete_fact( FindInFile, kwargs={ "path": path, "pattern": match_line, "interpolate_variables": interpolate_variables, }, ) # Line(s) exists and we have want to ensure they're correct elif present_lines and present: # If any of lines are different, sed replace them if replace and any(line != replace for line in present_lines): yield sed_replace_command del present_lines[:] # TODO: use .clear() when py3+ present_lines.append(replace) else: host.noop('line "{0}" exists in {1}'.format(replace or line, path))
def directory( path, present=True, assume_present=False, user=None, group=None, mode=None, recursive=False, force=False, force_backup=True, force_backup_dir=None, _no_check_owner_mode=False, _no_fail_on_link=False, ): """ Add/remove/update directories. + path: path of the remote folder + present: whether the folder should exist + assume_present: whether to assume the directory exists + user: user to own the folder + group: group to own the folder + mode: permissions of the folder + recursive: recursively apply user/group/mode + force: if the target exists and is not a file, move or remove it and continue + force_backup: set to ``False`` to remove any existing non-file when ``force=True`` + force_backup_dir: directory to move any backup to when ``force=True`` **Examples:** .. code:: python files.directory( name="Ensure the /tmp/dir_that_we_want_removed is removed", path="/tmp/dir_that_we_want_removed", present=False, ) files.directory( name="Ensure /web exists", path="/web", user="******", group="myweb", ) # Multiple directories for dir in ["/netboot/tftp", "/netboot/nfs"]: files.directory( name="Ensure the directory `{}` exists".format(dir), path=dir, ) """ path = _validate_path(path) mode = ensure_mode_int(mode) info = host.get_fact(Directory, path=path) # Not a directory?! if info is False: if _no_fail_on_link and host.get_fact(Link, path=path): host.noop("directory {0} already exists (as a link)".format(path)) return yield from _raise_or_remove_invalid_path( "directory", path, force, force_backup, force_backup_dir, ) info = None # Doesn't exist & we want it if not assume_present and info is None and present: yield StringCommand("mkdir", "-p", QuoteString(path)) if mode: yield file_utils.chmod(path, mode, recursive=recursive) if user or group: yield file_utils.chown(path, user, group, recursive=recursive) host.create_fact( Directory, kwargs={"path": path}, data={ "mode": mode, "group": group, "user": user }, ) # It exists and we don't want it elif (assume_present or info) and not present: yield StringCommand("rm", "-rf", QuoteString(path)) host.delete_fact(Directory, kwargs={"path": path}) # It exists & we want to ensure its state elif (assume_present or info) and present: if assume_present and not info: info = {"mode": None, "group": None, "user": None} host.create_fact(Directory, kwargs={"path": path}, data=info) if _no_check_owner_mode: return changed = False if mode and (not info or info["mode"] != mode): yield file_utils.chmod(path, mode, recursive=recursive) info["mode"] = mode changed = True if ((not info and (user or group)) or (user and info["user"] != user) or (group and info["group"] != group)): yield file_utils.chown(path, user, group, recursive=recursive) changed = True if user: info["user"] = user if group: info["group"] = group if not changed: host.noop("directory {0} already exists".format(path))
def file( path, present=True, assume_present=False, user=None, group=None, mode=None, touch=False, create_remote_dir=True, force=False, force_backup=True, force_backup_dir=None, ): """ Add/remove/update files. + path: name/path of the remote file + present: whether the file should exist + assume_present: whether to assume the file exists + user: user to own the files + group: group to own the files + mode: permissions of the files as an integer, eg: 755 + touch: whether to touch the file + create_remote_dir: create the remote directory if it doesn't exist + force: if the target exists and is not a file, move or remove it and continue + force_backup: set to ``False`` to remove any existing non-file when ``force=True`` + force_backup_dir: directory to move any backup to when ``force=True`` ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. **Example:** .. code:: python # Note: The directory /tmp/secret will get created with the default umask. files.file( name="Create /tmp/secret/file", path="/tmp/secret/file", mode="600", user="******", group="root", touch=True, create_remote_dir=True, ) """ path = _validate_path(path) mode = ensure_mode_int(mode) info = host.get_fact(File, path=path) # Not a file?! if info is False: yield from _raise_or_remove_invalid_path( "file", path, force, force_backup, force_backup_dir, ) info = None # Doesn't exist & we want it if not assume_present and info is None and present: if create_remote_dir: yield from _create_remote_dir(state, host, path, user, group) yield StringCommand("touch", QuoteString(path)) if mode: yield file_utils.chmod(path, mode) if user or group: yield file_utils.chown(path, user, group) host.create_fact( File, kwargs={"path": path}, data={ "mode": mode, "group": group, "user": user }, ) # It exists and we don't want it elif (assume_present or info) and not present: yield StringCommand("rm", "-f", QuoteString(path)) host.delete_fact(File, kwargs={"path": path}) # It exists & we want to ensure its state elif (assume_present or info) and present: if assume_present and not info: info = {"mode": None, "group": None, "user": None} host.create_fact(File, kwargs={"path": path}, data=info) changed = False if touch: changed = True yield StringCommand("touch", QuoteString(path)) # Check mode if mode and (not info or info["mode"] != mode): yield file_utils.chmod(path, mode) info["mode"] = mode changed = True # Check user/group if ((not info and (user or group)) or (user and info["user"] != user) or (group and info["group"] != group)): yield file_utils.chown(path, user, group) changed = True if user: info["user"] = user if group: info["group"] = group if not changed: host.noop("file {0} already exists".format(path))
def link( path, target=None, present=True, assume_present=False, user=None, group=None, symbolic=True, create_remote_dir=True, force=False, force_backup=True, force_backup_dir=None, ): """ Add/remove/update links. + path: the name of the link + target: the file/directory the link points to + present: whether the link should exist + assume_present: whether to assume the link exists + user: user to own the link + group: group to own the link + symbolic: whether to make a symbolic link (vs hard link) + create_remote_dir: create the remote directory if it doesn't exist + force: if the target exists and is not a file, move or remove it and continue + force_backup: set to ``False`` to remove any existing non-file when ``force=True`` + force_backup_dir: directory to move any backup to when ``force=True`` ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Source changes: If the link exists and points to a different target, pyinfra will remove it and recreate a new one pointing to then new target. **Examples:** .. code:: python files.link( name="Create link /etc/issue2 that points to /etc/issue", path="/etc/issue2", target="/etc/issue", ) # Complex example demonstrating the assume_present option from pyinfra.operations import apt, files install_nginx = apt.packages( name="Install nginx", packages=["nginx"], ) files.link( name="Remove default nginx site", path="/etc/nginx/sites-enabled/default", present=False, assume_present=install_nginx.changed, ) """ path = _validate_path(path) if present and not target: raise OperationError("If present is True target must be provided") info = host.get_fact(Link, path=path) # Not a link? if info is False: yield from _raise_or_remove_invalid_path( "link", path, force, force_backup, force_backup_dir, ) info = None add_args = ["ln"] if symbolic: add_args.append("-s") add_cmd = StringCommand(" ".join(add_args), QuoteString(target), QuoteString(path)) remove_cmd = StringCommand("rm", "-f", QuoteString(path)) # No link and we want it if not assume_present and info is None and present: if create_remote_dir: yield from _create_remote_dir(state, host, path, user, group) yield add_cmd if user or group: yield file_utils.chown(path, user, group, dereference=False) host.create_fact( Link, kwargs={"path": path}, data={ "link_target": target, "group": group, "user": user }, ) # It exists and we don't want it elif (assume_present or info) and not present: yield remove_cmd host.delete_fact(Link, kwargs={"path": path}) # Exists and want to ensure it's state elif (assume_present or info) and present: if assume_present and not info: info = {"link_target": None, "group": None, "user": None} host.create_fact(Link, kwargs={"path": path}, data=info) # If we have an absolute path - prepend to any non-absolute values from the fact # and/or the source. if os.path.isabs(path): link_dirname = os.path.dirname(path) if not os.path.isabs(target): target = os.path.normpath(unix_path_join(link_dirname, target)) if info and not os.path.isabs(info["link_target"]): info["link_target"] = os.path.normpath( unix_path_join(link_dirname, info["link_target"]), ) changed = False # If the target is wrong, remove & recreate the link if not info or info["link_target"] != target: changed = True yield remove_cmd yield add_cmd info["link_target"] = target # Check user/group if ((not info and (user or group)) or (user and info["user"] != user) or (group and info["group"] != group)): yield file_utils.chown(path, user, group, dereference=False) changed = True if user: info["user"] = user if group: info["group"] = group if not changed: host.noop("link {0} already exists".format(path))
def repo( src, dest, branch=None, pull=True, rebase=False, user=None, group=None, ssh_keyscan=False, update_submodules=False, recursive_submodules=False, ): """ Clone/pull git repositories. + src: the git source URL + dest: directory to clone to + branch: branch to pull/checkout + pull: pull any changes for the branch + rebase: when pulling, use ``--rebase`` + user: chown files to this user after + group: chown files to this group after + ssh_keyscan: keyscan the remote host if not in known_hosts before clone/pull + update_submodules: update any git submodules + recursive_submodules: update git submodules recursively **Example:** .. code:: python git.repo( name="Clone repo", src="https://github.com/Fizzadar/pyinfra.git", dest="/usr/local/src/pyinfra", ) """ # Ensure our target directory exists yield from files.directory(dest) # Do we need to scan for the remote host key? if ssh_keyscan: # Attempt to parse the domain from the git repository domain = re.match(r"^[a-zA-Z0-9]+@([0-9a-zA-Z\.\-]+)", src) if domain: yield from ssh.keyscan(domain.group(1)) else: raise OperationError( "Could not parse domain (to SSH keyscan) from: {0}".format( src), ) # Store git commands for directory prefix git_commands = [] git_dir = unix_path_join(dest, ".git") is_repo = host.get_fact(Directory, path=git_dir) # Cloning new repo? if not is_repo: if branch: git_commands.append("clone {0} --branch {1} .".format(src, branch)) else: git_commands.append("clone {0} .".format(src)) host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch) host.create_fact( Directory, kwargs={"path": git_dir}, data={ "user": user, "group": group }, ) # Ensuring existing repo else: if branch and host.get_fact(GitBranch, repo=dest) != branch: git_commands.append( "fetch") # fetch to ensure we have the branch locally git_commands.append("checkout {0}".format(branch)) host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch) if pull: if rebase: git_commands.append("pull --rebase") else: git_commands.append("pull") if update_submodules: if recursive_submodules: git_commands.append("submodule update --init --recursive") else: git_commands.append("submodule update --init") # Attach prefixes for directory command_prefix = "cd {0} && git".format(dest) git_commands = [ "{0} {1}".format(command_prefix, command) for command in git_commands ] for cmd in git_commands: yield cmd # Apply any user or group if we did anything if git_commands and (user or group): yield chown(dest, user, group, recursive=True)
def worktree( worktree, repo=None, detached=False, new_branch=None, commitish=None, pull=True, rebase=False, from_remote_branch=None, present=True, assume_repo_exists=False, force=False, user=None, group=None, ): """ Manage git worktrees. + worktree: git working tree directory + repo: git main repository directory + detached: create a working tree with a detached HEAD + new_branch: local branch name created at the same time than the worktree + commitish: from which git commit, branch, ... the worktree is created + pull: pull any changes from a remote branch if set + rebase: when pulling, use ``--rebase`` + from_remote_branch: a 2-tuple ``(remote, branch)`` that identifies a remote branch + present: whether the working tree should exist + assume_repo_exists: whether to assume the main repo exists + force: remove unclean working tree if should not exist + user: chown files to this user after + group: chown files to this group after **Examples:** .. code:: python git.worktree( name="Create a worktree from the current repo `HEAD`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix" ) git.worktree( name="Create a worktree from the commit `4e091aa0`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", commitish="4e091aa0" ) git.worktree( name="Create a worktree with a new local branch `v1.0`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", new_branch="v1.0", ) git.worktree( name="Create a worktree from the commit 4e091aa0 with the new local branch `v1.0`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", new_branch="v1.0", commitish="4e091aa0" ) git.worktree( name="Create a worktree with a detached `HEAD`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", detached=True, ) git.worktree( name="Create a worktree with a detached `HEAD` from commit `4e091aa0`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", commitish="4e091aa0", detached=True, ) git.worktree( name="Create a worktree from the existing local branch `v1.0`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", commitish="v1.0" ) git.worktree( name="Create a worktree with a new branch `v1.0` that tracks `origin/v1.0`", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", new_branch="v1.0", commitish="v1.0" ) git.worktree( name="Pull an existing worktree already linked to a tracking branch", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix" ) git.worktree( name="Pull an existing worktree from a specific remote branch", repo="/usr/local/src/pyinfra/master", worktree="/usr/local/src/pyinfra/hotfix", from_remote_branch=("origin", "master") ) git.worktree( name="Remove a worktree", worktree="/usr/local/src/pyinfra/hotfix", present=False, ) git.worktree( name="Remove an unclean worktree", worktree="/usr/local/src/pyinfra/hotfix", present=False, force=True, ) """ # Doesn't exist & we want it if not host.get_fact(Directory, path=worktree) and present: # be sure that `repo` is a GIT repository if not assume_repo_exists and not host.get_fact( Directory, path=unix_path_join(repo, ".git"), ): raise OperationError( "The following folder is not a valid GIT repository : {0}". format(repo), ) command_parts = ["cd {0} && git worktree add".format(repo)] if new_branch: command_parts.append("-b {0}".format(new_branch)) elif detached: command_parts.append("--detach") command_parts.append(worktree) if commitish: command_parts.append(commitish) yield " ".join(command_parts) # Apply any user or group if user or group: yield chown(worktree, user, group, recursive=True) host.create_fact(Directory, kwargs={"path": worktree}, data={ "user": user, "group": group }) host.create_fact(GitTrackingBranch, kwargs={"repo": worktree}, data=new_branch) # It exists and we don't want it elif host.get_fact(Directory, path=worktree) and not present: command = "cd {0} && git worktree remove .".format(worktree) if force: command += " --force" yield command host.delete_fact(Directory, kwargs={"path": worktree}) host.create_fact(GitTrackingBranch, kwargs={"repo": worktree}) # It exists and we still want it => pull/rebase it elif host.get_fact(Directory, path=worktree) and present: # pull the worktree only if it's already linked to a tracking branch or # if a remote branch is set if host.get_fact(GitTrackingBranch, repo=worktree) or from_remote_branch: command = "cd {0} && git pull".format(worktree) if rebase: command += " --rebase" if from_remote_branch: if len(from_remote_branch) != 2 or type( from_remote_branch) not in (tuple, list): raise OperationError( "The remote branch must be a 2-tuple (remote, branch) such as " '("origin", "master")', ) command += " {0} {1}".format(*from_remote_branch) yield command
def virtualenv( path, python=None, venv=False, site_packages=False, always_copy=False, present=True, ): """ Add/remove Python virtualenvs. + python: python interpreter to use + venv: use standard venv module instead of virtualenv + site_packages: give access to the global site-packages + always_copy: always copy files rather than symlinking + present: whether the virtualenv should exist **Example:** .. code:: python pip.virtualenv( name="Create a virtualenv", path="/usr/local/bin/venv", ) """ # Check for *contents* of a virtualenv, ie don't accept an empty directory # as a valid virtualenv but ensure the activate script exists. activate_script_path = "{0}/bin/activate".format(path) if present is False: if host.get_fact(File, path=activate_script_path): yield from files.directory(path, present=False) else: host.noop("virtualenv {0} does not exist".format(path)) if present: if not host.get_fact(File, path=activate_script_path): # Create missing virtualenv command = ["virtualenv"] if venv: command = [python or "python", "-m", "venv"] if python and not venv: command.append("-p {0}".format(python)) if site_packages: command.append("--system-site-packages") if always_copy and not venv: command.append("--always-copy") elif always_copy and venv: command.append("--copies") command.append(path) yield " ".join(command) host.create_fact( File, kwargs={"path": activate_script_path}, data={ "user": None, "group": None }, ) else: host.noop("virtualenv {0} exists".format(path))
def download( hostname, filename, local_filename=None, force=False, port=22, user=None, ssh_keyscan=False, ): """ Download files from other servers using ``scp``. + hostname: hostname to upload to + filename: file to download + local_filename: where to download the file to (defaults to ``filename``) + force: always download the file, even if present locally + port: connect to this port + user: connect with this user + ssh_keyscan: execute ``ssh.keyscan`` before uploading the file """ local_filename = local_filename or filename # Get local file info local_file_info = host.get_fact(File, path=local_filename) # Local file exists but isn't a file? if local_file_info is False: raise OperationError( "Local destination {0} already exists and is not a file".format( local_filename, ), ) # If the local file exists and we're not forcing a re-download, no-op if local_file_info and not force: host.noop("file {0} is already downloaded".format(filename)) return # Figure out where we're connecting (host or user@host) connection_target = hostname if user: connection_target = "@".join((user, hostname)) if ssh_keyscan: yield from keyscan(hostname) # Download the file with scp yield "scp -P {0} {1}:{2} {3}".format( port, connection_target, filename, local_filename, ) host.create_fact( File, kwargs={"path": local_filename}, data={ "mode": None, "group": None, "user": user, "mtime": None }, )
def directory( path, present=True, assume_present=False, user=None, group=None, mode=None, recursive=False, ): """ Add/remove/update directories. + path: path of the remote folder + present: whether the folder should exist + assume_present: whether to assume the directory exists + TODO: user: user to own the folder + TODO: group: group to own the folder + TODO: mode: permissions of the folder + TODO: recursive: recursively apply user/group/mode **Examples:** .. code:: python files.directory( name="Ensure the c:\\temp\\dir_that_we_want_removed is removed", path="c:\\temp\\dir_that_we_want_removed", present=False, ) files.directory( name="Ensure c:\\temp\\foo\\foo_dir exists", path="c:\\temp\\foo\\foo_dir", recursive=True, ) # multiple directories dirs = ["c:\\temp\\foo_dir1", "c:\\temp\\foo_dir2"] for dir in dirs: files.directory( name="Ensure the directory `{}` exists".format(dir), path=dir, ) """ if not isinstance(path, str): raise OperationTypeError("Name must be a string") info = host.get_fact(Directory, path=path) # Not a directory?! if info is False: raise OperationError("{0} exists and is not a directory".format(path)) # Doesn't exist & we want it if not assume_present and info is None and present: yield "New-Item -Path {0} -ItemType Directory".format(path) # if mode: # yield chmod(path, mode, recursive=recursive) # if user or group: # yield chown(path, user, group, recursive=recursive) # # Somewhat bare fact, should flesh out more host.create_fact( Date, kwargs={"path": path}, data={"type": "directory"}, ) # It exists and we don't want it elif (assume_present or info) and not present: # TODO: how to ensure we use 'ps'? # remove anything in the directory yield "Get-ChildItem {0} -Recurse | Remove-Item".format(path) # remove directory yield "Remove-Item {0}".format(path)