def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
headers=None,
insecure=False,
):
"""
Download files from remote locations using ``curl`` or ``wget``.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
+ headers: optional dictionary of headers to set for the HTTP request
+ insecure: disable SSL verification for the HTTP request
**Example:**
.. code:: python
files.download(
name="Download the Docker repo file",
src="https://download.docker.com/linux/centos/docker-ce.repo",
dest="/etc/yum.repos.d/docker-ce.repo",
)
"""
info = host.get_fact(File, path=dest)
host_datetime = host.get_fact(Date).replace(tzinfo=None)
# Destination is a directory?
if info is False:
raise OperationError(
"Destination {0} already exists and is not a file".format(dest), )
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from the Date fact before comparison.
cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(
seconds=cache_time)
if info["mtime"] and info["mtime"] < cache_time:
download = True
if sha1sum:
if sha1sum != host.get_fact(Sha1File, path=dest):
download = True
if sha256sum:
if sha256sum != host.get_fact(Sha256File, path=dest):
download = True
if md5sum:
if md5sum != host.get_fact(Md5File, path=dest):
download = True
# If we download, always do user/group/mode as SSH user may be different
if download:
temp_file = state.get_temp_filename(dest)
curl_args = ["-sSLf"]
wget_args = ["-q"]
if insecure:
curl_args.append("--insecure")
wget_args.append("--no-check-certificate")
if headers:
for key, value in headers.items():
header_arg = StringCommand("--header",
QuoteString(f"{key}: {value}"))
curl_args.append(header_arg)
wget_args.append(header_arg)
curl_command = make_formatted_string_command(
"curl {0} {1} -o {2}",
StringCommand(*curl_args),
QuoteString(src),
QuoteString(temp_file),
)
wget_command = make_formatted_string_command(
"wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )",
StringCommand(*wget_args),
QuoteString(src),
QuoteString(temp_file),
)
if host.get_fact(Which, command="curl"):
yield curl_command
elif host.get_fact(Which, command="wget"):
yield wget_command
else:
yield "( {0} ) || ( {1} )".format(curl_command, wget_command)
yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest))
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
if sha1sum:
yield make_formatted_string_command(
("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
sha1sum,
QuoteString("SHA1 did not match!"),
)
if sha256sum:
yield make_formatted_string_command(
("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) "
"| grep {1}) || ( echo {2} && exit 1 )"),
QuoteString(dest),
sha256sum,
QuoteString("SHA256 did not match!"),
)
if md5sum:
yield make_formatted_string_command(
("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
md5sum,
QuoteString("MD5 did not match!"),
)
host.create_fact(
File,
kwargs={"path": dest},
data={
"mode": mode,
"group": group,
"user": user,
"mtime": host_datetime
},
)
# Remove any checksum facts as we don't know the correct values
for value, fact_cls in (
(sha1sum, Sha1File),
(sha256sum, Sha256File),
(md5sum, Md5File),
):
fact_kwargs = {"path": dest}
if value:
host.create_fact(fact_cls, kwargs=fact_kwargs, data=value)
else:
host.delete_fact(fact_cls, kwargs=fact_kwargs)
else:
host.noop("file {0} has already been downloaded".format(dest))
def line(
path,
line,
present=True,
replace=None,
flags=None,
backup=False,
interpolate_variables=False,
escape_regex_characters=False,
assume_present=False,
):
"""
Ensure lines in files using grep to locate and sed to replace.
+ path: target remote file to edit
+ line: string or regex matching the target line
+ present: whether the line should be in the file
+ replace: text to replace entire matching lines when ``present=True``
+ flags: list of flags to pass to sed when replacing/deleting
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
+ assume_present: whether to assume a matching line already exists in the file
+ escape_regex_characters: whether to escape regex characters from the matching line
Regex line matching:
Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that
it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To
change bits of lines, see ``files.replace``.
Regex line escaping:
If matching special characters (eg a crontab line containing *), remember to escape
it first using Python's ``re.escape``.
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
Append:
If ``line`` is not in the file but we want it (``present`` set to ``True``), then
it will be append to the end of the file.
**Examples:**
.. code:: python
# prepare to do some maintenance
maintenance_line = "SYSTEM IS DOWN FOR MAINTENANCE"
files.line(
name="Add the down-for-maintence line in /etc/motd",
path="/etc/motd",
line=maintenance_line,
)
# Then, after the mantenance is done, remove the maintenance line
files.line(
name="Remove the down-for-maintenance line in /etc/motd",
path="/etc/motd",
line=maintenance_line,
replace="",
present=False,
)
# example where there is '*' in the line
files.line(
name="Ensure /netboot/nfs is in /etc/exports",
path="/etc/exports",
line=r"/netboot/nfs .*",
replace="/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash,"
"insecure,no_subtree_check)",
)
files.line(
name="Ensure myweb can run /usr/bin/python3 without password",
path="/etc/sudoers",
line=r"myweb .*",
replace="myweb ALL=(ALL) NOPASSWD: /usr/bin/python3",
)
# example when there are double quotes (")
line = 'QUOTAUSER=""'
files.line(
name="Example with double quotes (")",
path="/etc/adduser.conf",
line="^{}$".format(line),
replace=line,
)
"""
match_line = line
if escape_regex_characters:
match_line = re.sub(r"([\.\\\+\*\?\[\^\]\$\(\)\{\}\-])", r"\\\1",
match_line)
# Ensure we're matching a whole line, note: match may be a partial line so we
# put any matches on either side.
if not match_line.startswith("^"):
match_line = "^.*{0}".format(match_line)
if not match_line.endswith("$"):
match_line = "{0}.*$".format(match_line)
# Is there a matching line in this file?
if assume_present:
present_lines = [line]
else:
present_lines = host.get_fact(
FindInFile,
path=path,
pattern=match_line,
interpolate_variables=interpolate_variables,
)
# If replace present, use that over the matching line
if replace:
line = replace
# We must provide some kind of replace to sed_replace_command below
else:
replace = ""
# Save commands for re-use in dynamic script when file not present at fact stage
echo_command = make_formatted_string_command(
"echo {0} >> {1}",
'"{0}"'.format(line) if interpolate_variables else QuoteString(line),
QuoteString(path),
)
if backup:
backup_filename = "{0}.{1}".format(path, get_timestamp())
echo_command = StringCommand(
make_formatted_string_command(
"cp {0} {1} && ",
QuoteString(path),
QuoteString(backup_filename),
),
echo_command,
)
sed_replace_command = sed_replace(
path,
match_line,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
# No line and we want it, append it
if not present_lines and present:
# If the file does not exist - it *might* be created, so we handle it
# dynamically with a little script.
if present_lines is None:
yield make_formatted_string_command(
"""
if [ -f '{target}' ]; then
( grep {match_line} '{target}' && \
{sed_replace_command}) 2> /dev/null || \
{echo_command} ;
else
{echo_command} ;
fi
""",
target=QuoteString(path),
match_line=QuoteString(match_line),
echo_command=echo_command,
sed_replace_command=sed_replace_command,
)
# File exists but has no matching lines - append it.
else:
# If we're doing replacement, only append if the *replacement* line
# does not exist (as we are appending the replacement).
if replace:
# Ensure replace explicitly matches a whole line
replace_line = replace
if not replace_line.startswith("^"):
replace_line = f"^{replace_line}"
if not replace_line.endswith("$"):
replace_line = f"{replace_line}$"
present_lines = host.get_fact(
FindInFile,
path=path,
pattern=replace_line,
interpolate_variables=interpolate_variables,
)
if not present_lines:
yield echo_command
else:
host.noop('line "{0}" exists in {1}'.format(
replace or line, path))
host.create_fact(
FindInFile,
kwargs={
"path": path,
"pattern": match_line,
"interpolate_variables": interpolate_variables,
},
data=[replace or line],
)
# Line(s) exists and we want to remove them, replace with nothing
elif present_lines and not present:
yield sed_replace(
path,
match_line,
"",
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
host.delete_fact(
FindInFile,
kwargs={
"path": path,
"pattern": match_line,
"interpolate_variables": interpolate_variables,
},
)
# Line(s) exists and we have want to ensure they're correct
elif present_lines and present:
# If any of lines are different, sed replace them
if replace and any(line != replace for line in present_lines):
yield sed_replace_command
del present_lines[:] # TODO: use .clear() when py3+
present_lines.append(replace)
else:
host.noop('line "{0}" exists in {1}'.format(replace or line, path))
def directory(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
force=False,
force_backup=True,
force_backup_dir=None,
_no_check_owner_mode=False,
_no_fail_on_link=False,
):
"""
Add/remove/update directories.
+ path: path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ user: user to own the folder
+ group: group to own the folder
+ mode: permissions of the folder
+ recursive: recursively apply user/group/mode
+ force: if the target exists and is not a file, move or remove it and continue
+ force_backup: set to ``False`` to remove any existing non-file when ``force=True``
+ force_backup_dir: directory to move any backup to when ``force=True``
**Examples:**
.. code:: python
files.directory(
name="Ensure the /tmp/dir_that_we_want_removed is removed",
path="/tmp/dir_that_we_want_removed",
present=False,
)
files.directory(
name="Ensure /web exists",
path="/web",
user="******",
group="myweb",
)
# Multiple directories
for dir in ["/netboot/tftp", "/netboot/nfs"]:
files.directory(
name="Ensure the directory `{}` exists".format(dir),
path=dir,
)
"""
path = _validate_path(path)
mode = ensure_mode_int(mode)
info = host.get_fact(Directory, path=path)
# Not a directory?!
if info is False:
if _no_fail_on_link and host.get_fact(Link, path=path):
host.noop("directory {0} already exists (as a link)".format(path))
return
yield from _raise_or_remove_invalid_path(
"directory",
path,
force,
force_backup,
force_backup_dir,
)
info = None
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield StringCommand("mkdir", "-p", QuoteString(path))
if mode:
yield file_utils.chmod(path, mode, recursive=recursive)
if user or group:
yield file_utils.chown(path, user, group, recursive=recursive)
host.create_fact(
Directory,
kwargs={"path": path},
data={
"mode": mode,
"group": group,
"user": user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield StringCommand("rm", "-rf", QuoteString(path))
host.delete_fact(Directory, kwargs={"path": path})
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {"mode": None, "group": None, "user": None}
host.create_fact(Directory, kwargs={"path": path}, data=info)
if _no_check_owner_mode:
return
changed = False
if mode and (not info or info["mode"] != mode):
yield file_utils.chmod(path, mode, recursive=recursive)
info["mode"] = mode
changed = True
if ((not info and (user or group)) or (user and info["user"] != user)
or (group and info["group"] != group)):
yield file_utils.chown(path, user, group, recursive=recursive)
changed = True
if user:
info["user"] = user
if group:
info["group"] = group
if not changed:
host.noop("directory {0} already exists".format(path))
def file(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=True,
force=False,
force_backup=True,
force_backup_dir=None,
):
"""
Add/remove/update files.
+ path: name/path of the remote file
+ present: whether the file should exist
+ assume_present: whether to assume the file exists
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: if the target exists and is not a file, move or remove it and continue
+ force_backup: set to ``False`` to remove any existing non-file when ``force=True``
+ force_backup_dir: directory to move any backup to when ``force=True``
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
**Example:**
.. code:: python
# Note: The directory /tmp/secret will get created with the default umask.
files.file(
name="Create /tmp/secret/file",
path="/tmp/secret/file",
mode="600",
user="******",
group="root",
touch=True,
create_remote_dir=True,
)
"""
path = _validate_path(path)
mode = ensure_mode_int(mode)
info = host.get_fact(File, path=path)
# Not a file?!
if info is False:
yield from _raise_or_remove_invalid_path(
"file",
path,
force,
force_backup,
force_backup_dir,
)
info = None
# Doesn't exist & we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield from _create_remote_dir(state, host, path, user, group)
yield StringCommand("touch", QuoteString(path))
if mode:
yield file_utils.chmod(path, mode)
if user or group:
yield file_utils.chown(path, user, group)
host.create_fact(
File,
kwargs={"path": path},
data={
"mode": mode,
"group": group,
"user": user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield StringCommand("rm", "-f", QuoteString(path))
host.delete_fact(File, kwargs={"path": path})
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {"mode": None, "group": None, "user": None}
host.create_fact(File, kwargs={"path": path}, data=info)
changed = False
if touch:
changed = True
yield StringCommand("touch", QuoteString(path))
# Check mode
if mode and (not info or info["mode"] != mode):
yield file_utils.chmod(path, mode)
info["mode"] = mode
changed = True
# Check user/group
if ((not info and (user or group)) or (user and info["user"] != user)
or (group and info["group"] != group)):
yield file_utils.chown(path, user, group)
changed = True
if user:
info["user"] = user
if group:
info["group"] = group
if not changed:
host.noop("file {0} already exists".format(path))
def link(
path,
target=None,
present=True,
assume_present=False,
user=None,
group=None,
symbolic=True,
create_remote_dir=True,
force=False,
force_backup=True,
force_backup_dir=None,
):
"""
Add/remove/update links.
+ path: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ assume_present: whether to assume the link exists
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: if the target exists and is not a file, move or remove it and continue
+ force_backup: set to ``False`` to remove any existing non-file when ``force=True``
+ force_backup_dir: directory to move any backup to when ``force=True``
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
**Examples:**
.. code:: python
files.link(
name="Create link /etc/issue2 that points to /etc/issue",
path="/etc/issue2",
target="/etc/issue",
)
# Complex example demonstrating the assume_present option
from pyinfra.operations import apt, files
install_nginx = apt.packages(
name="Install nginx",
packages=["nginx"],
)
files.link(
name="Remove default nginx site",
path="/etc/nginx/sites-enabled/default",
present=False,
assume_present=install_nginx.changed,
)
"""
path = _validate_path(path)
if present and not target:
raise OperationError("If present is True target must be provided")
info = host.get_fact(Link, path=path)
# Not a link?
if info is False:
yield from _raise_or_remove_invalid_path(
"link",
path,
force,
force_backup,
force_backup_dir,
)
info = None
add_args = ["ln"]
if symbolic:
add_args.append("-s")
add_cmd = StringCommand(" ".join(add_args), QuoteString(target),
QuoteString(path))
remove_cmd = StringCommand("rm", "-f", QuoteString(path))
# No link and we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield from _create_remote_dir(state, host, path, user, group)
yield add_cmd
if user or group:
yield file_utils.chown(path, user, group, dereference=False)
host.create_fact(
Link,
kwargs={"path": path},
data={
"link_target": target,
"group": group,
"user": user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield remove_cmd
host.delete_fact(Link, kwargs={"path": path})
# Exists and want to ensure it's state
elif (assume_present or info) and present:
if assume_present and not info:
info = {"link_target": None, "group": None, "user": None}
host.create_fact(Link, kwargs={"path": path}, data=info)
# If we have an absolute path - prepend to any non-absolute values from the fact
# and/or the source.
if os.path.isabs(path):
link_dirname = os.path.dirname(path)
if not os.path.isabs(target):
target = os.path.normpath(unix_path_join(link_dirname, target))
if info and not os.path.isabs(info["link_target"]):
info["link_target"] = os.path.normpath(
unix_path_join(link_dirname, info["link_target"]), )
changed = False
# If the target is wrong, remove & recreate the link
if not info or info["link_target"] != target:
changed = True
yield remove_cmd
yield add_cmd
info["link_target"] = target
# Check user/group
if ((not info and (user or group)) or (user and info["user"] != user)
or (group and info["group"] != group)):
yield file_utils.chown(path, user, group, dereference=False)
changed = True
if user:
info["user"] = user
if group:
info["group"] = group
if not changed:
host.noop("link {0} already exists".format(path))
def worktree(
worktree,
repo=None,
detached=False,
new_branch=None,
commitish=None,
pull=True,
rebase=False,
from_remote_branch=None,
present=True,
assume_repo_exists=False,
force=False,
user=None,
group=None,
):
"""
Manage git worktrees.
+ worktree: git working tree directory
+ repo: git main repository directory
+ detached: create a working tree with a detached HEAD
+ new_branch: local branch name created at the same time than the worktree
+ commitish: from which git commit, branch, ... the worktree is created
+ pull: pull any changes from a remote branch if set
+ rebase: when pulling, use ``--rebase``
+ from_remote_branch: a 2-tuple ``(remote, branch)`` that identifies a remote branch
+ present: whether the working tree should exist
+ assume_repo_exists: whether to assume the main repo exists
+ force: remove unclean working tree if should not exist
+ user: chown files to this user after
+ group: chown files to this group after
**Examples:**
.. code:: python
git.worktree(
name="Create a worktree from the current repo `HEAD`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix"
)
git.worktree(
name="Create a worktree from the commit `4e091aa0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="4e091aa0"
)
git.worktree(
name="Create a worktree with a new local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
)
git.worktree(
name="Create a worktree from the commit 4e091aa0 with the new local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
commitish="4e091aa0"
)
git.worktree(
name="Create a worktree with a detached `HEAD`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
detached=True,
)
git.worktree(
name="Create a worktree with a detached `HEAD` from commit `4e091aa0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="4e091aa0",
detached=True,
)
git.worktree(
name="Create a worktree from the existing local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="v1.0"
)
git.worktree(
name="Create a worktree with a new branch `v1.0` that tracks `origin/v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
commitish="v1.0"
)
git.worktree(
name="Pull an existing worktree already linked to a tracking branch",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix"
)
git.worktree(
name="Pull an existing worktree from a specific remote branch",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
from_remote_branch=("origin", "master")
)
git.worktree(
name="Remove a worktree",
worktree="/usr/local/src/pyinfra/hotfix",
present=False,
)
git.worktree(
name="Remove an unclean worktree",
worktree="/usr/local/src/pyinfra/hotfix",
present=False,
force=True,
)
"""
# Doesn't exist & we want it
if not host.get_fact(Directory, path=worktree) and present:
# be sure that `repo` is a GIT repository
if not assume_repo_exists and not host.get_fact(
Directory,
path=unix_path_join(repo, ".git"),
):
raise OperationError(
"The following folder is not a valid GIT repository : {0}".
format(repo), )
command_parts = ["cd {0} && git worktree add".format(repo)]
if new_branch:
command_parts.append("-b {0}".format(new_branch))
elif detached:
command_parts.append("--detach")
command_parts.append(worktree)
if commitish:
command_parts.append(commitish)
yield " ".join(command_parts)
# Apply any user or group
if user or group:
yield chown(worktree, user, group, recursive=True)
host.create_fact(Directory,
kwargs={"path": worktree},
data={
"user": user,
"group": group
})
host.create_fact(GitTrackingBranch,
kwargs={"repo": worktree},
data=new_branch)
# It exists and we don't want it
elif host.get_fact(Directory, path=worktree) and not present:
command = "cd {0} && git worktree remove .".format(worktree)
if force:
command += " --force"
yield command
host.delete_fact(Directory, kwargs={"path": worktree})
host.create_fact(GitTrackingBranch, kwargs={"repo": worktree})
# It exists and we still want it => pull/rebase it
elif host.get_fact(Directory, path=worktree) and present:
# pull the worktree only if it's already linked to a tracking branch or
# if a remote branch is set
if host.get_fact(GitTrackingBranch,
repo=worktree) or from_remote_branch:
command = "cd {0} && git pull".format(worktree)
if rebase:
command += " --rebase"
if from_remote_branch:
if len(from_remote_branch) != 2 or type(
from_remote_branch) not in (tuple, list):
raise OperationError(
"The remote branch must be a 2-tuple (remote, branch) such as "
'("origin", "master")', )
command += " {0} {1}".format(*from_remote_branch)
yield command