def keyscan(hostname, force=False, port=22):
"""
Check/add hosts to the ``~/.ssh/known_hosts`` file.
+ hostname: hostname that should have a key in ``known_hosts``
+ force: if the key already exists, remove and rescan
**Example:**
.. code:: python
ssh.keyscan(
name="Set add server two to known_hosts on one",
hostname="two.example.com",
)
"""
homedir = host.get_fact(Home)
yield from files.directory(
"{0}/.ssh".format(homedir),
mode=700,
)
hostname_present = host.get_fact(
FindInFile,
path="{0}/.ssh/known_hosts".format(homedir),
pattern=hostname,
)
did_keyscan = False
keyscan_command = "ssh-keyscan -p {0} {1} >> {2}/.ssh/known_hosts".format(
port,
hostname,
homedir,
)
if not hostname_present:
yield keyscan_command
did_keyscan = True
elif force:
yield "ssh-keygen -R {0}".format(hostname)
yield keyscan_command
did_keyscan = True
else:
host.noop("host key for {0} already exists".format(hostname))
if did_keyscan:
host.create_fact(
FindInFile,
kwargs={
"path": "{0}/.ssh/known_hosts".format(homedir),
"pattern": hostname
},
data=["{0} unknown unknown".format(hostname)],
)
def hostname(hostname, hostname_file=None):
"""
Set the system hostname using ``hostnamectl`` or ``hostname`` on older systems.
+ hostname: the hostname that should be set
+ hostname_file: the file that permanently sets the hostname
Hostname file:
The hostname file only matters no systems that do not have ``hostnamectl``,
which is part of ``systemd``.
By default pyinfra will auto detect this by targeting ``/etc/hostname``
on Linux and ``/etc/myname`` on OpenBSD.
To completely disable writing the hostname file, set ``hostname_file=False``.
**Example:**
.. code:: python
server.hostname(
name="Set the hostname",
hostname="server1.example.com",
)
"""
current_hostname = host.get_fact(Hostname)
if host.get_fact(Which, command="hostnamectl"):
if current_hostname != hostname:
yield "hostnamectl set-hostname {0}".format(hostname)
host.create_fact(Hostname, data=hostname)
else:
host.noop("hostname is set")
return
if hostname_file is None:
os = host.get_fact(Os)
if os == "Linux":
hostname_file = "/etc/hostname"
elif os == "OpenBSD":
hostname_file = "/etc/myname"
if current_hostname != hostname:
yield "hostname {0}".format(hostname)
host.create_fact(Hostname, data=hostname)
else:
host.noop("hostname is set")
if hostname_file:
# Create a whole new hostname file
file = StringIO("{0}\n".format(hostname))
# And ensure it exists
yield from files.put(file, hostname_file)
def bare_repo(
path,
user=None,
group=None,
present=True,
):
"""
Create bare git repositories.
+ path: path to the folder
+ present: whether the bare repository should exist
+ user: chown files to this user after
+ group: chown files to this group after
**Example:**
.. code:: python
git.bare_repo(
name="Create bare repo",
path="/home/git/test.git",
)
"""
yield from files.directory(path, present=present)
if present:
head_filename = unix_path_join(path, "HEAD")
head_file = host.get_fact(File, path=head_filename)
if not head_file:
yield "git init --bare {0}".format(path)
if user or group:
yield chown(path, user, group, recursive=True)
else:
if (user and head_file["user"] != user) or (
group and head_file["group"] != group):
yield chown(path, user, group, recursive=True)
host.create_fact(
File,
kwargs={"path": head_filename},
data={
"user": user,
"group": group,
"mode": None
},
)
def update(cache_time=None):
"""
Updates apt repositories.
+ cache_time: cache updates for this many seconds
**Example:**
.. code:: python
apt.update(
name="Update apt repositories",
cache_time=3600,
)
"""
# If cache_time check when apt was last updated, prevent updates if within time
if cache_time:
# Ubuntu provides this handy file
cache_info = host.get_fact(File, path=APT_UPDATE_FILENAME)
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from the Date fact before comparison.
host_cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(
seconds=cache_time)
if cache_info and cache_info[
"mtime"] and cache_info["mtime"] > host_cache_time:
host.noop("apt is already up to date")
return
yield "apt-get update"
# Some apt systems (Debian) have the /var/lib/apt/periodic directory, but
# don't bother touching anything in there - so pyinfra does it, enabling
# cache_time to work.
if cache_time:
yield "touch {0}".format(APT_UPDATE_FILENAME)
if cache_info is None:
host.create_fact(
File,
kwargs={"path": APT_UPDATE_FILENAME},
data={"mtime": datetime.utcnow()},
)
else:
cache_info["mtime"] = datetime.utcnow()
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
):
"""
Upload a local file, or file-like object, to the remote system.
+ src: filename or IO-like object to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files, use ``True`` to copy the local file
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``dest``:
If this is a directory that already exists on the remote side, the local
file will be uploaded to that directory with the same filename.
``mode``:
When set to ``True`` the permissions of the local file are applied to the
remote file after the upload is complete.
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
**Examples:**
.. code:: python
files.put(
name="Update the message of the day file",
src="files/motd",
dest="/etc/motd",
mode="644",
)
files.put(
name="Upload a StringIO object",
src=StringIO("file contents"),
dest="/etc/motd",
)
"""
# Upload IO objects as-is
if hasattr(src, "read"):
local_file = src
local_sum = get_file_sha1(src)
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.cwd:
src = os.path.join(state.cwd, src)
local_file = src
if os.path.isfile(local_file):
local_sum = get_file_sha1(local_file)
elif assume_exists:
local_sum = None
else:
raise IOError("No such file: {0}".format(local_file))
if mode is True:
if os.path.isfile(local_file):
mode = get_path_permissions_mode(local_file)
else:
logger.warning((
"No local file exists to get permissions from with `mode=True` ({0})"
).format(get_call_location(), ), )
else:
mode = ensure_mode_int(mode)
remote_file = host.get_fact(File, path=dest)
if not remote_file and host.get_fact(Directory, path=dest):
dest = unix_path_join(dest, os.path.basename(src))
remote_file = host.get_fact(File, path=dest)
if create_remote_dir:
yield from _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
remote_sum = host.get_fact(Sha1File, path=dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
else:
changed = False
# Check mode
if mode and remote_file["mode"] != mode:
yield file_utils.chmod(dest, mode)
changed = True
# Check user/group
if (user and remote_file["user"] != user) or (
group and remote_file["group"] != group):
yield file_utils.chown(dest, user, group)
changed = True
if not changed:
host.noop("file {0} is already uploaded".format(dest))
# Now we've uploaded the file and ensured user/group/mode, update the relevant fact data
host.create_fact(Sha1File, kwargs={"path": dest}, data=local_sum)
host.create_fact(
File,
kwargs={"path": dest},
data={
"user": user,
"group": group,
"mode": mode
},
)
def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
headers=None,
insecure=False,
):
"""
Download files from remote locations using ``curl`` or ``wget``.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
+ headers: optional dictionary of headers to set for the HTTP request
+ insecure: disable SSL verification for the HTTP request
**Example:**
.. code:: python
files.download(
name="Download the Docker repo file",
src="https://download.docker.com/linux/centos/docker-ce.repo",
dest="/etc/yum.repos.d/docker-ce.repo",
)
"""
info = host.get_fact(File, path=dest)
host_datetime = host.get_fact(Date).replace(tzinfo=None)
# Destination is a directory?
if info is False:
raise OperationError(
"Destination {0} already exists and is not a file".format(dest), )
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from the Date fact before comparison.
cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(
seconds=cache_time)
if info["mtime"] and info["mtime"] < cache_time:
download = True
if sha1sum:
if sha1sum != host.get_fact(Sha1File, path=dest):
download = True
if sha256sum:
if sha256sum != host.get_fact(Sha256File, path=dest):
download = True
if md5sum:
if md5sum != host.get_fact(Md5File, path=dest):
download = True
# If we download, always do user/group/mode as SSH user may be different
if download:
temp_file = state.get_temp_filename(dest)
curl_args = ["-sSLf"]
wget_args = ["-q"]
if insecure:
curl_args.append("--insecure")
wget_args.append("--no-check-certificate")
if headers:
for key, value in headers.items():
header_arg = StringCommand("--header",
QuoteString(f"{key}: {value}"))
curl_args.append(header_arg)
wget_args.append(header_arg)
curl_command = make_formatted_string_command(
"curl {0} {1} -o {2}",
StringCommand(*curl_args),
QuoteString(src),
QuoteString(temp_file),
)
wget_command = make_formatted_string_command(
"wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )",
StringCommand(*wget_args),
QuoteString(src),
QuoteString(temp_file),
)
if host.get_fact(Which, command="curl"):
yield curl_command
elif host.get_fact(Which, command="wget"):
yield wget_command
else:
yield "( {0} ) || ( {1} )".format(curl_command, wget_command)
yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest))
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
if sha1sum:
yield make_formatted_string_command(
("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
sha1sum,
QuoteString("SHA1 did not match!"),
)
if sha256sum:
yield make_formatted_string_command(
("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) "
"| grep {1}) || ( echo {2} && exit 1 )"),
QuoteString(dest),
sha256sum,
QuoteString("SHA256 did not match!"),
)
if md5sum:
yield make_formatted_string_command(
("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
md5sum,
QuoteString("MD5 did not match!"),
)
host.create_fact(
File,
kwargs={"path": dest},
data={
"mode": mode,
"group": group,
"user": user,
"mtime": host_datetime
},
)
# Remove any checksum facts as we don't know the correct values
for value, fact_cls in (
(sha1sum, Sha1File),
(sha256sum, Sha256File),
(md5sum, Md5File),
):
fact_kwargs = {"path": dest}
if value:
host.create_fact(fact_cls, kwargs=fact_kwargs, data=value)
else:
host.delete_fact(fact_cls, kwargs=fact_kwargs)
else:
host.noop("file {0} has already been downloaded".format(dest))
def replace(
path,
text=None,
replace=None,
flags=None,
backup=False,
interpolate_variables=False,
match=None, # deprecated
):
"""
Replace contents of a file using ``sed``.
+ path: target remote file to edit
+ text: text/regex to match against
+ replace: text to replace with
+ flags: list of flags to pass to sed
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
**Example:**
.. code:: python
files.replace(
name="Change part of a line in a file",
path="/etc/motd",
text="verboten",
replace="forbidden",
)
"""
if text is None and match:
text = match
logger.warning(
("The `match` argument has been replaced by "
"`text` in the `files.replace` operation ({0})").format(
get_call_location()), )
if text is None:
raise TypeError(
"Missing argument `text` required in `files.replace` operation")
if replace is None:
raise TypeError(
"Missing argument `replace` required in `files.replace` operation")
existing_lines = host.get_fact(
FindInFile,
path=path,
pattern=text,
interpolate_variables=interpolate_variables,
)
# Only do the replacement if the file does not exist (it may be created earlier)
# or we have matching lines.
if existing_lines is None or existing_lines:
yield sed_replace(
path,
text,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
host.create_fact(
FindInFile,
kwargs={
"path": path,
"pattern": text,
"interpolate_variables": interpolate_variables,
},
data=[],
)
else:
host.noop('string "{0}" does not exist in {1}'.format(text, path))
def line(
path,
line,
present=True,
replace=None,
flags=None,
backup=False,
interpolate_variables=False,
escape_regex_characters=False,
assume_present=False,
):
"""
Ensure lines in files using grep to locate and sed to replace.
+ path: target remote file to edit
+ line: string or regex matching the target line
+ present: whether the line should be in the file
+ replace: text to replace entire matching lines when ``present=True``
+ flags: list of flags to pass to sed when replacing/deleting
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
+ assume_present: whether to assume a matching line already exists in the file
+ escape_regex_characters: whether to escape regex characters from the matching line
Regex line matching:
Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that
it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To
change bits of lines, see ``files.replace``.
Regex line escaping:
If matching special characters (eg a crontab line containing *), remember to escape
it first using Python's ``re.escape``.
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
Append:
If ``line`` is not in the file but we want it (``present`` set to ``True``), then
it will be append to the end of the file.
**Examples:**
.. code:: python
# prepare to do some maintenance
maintenance_line = "SYSTEM IS DOWN FOR MAINTENANCE"
files.line(
name="Add the down-for-maintence line in /etc/motd",
path="/etc/motd",
line=maintenance_line,
)
# Then, after the mantenance is done, remove the maintenance line
files.line(
name="Remove the down-for-maintenance line in /etc/motd",
path="/etc/motd",
line=maintenance_line,
replace="",
present=False,
)
# example where there is '*' in the line
files.line(
name="Ensure /netboot/nfs is in /etc/exports",
path="/etc/exports",
line=r"/netboot/nfs .*",
replace="/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash,"
"insecure,no_subtree_check)",
)
files.line(
name="Ensure myweb can run /usr/bin/python3 without password",
path="/etc/sudoers",
line=r"myweb .*",
replace="myweb ALL=(ALL) NOPASSWD: /usr/bin/python3",
)
# example when there are double quotes (")
line = 'QUOTAUSER=""'
files.line(
name="Example with double quotes (")",
path="/etc/adduser.conf",
line="^{}$".format(line),
replace=line,
)
"""
match_line = line
if escape_regex_characters:
match_line = re.sub(r"([\.\\\+\*\?\[\^\]\$\(\)\{\}\-])", r"\\\1",
match_line)
# Ensure we're matching a whole line, note: match may be a partial line so we
# put any matches on either side.
if not match_line.startswith("^"):
match_line = "^.*{0}".format(match_line)
if not match_line.endswith("$"):
match_line = "{0}.*$".format(match_line)
# Is there a matching line in this file?
if assume_present:
present_lines = [line]
else:
present_lines = host.get_fact(
FindInFile,
path=path,
pattern=match_line,
interpolate_variables=interpolate_variables,
)
# If replace present, use that over the matching line
if replace:
line = replace
# We must provide some kind of replace to sed_replace_command below
else:
replace = ""
# Save commands for re-use in dynamic script when file not present at fact stage
echo_command = make_formatted_string_command(
"echo {0} >> {1}",
'"{0}"'.format(line) if interpolate_variables else QuoteString(line),
QuoteString(path),
)
if backup:
backup_filename = "{0}.{1}".format(path, get_timestamp())
echo_command = StringCommand(
make_formatted_string_command(
"cp {0} {1} && ",
QuoteString(path),
QuoteString(backup_filename),
),
echo_command,
)
sed_replace_command = sed_replace(
path,
match_line,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
# No line and we want it, append it
if not present_lines and present:
# If the file does not exist - it *might* be created, so we handle it
# dynamically with a little script.
if present_lines is None:
yield make_formatted_string_command(
"""
if [ -f '{target}' ]; then
( grep {match_line} '{target}' && \
{sed_replace_command}) 2> /dev/null || \
{echo_command} ;
else
{echo_command} ;
fi
""",
target=QuoteString(path),
match_line=QuoteString(match_line),
echo_command=echo_command,
sed_replace_command=sed_replace_command,
)
# File exists but has no matching lines - append it.
else:
# If we're doing replacement, only append if the *replacement* line
# does not exist (as we are appending the replacement).
if replace:
# Ensure replace explicitly matches a whole line
replace_line = replace
if not replace_line.startswith("^"):
replace_line = f"^{replace_line}"
if not replace_line.endswith("$"):
replace_line = f"{replace_line}$"
present_lines = host.get_fact(
FindInFile,
path=path,
pattern=replace_line,
interpolate_variables=interpolate_variables,
)
if not present_lines:
yield echo_command
else:
host.noop('line "{0}" exists in {1}'.format(
replace or line, path))
host.create_fact(
FindInFile,
kwargs={
"path": path,
"pattern": match_line,
"interpolate_variables": interpolate_variables,
},
data=[replace or line],
)
# Line(s) exists and we want to remove them, replace with nothing
elif present_lines and not present:
yield sed_replace(
path,
match_line,
"",
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
host.delete_fact(
FindInFile,
kwargs={
"path": path,
"pattern": match_line,
"interpolate_variables": interpolate_variables,
},
)
# Line(s) exists and we have want to ensure they're correct
elif present_lines and present:
# If any of lines are different, sed replace them
if replace and any(line != replace for line in present_lines):
yield sed_replace_command
del present_lines[:] # TODO: use .clear() when py3+
present_lines.append(replace)
else:
host.noop('line "{0}" exists in {1}'.format(replace or line, path))
def directory(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
force=False,
force_backup=True,
force_backup_dir=None,
_no_check_owner_mode=False,
_no_fail_on_link=False,
):
"""
Add/remove/update directories.
+ path: path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ user: user to own the folder
+ group: group to own the folder
+ mode: permissions of the folder
+ recursive: recursively apply user/group/mode
+ force: if the target exists and is not a file, move or remove it and continue
+ force_backup: set to ``False`` to remove any existing non-file when ``force=True``
+ force_backup_dir: directory to move any backup to when ``force=True``
**Examples:**
.. code:: python
files.directory(
name="Ensure the /tmp/dir_that_we_want_removed is removed",
path="/tmp/dir_that_we_want_removed",
present=False,
)
files.directory(
name="Ensure /web exists",
path="/web",
user="******",
group="myweb",
)
# Multiple directories
for dir in ["/netboot/tftp", "/netboot/nfs"]:
files.directory(
name="Ensure the directory `{}` exists".format(dir),
path=dir,
)
"""
path = _validate_path(path)
mode = ensure_mode_int(mode)
info = host.get_fact(Directory, path=path)
# Not a directory?!
if info is False:
if _no_fail_on_link and host.get_fact(Link, path=path):
host.noop("directory {0} already exists (as a link)".format(path))
return
yield from _raise_or_remove_invalid_path(
"directory",
path,
force,
force_backup,
force_backup_dir,
)
info = None
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield StringCommand("mkdir", "-p", QuoteString(path))
if mode:
yield file_utils.chmod(path, mode, recursive=recursive)
if user or group:
yield file_utils.chown(path, user, group, recursive=recursive)
host.create_fact(
Directory,
kwargs={"path": path},
data={
"mode": mode,
"group": group,
"user": user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield StringCommand("rm", "-rf", QuoteString(path))
host.delete_fact(Directory, kwargs={"path": path})
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {"mode": None, "group": None, "user": None}
host.create_fact(Directory, kwargs={"path": path}, data=info)
if _no_check_owner_mode:
return
changed = False
if mode and (not info or info["mode"] != mode):
yield file_utils.chmod(path, mode, recursive=recursive)
info["mode"] = mode
changed = True
if ((not info and (user or group)) or (user and info["user"] != user)
or (group and info["group"] != group)):
yield file_utils.chown(path, user, group, recursive=recursive)
changed = True
if user:
info["user"] = user
if group:
info["group"] = group
if not changed:
host.noop("directory {0} already exists".format(path))
def file(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=True,
force=False,
force_backup=True,
force_backup_dir=None,
):
"""
Add/remove/update files.
+ path: name/path of the remote file
+ present: whether the file should exist
+ assume_present: whether to assume the file exists
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: if the target exists and is not a file, move or remove it and continue
+ force_backup: set to ``False`` to remove any existing non-file when ``force=True``
+ force_backup_dir: directory to move any backup to when ``force=True``
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
**Example:**
.. code:: python
# Note: The directory /tmp/secret will get created with the default umask.
files.file(
name="Create /tmp/secret/file",
path="/tmp/secret/file",
mode="600",
user="******",
group="root",
touch=True,
create_remote_dir=True,
)
"""
path = _validate_path(path)
mode = ensure_mode_int(mode)
info = host.get_fact(File, path=path)
# Not a file?!
if info is False:
yield from _raise_or_remove_invalid_path(
"file",
path,
force,
force_backup,
force_backup_dir,
)
info = None
# Doesn't exist & we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield from _create_remote_dir(state, host, path, user, group)
yield StringCommand("touch", QuoteString(path))
if mode:
yield file_utils.chmod(path, mode)
if user or group:
yield file_utils.chown(path, user, group)
host.create_fact(
File,
kwargs={"path": path},
data={
"mode": mode,
"group": group,
"user": user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield StringCommand("rm", "-f", QuoteString(path))
host.delete_fact(File, kwargs={"path": path})
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {"mode": None, "group": None, "user": None}
host.create_fact(File, kwargs={"path": path}, data=info)
changed = False
if touch:
changed = True
yield StringCommand("touch", QuoteString(path))
# Check mode
if mode and (not info or info["mode"] != mode):
yield file_utils.chmod(path, mode)
info["mode"] = mode
changed = True
# Check user/group
if ((not info and (user or group)) or (user and info["user"] != user)
or (group and info["group"] != group)):
yield file_utils.chown(path, user, group)
changed = True
if user:
info["user"] = user
if group:
info["group"] = group
if not changed:
host.noop("file {0} already exists".format(path))
def link(
path,
target=None,
present=True,
assume_present=False,
user=None,
group=None,
symbolic=True,
create_remote_dir=True,
force=False,
force_backup=True,
force_backup_dir=None,
):
"""
Add/remove/update links.
+ path: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ assume_present: whether to assume the link exists
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: if the target exists and is not a file, move or remove it and continue
+ force_backup: set to ``False`` to remove any existing non-file when ``force=True``
+ force_backup_dir: directory to move any backup to when ``force=True``
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
**Examples:**
.. code:: python
files.link(
name="Create link /etc/issue2 that points to /etc/issue",
path="/etc/issue2",
target="/etc/issue",
)
# Complex example demonstrating the assume_present option
from pyinfra.operations import apt, files
install_nginx = apt.packages(
name="Install nginx",
packages=["nginx"],
)
files.link(
name="Remove default nginx site",
path="/etc/nginx/sites-enabled/default",
present=False,
assume_present=install_nginx.changed,
)
"""
path = _validate_path(path)
if present and not target:
raise OperationError("If present is True target must be provided")
info = host.get_fact(Link, path=path)
# Not a link?
if info is False:
yield from _raise_or_remove_invalid_path(
"link",
path,
force,
force_backup,
force_backup_dir,
)
info = None
add_args = ["ln"]
if symbolic:
add_args.append("-s")
add_cmd = StringCommand(" ".join(add_args), QuoteString(target),
QuoteString(path))
remove_cmd = StringCommand("rm", "-f", QuoteString(path))
# No link and we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield from _create_remote_dir(state, host, path, user, group)
yield add_cmd
if user or group:
yield file_utils.chown(path, user, group, dereference=False)
host.create_fact(
Link,
kwargs={"path": path},
data={
"link_target": target,
"group": group,
"user": user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield remove_cmd
host.delete_fact(Link, kwargs={"path": path})
# Exists and want to ensure it's state
elif (assume_present or info) and present:
if assume_present and not info:
info = {"link_target": None, "group": None, "user": None}
host.create_fact(Link, kwargs={"path": path}, data=info)
# If we have an absolute path - prepend to any non-absolute values from the fact
# and/or the source.
if os.path.isabs(path):
link_dirname = os.path.dirname(path)
if not os.path.isabs(target):
target = os.path.normpath(unix_path_join(link_dirname, target))
if info and not os.path.isabs(info["link_target"]):
info["link_target"] = os.path.normpath(
unix_path_join(link_dirname, info["link_target"]), )
changed = False
# If the target is wrong, remove & recreate the link
if not info or info["link_target"] != target:
changed = True
yield remove_cmd
yield add_cmd
info["link_target"] = target
# Check user/group
if ((not info and (user or group)) or (user and info["user"] != user)
or (group and info["group"] != group)):
yield file_utils.chown(path, user, group, dereference=False)
changed = True
if user:
info["user"] = user
if group:
info["group"] = group
if not changed:
host.noop("link {0} already exists".format(path))
def repo(
src,
dest,
branch=None,
pull=True,
rebase=False,
user=None,
group=None,
ssh_keyscan=False,
update_submodules=False,
recursive_submodules=False,
):
"""
Clone/pull git repositories.
+ src: the git source URL
+ dest: directory to clone to
+ branch: branch to pull/checkout
+ pull: pull any changes for the branch
+ rebase: when pulling, use ``--rebase``
+ user: chown files to this user after
+ group: chown files to this group after
+ ssh_keyscan: keyscan the remote host if not in known_hosts before clone/pull
+ update_submodules: update any git submodules
+ recursive_submodules: update git submodules recursively
**Example:**
.. code:: python
git.repo(
name="Clone repo",
src="https://github.com/Fizzadar/pyinfra.git",
dest="/usr/local/src/pyinfra",
)
"""
# Ensure our target directory exists
yield from files.directory(dest)
# Do we need to scan for the remote host key?
if ssh_keyscan:
# Attempt to parse the domain from the git repository
domain = re.match(r"^[a-zA-Z0-9]+@([0-9a-zA-Z\.\-]+)", src)
if domain:
yield from ssh.keyscan(domain.group(1))
else:
raise OperationError(
"Could not parse domain (to SSH keyscan) from: {0}".format(
src), )
# Store git commands for directory prefix
git_commands = []
git_dir = unix_path_join(dest, ".git")
is_repo = host.get_fact(Directory, path=git_dir)
# Cloning new repo?
if not is_repo:
if branch:
git_commands.append("clone {0} --branch {1} .".format(src, branch))
else:
git_commands.append("clone {0} .".format(src))
host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
host.create_fact(
Directory,
kwargs={"path": git_dir},
data={
"user": user,
"group": group
},
)
# Ensuring existing repo
else:
if branch and host.get_fact(GitBranch, repo=dest) != branch:
git_commands.append(
"fetch") # fetch to ensure we have the branch locally
git_commands.append("checkout {0}".format(branch))
host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
if pull:
if rebase:
git_commands.append("pull --rebase")
else:
git_commands.append("pull")
if update_submodules:
if recursive_submodules:
git_commands.append("submodule update --init --recursive")
else:
git_commands.append("submodule update --init")
# Attach prefixes for directory
command_prefix = "cd {0} && git".format(dest)
git_commands = [
"{0} {1}".format(command_prefix, command) for command in git_commands
]
for cmd in git_commands:
yield cmd
# Apply any user or group if we did anything
if git_commands and (user or group):
yield chown(dest, user, group, recursive=True)
def worktree(
worktree,
repo=None,
detached=False,
new_branch=None,
commitish=None,
pull=True,
rebase=False,
from_remote_branch=None,
present=True,
assume_repo_exists=False,
force=False,
user=None,
group=None,
):
"""
Manage git worktrees.
+ worktree: git working tree directory
+ repo: git main repository directory
+ detached: create a working tree with a detached HEAD
+ new_branch: local branch name created at the same time than the worktree
+ commitish: from which git commit, branch, ... the worktree is created
+ pull: pull any changes from a remote branch if set
+ rebase: when pulling, use ``--rebase``
+ from_remote_branch: a 2-tuple ``(remote, branch)`` that identifies a remote branch
+ present: whether the working tree should exist
+ assume_repo_exists: whether to assume the main repo exists
+ force: remove unclean working tree if should not exist
+ user: chown files to this user after
+ group: chown files to this group after
**Examples:**
.. code:: python
git.worktree(
name="Create a worktree from the current repo `HEAD`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix"
)
git.worktree(
name="Create a worktree from the commit `4e091aa0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="4e091aa0"
)
git.worktree(
name="Create a worktree with a new local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
)
git.worktree(
name="Create a worktree from the commit 4e091aa0 with the new local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
commitish="4e091aa0"
)
git.worktree(
name="Create a worktree with a detached `HEAD`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
detached=True,
)
git.worktree(
name="Create a worktree with a detached `HEAD` from commit `4e091aa0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="4e091aa0",
detached=True,
)
git.worktree(
name="Create a worktree from the existing local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="v1.0"
)
git.worktree(
name="Create a worktree with a new branch `v1.0` that tracks `origin/v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
commitish="v1.0"
)
git.worktree(
name="Pull an existing worktree already linked to a tracking branch",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix"
)
git.worktree(
name="Pull an existing worktree from a specific remote branch",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
from_remote_branch=("origin", "master")
)
git.worktree(
name="Remove a worktree",
worktree="/usr/local/src/pyinfra/hotfix",
present=False,
)
git.worktree(
name="Remove an unclean worktree",
worktree="/usr/local/src/pyinfra/hotfix",
present=False,
force=True,
)
"""
# Doesn't exist & we want it
if not host.get_fact(Directory, path=worktree) and present:
# be sure that `repo` is a GIT repository
if not assume_repo_exists and not host.get_fact(
Directory,
path=unix_path_join(repo, ".git"),
):
raise OperationError(
"The following folder is not a valid GIT repository : {0}".
format(repo), )
command_parts = ["cd {0} && git worktree add".format(repo)]
if new_branch:
command_parts.append("-b {0}".format(new_branch))
elif detached:
command_parts.append("--detach")
command_parts.append(worktree)
if commitish:
command_parts.append(commitish)
yield " ".join(command_parts)
# Apply any user or group
if user or group:
yield chown(worktree, user, group, recursive=True)
host.create_fact(Directory,
kwargs={"path": worktree},
data={
"user": user,
"group": group
})
host.create_fact(GitTrackingBranch,
kwargs={"repo": worktree},
data=new_branch)
# It exists and we don't want it
elif host.get_fact(Directory, path=worktree) and not present:
command = "cd {0} && git worktree remove .".format(worktree)
if force:
command += " --force"
yield command
host.delete_fact(Directory, kwargs={"path": worktree})
host.create_fact(GitTrackingBranch, kwargs={"repo": worktree})
# It exists and we still want it => pull/rebase it
elif host.get_fact(Directory, path=worktree) and present:
# pull the worktree only if it's already linked to a tracking branch or
# if a remote branch is set
if host.get_fact(GitTrackingBranch,
repo=worktree) or from_remote_branch:
command = "cd {0} && git pull".format(worktree)
if rebase:
command += " --rebase"
if from_remote_branch:
if len(from_remote_branch) != 2 or type(
from_remote_branch) not in (tuple, list):
raise OperationError(
"The remote branch must be a 2-tuple (remote, branch) such as "
'("origin", "master")', )
command += " {0} {1}".format(*from_remote_branch)
yield command
def virtualenv(
path,
python=None,
venv=False,
site_packages=False,
always_copy=False,
present=True,
):
"""
Add/remove Python virtualenvs.
+ python: python interpreter to use
+ venv: use standard venv module instead of virtualenv
+ site_packages: give access to the global site-packages
+ always_copy: always copy files rather than symlinking
+ present: whether the virtualenv should exist
**Example:**
.. code:: python
pip.virtualenv(
name="Create a virtualenv",
path="/usr/local/bin/venv",
)
"""
# Check for *contents* of a virtualenv, ie don't accept an empty directory
# as a valid virtualenv but ensure the activate script exists.
activate_script_path = "{0}/bin/activate".format(path)
if present is False:
if host.get_fact(File, path=activate_script_path):
yield from files.directory(path, present=False)
else:
host.noop("virtualenv {0} does not exist".format(path))
if present:
if not host.get_fact(File, path=activate_script_path):
# Create missing virtualenv
command = ["virtualenv"]
if venv:
command = [python or "python", "-m", "venv"]
if python and not venv:
command.append("-p {0}".format(python))
if site_packages:
command.append("--system-site-packages")
if always_copy and not venv:
command.append("--always-copy")
elif always_copy and venv:
command.append("--copies")
command.append(path)
yield " ".join(command)
host.create_fact(
File,
kwargs={"path": activate_script_path},
data={
"user": None,
"group": None
},
)
else:
host.noop("virtualenv {0} exists".format(path))
def download(
hostname,
filename,
local_filename=None,
force=False,
port=22,
user=None,
ssh_keyscan=False,
):
"""
Download files from other servers using ``scp``.
+ hostname: hostname to upload to
+ filename: file to download
+ local_filename: where to download the file to (defaults to ``filename``)
+ force: always download the file, even if present locally
+ port: connect to this port
+ user: connect with this user
+ ssh_keyscan: execute ``ssh.keyscan`` before uploading the file
"""
local_filename = local_filename or filename
# Get local file info
local_file_info = host.get_fact(File, path=local_filename)
# Local file exists but isn't a file?
if local_file_info is False:
raise OperationError(
"Local destination {0} already exists and is not a file".format(
local_filename, ), )
# If the local file exists and we're not forcing a re-download, no-op
if local_file_info and not force:
host.noop("file {0} is already downloaded".format(filename))
return
# Figure out where we're connecting (host or user@host)
connection_target = hostname
if user:
connection_target = "@".join((user, hostname))
if ssh_keyscan:
yield from keyscan(hostname)
# Download the file with scp
yield "scp -P {0} {1}:{2} {3}".format(
port,
connection_target,
filename,
local_filename,
)
host.create_fact(
File,
kwargs={"path": local_filename},
data={
"mode": None,
"group": None,
"user": user,
"mtime": None
},
)
def directory(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
):
"""
Add/remove/update directories.
+ path: path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ TODO: user: user to own the folder
+ TODO: group: group to own the folder
+ TODO: mode: permissions of the folder
+ TODO: recursive: recursively apply user/group/mode
**Examples:**
.. code:: python
files.directory(
name="Ensure the c:\\temp\\dir_that_we_want_removed is removed",
path="c:\\temp\\dir_that_we_want_removed",
present=False,
)
files.directory(
name="Ensure c:\\temp\\foo\\foo_dir exists",
path="c:\\temp\\foo\\foo_dir",
recursive=True,
)
# multiple directories
dirs = ["c:\\temp\\foo_dir1", "c:\\temp\\foo_dir2"]
for dir in dirs:
files.directory(
name="Ensure the directory `{}` exists".format(dir),
path=dir,
)
"""
if not isinstance(path, str):
raise OperationTypeError("Name must be a string")
info = host.get_fact(Directory, path=path)
# Not a directory?!
if info is False:
raise OperationError("{0} exists and is not a directory".format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield "New-Item -Path {0} -ItemType Directory".format(path)
# if mode:
# yield chmod(path, mode, recursive=recursive)
# if user or group:
# yield chown(path, user, group, recursive=recursive)
#
# Somewhat bare fact, should flesh out more
host.create_fact(
Date,
kwargs={"path": path},
data={"type": "directory"},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
# TODO: how to ensure we use 'ps'?
# remove anything in the directory
yield "Get-ChildItem {0} -Recurse | Remove-Item".format(path)
# remove directory
yield "Remove-Item {0}".format(path)
