def command(self, path):
self.path = path
return make_formatted_string_command(
("test -e {0} && ( "
"sha1sum {0} 2> /dev/null || shasum {0} 2> /dev/null || sha1 {0} "
") || true"),
QuoteString(path),
)
def command(self, path):
return make_formatted_string_command(
(
# only stat if the path exists (file or symlink)
"! (test -e {0} || test -L {0} ) || "
"( {linux_stat_command} {0} 2> /dev/null || {bsd_stat_command} {0} )"
),
QuoteString(path),
linux_stat_command=LINUX_STAT_COMMAND,
bsd_stat_command=BSD_STAT_COMMAND,
)
def command(self, path, pattern, interpolate_variables=False):
self.exists_flag = "__pyinfra_exists_{0}".format(path)
if interpolate_variables:
pattern = '"{0}"'.format(pattern.replace('"', '\\"'))
else:
pattern = QuoteString(pattern)
return make_formatted_string_command(
("grep -e {0} {1} 2> /dev/null || "
"( find {1} -type f > /dev/null && echo {2} || true )"),
pattern,
QuoteString(path),
QuoteString(self.exists_flag),
)
def command(self, path, quote_path=True):
return make_formatted_string_command(
"find {0} -type {type_flag} || true",
QuoteString(path) if quote_path else path,
type_flag=self.type_flag,
)
def command(self, path):
self.path = path
return make_formatted_string_command(
"test -e {0} && ( md5sum {0} 2> /dev/null || md5 {0} ) || true",
QuoteString(path),
)
def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
headers=None,
insecure=False,
):
"""
Download files from remote locations using ``curl`` or ``wget``.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
+ headers: optional dictionary of headers to set for the HTTP request
+ insecure: disable SSL verification for the HTTP request
**Example:**
.. code:: python
files.download(
name="Download the Docker repo file",
src="https://download.docker.com/linux/centos/docker-ce.repo",
dest="/etc/yum.repos.d/docker-ce.repo",
)
"""
info = host.get_fact(File, path=dest)
host_datetime = host.get_fact(Date).replace(tzinfo=None)
# Destination is a directory?
if info is False:
raise OperationError(
"Destination {0} already exists and is not a file".format(dest), )
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from the Date fact before comparison.
cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(
seconds=cache_time)
if info["mtime"] and info["mtime"] < cache_time:
download = True
if sha1sum:
if sha1sum != host.get_fact(Sha1File, path=dest):
download = True
if sha256sum:
if sha256sum != host.get_fact(Sha256File, path=dest):
download = True
if md5sum:
if md5sum != host.get_fact(Md5File, path=dest):
download = True
# If we download, always do user/group/mode as SSH user may be different
if download:
temp_file = state.get_temp_filename(dest)
curl_args = ["-sSLf"]
wget_args = ["-q"]
if insecure:
curl_args.append("--insecure")
wget_args.append("--no-check-certificate")
if headers:
for key, value in headers.items():
header_arg = StringCommand("--header",
QuoteString(f"{key}: {value}"))
curl_args.append(header_arg)
wget_args.append(header_arg)
curl_command = make_formatted_string_command(
"curl {0} {1} -o {2}",
StringCommand(*curl_args),
QuoteString(src),
QuoteString(temp_file),
)
wget_command = make_formatted_string_command(
"wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )",
StringCommand(*wget_args),
QuoteString(src),
QuoteString(temp_file),
)
if host.get_fact(Which, command="curl"):
yield curl_command
elif host.get_fact(Which, command="wget"):
yield wget_command
else:
yield "( {0} ) || ( {1} )".format(curl_command, wget_command)
yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest))
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
if sha1sum:
yield make_formatted_string_command(
("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
sha1sum,
QuoteString("SHA1 did not match!"),
)
if sha256sum:
yield make_formatted_string_command(
("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) "
"| grep {1}) || ( echo {2} && exit 1 )"),
QuoteString(dest),
sha256sum,
QuoteString("SHA256 did not match!"),
)
if md5sum:
yield make_formatted_string_command(
("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
md5sum,
QuoteString("MD5 did not match!"),
)
host.create_fact(
File,
kwargs={"path": dest},
data={
"mode": mode,
"group": group,
"user": user,
"mtime": host_datetime
},
)
# Remove any checksum facts as we don't know the correct values
for value, fact_cls in (
(sha1sum, Sha1File),
(sha256sum, Sha256File),
(md5sum, Md5File),
):
fact_kwargs = {"path": dest}
if value:
host.create_fact(fact_cls, kwargs=fact_kwargs, data=value)
else:
host.delete_fact(fact_cls, kwargs=fact_kwargs)
else:
host.noop("file {0} has already been downloaded".format(dest))
def line(
path,
line,
present=True,
replace=None,
flags=None,
backup=False,
interpolate_variables=False,
escape_regex_characters=False,
assume_present=False,
):
"""
Ensure lines in files using grep to locate and sed to replace.
+ path: target remote file to edit
+ line: string or regex matching the target line
+ present: whether the line should be in the file
+ replace: text to replace entire matching lines when ``present=True``
+ flags: list of flags to pass to sed when replacing/deleting
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
+ assume_present: whether to assume a matching line already exists in the file
+ escape_regex_characters: whether to escape regex characters from the matching line
Regex line matching:
Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that
it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To
change bits of lines, see ``files.replace``.
Regex line escaping:
If matching special characters (eg a crontab line containing *), remember to escape
it first using Python's ``re.escape``.
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
Append:
If ``line`` is not in the file but we want it (``present`` set to ``True``), then
it will be append to the end of the file.
**Examples:**
.. code:: python
# prepare to do some maintenance
maintenance_line = "SYSTEM IS DOWN FOR MAINTENANCE"
files.line(
name="Add the down-for-maintence line in /etc/motd",
path="/etc/motd",
line=maintenance_line,
)
# Then, after the mantenance is done, remove the maintenance line
files.line(
name="Remove the down-for-maintenance line in /etc/motd",
path="/etc/motd",
line=maintenance_line,
replace="",
present=False,
)
# example where there is '*' in the line
files.line(
name="Ensure /netboot/nfs is in /etc/exports",
path="/etc/exports",
line=r"/netboot/nfs .*",
replace="/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash,"
"insecure,no_subtree_check)",
)
files.line(
name="Ensure myweb can run /usr/bin/python3 without password",
path="/etc/sudoers",
line=r"myweb .*",
replace="myweb ALL=(ALL) NOPASSWD: /usr/bin/python3",
)
# example when there are double quotes (")
line = 'QUOTAUSER=""'
files.line(
name="Example with double quotes (")",
path="/etc/adduser.conf",
line="^{}$".format(line),
replace=line,
)
"""
match_line = line
if escape_regex_characters:
match_line = re.sub(r"([\.\\\+\*\?\[\^\]\$\(\)\{\}\-])", r"\\\1",
match_line)
# Ensure we're matching a whole line, note: match may be a partial line so we
# put any matches on either side.
if not match_line.startswith("^"):
match_line = "^.*{0}".format(match_line)
if not match_line.endswith("$"):
match_line = "{0}.*$".format(match_line)
# Is there a matching line in this file?
if assume_present:
present_lines = [line]
else:
present_lines = host.get_fact(
FindInFile,
path=path,
pattern=match_line,
interpolate_variables=interpolate_variables,
)
# If replace present, use that over the matching line
if replace:
line = replace
# We must provide some kind of replace to sed_replace_command below
else:
replace = ""
# Save commands for re-use in dynamic script when file not present at fact stage
echo_command = make_formatted_string_command(
"echo {0} >> {1}",
'"{0}"'.format(line) if interpolate_variables else QuoteString(line),
QuoteString(path),
)
if backup:
backup_filename = "{0}.{1}".format(path, get_timestamp())
echo_command = StringCommand(
make_formatted_string_command(
"cp {0} {1} && ",
QuoteString(path),
QuoteString(backup_filename),
),
echo_command,
)
sed_replace_command = sed_replace(
path,
match_line,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
# No line and we want it, append it
if not present_lines and present:
# If the file does not exist - it *might* be created, so we handle it
# dynamically with a little script.
if present_lines is None:
yield make_formatted_string_command(
"""
if [ -f '{target}' ]; then
( grep {match_line} '{target}' && \
{sed_replace_command}) 2> /dev/null || \
{echo_command} ;
else
{echo_command} ;
fi
""",
target=QuoteString(path),
match_line=QuoteString(match_line),
echo_command=echo_command,
sed_replace_command=sed_replace_command,
)
# File exists but has no matching lines - append it.
else:
# If we're doing replacement, only append if the *replacement* line
# does not exist (as we are appending the replacement).
if replace:
# Ensure replace explicitly matches a whole line
replace_line = replace
if not replace_line.startswith("^"):
replace_line = f"^{replace_line}"
if not replace_line.endswith("$"):
replace_line = f"{replace_line}$"
present_lines = host.get_fact(
FindInFile,
path=path,
pattern=replace_line,
interpolate_variables=interpolate_variables,
)
if not present_lines:
yield echo_command
else:
host.noop('line "{0}" exists in {1}'.format(
replace or line, path))
host.create_fact(
FindInFile,
kwargs={
"path": path,
"pattern": match_line,
"interpolate_variables": interpolate_variables,
},
data=[replace or line],
)
# Line(s) exists and we want to remove them, replace with nothing
elif present_lines and not present:
yield sed_replace(
path,
match_line,
"",
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
host.delete_fact(
FindInFile,
kwargs={
"path": path,
"pattern": match_line,
"interpolate_variables": interpolate_variables,
},
)
# Line(s) exists and we have want to ensure they're correct
elif present_lines and present:
# If any of lines are different, sed replace them
if replace and any(line != replace for line in present_lines):
yield sed_replace_command
del present_lines[:] # TODO: use .clear() when py3+
present_lines.append(replace)
else:
host.noop('line "{0}" exists in {1}'.format(replace or line, path))