def get(
src,
dest,
add_deploy_dir=True,
create_local_dir=False,
force=False,
):
"""
Download a file from the remote system.
+ src: the remote filename to download
+ dest: the local filename to download the file to
+ add_deploy_dir: dest is relative to the deploy directory
+ create_local_dir: create the local directory if it doesn't exist
+ force: always download the file, even if the local copy matches
Note:
This operation is not suitable for large files as it may involve copying
the remote file before downloading it.
**Example:**
.. code:: python
files.get(
name="Download a file from a remote",
src="/etc/centos-release",
dest="/tmp/whocares",
)
"""
if add_deploy_dir and state.cwd:
dest = os.path.join(state.cwd, dest)
if create_local_dir:
local_pathname = os.path.dirname(dest)
if not os.path.exists(local_pathname):
os.makedirs(local_pathname)
remote_file = host.get_fact(File, path=src)
# No remote file, so assume exists and download it "blind"
if not remote_file or force:
yield FileDownloadCommand(
src, dest, remote_temp_filename=state.get_temp_filename(dest))
# No local file, so always download
elif not os.path.exists(dest):
yield FileDownloadCommand(
src, dest, remote_temp_filename=state.get_temp_filename(dest))
# Remote file exists - check if it matches our local
else:
local_sum = get_file_sha1(dest)
remote_sum = host.get_fact(Sha1File, path=src)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileDownloadCommand(
src, dest, remote_temp_filename=state.get_temp_filename(dest))
def script_template(src, **data):
"""
Generate, upload and execute a local script template on the remote host.
+ src: local script template filename
**Example:**
.. code:: python
# Example showing how to pass python variable to a script template file.
# The .j2 file can use `{{ some_var }}` to be interpolated.
# To see output need to run pyinfra with '-v'
# Note: This assumes there is a file in templates/hello2.bash.j2 locally.
some_var = 'blah blah blah '
server.script_template(
name="Hello from script",
src="templates/hello2.bash.j2",
some_var=some_var,
)
"""
temp_file = state.get_temp_filename("{0}{1}".format(src, data))
yield from files.template(src, temp_file, **data)
yield chmod(temp_file, "+x")
yield temp_file
def script(src):
"""
Upload and execute a local script on the remote host.
+ src: local script filename to upload & execute
**Example:**
.. code:: python
# Note: This assumes there is a file in files/hello.bash locally.
server.script(
name="Hello",
src="files/hello.bash",
)
"""
temp_file = state.get_temp_filename(src)
yield from files.put(src, temp_file)
yield chmod(temp_file, "+x")
yield temp_file
password='******',
)
mysql.database(
{'Create the pyinfra_stuff database'},
'pyinfra_stuff',
user='******',
user_privileges=['SELECT', 'INSERT'],
charset='utf8',
)
# Upload & import a SQL file into the pyinfra_stuff database
#
filename = 'files/a_db.sql'
temp_filename = state.get_temp_filename(filename)
files.put(
{'Upload the a_db.sql file'},
filename,
temp_filename,
)
mysql.load(
{'Import the a_db.sql file'},
temp_filename,
database='pyinfra_stuff',
)
# Now duplicate the pyinfra_stuff database -> pyinfra_stuff_copy
#
def crontab(
command,
present=True,
user=None,
cron_name=None,
minute="*",
hour="*",
month="*",
day_of_week="*",
day_of_month="*",
special_time=None,
interpolate_variables=False,
):
"""
Add/remove/update crontab entries.
+ command: the command for the cron
+ present: whether this cron command should exist
+ user: the user whose crontab to manage
+ cron_name: name the cronjob so future changes to the command will overwrite
+ minute: which minutes to execute the cron
+ hour: which hours to execute the cron
+ month: which months to execute the cron
+ day_of_week: which day of the week to execute the cron
+ day_of_month: which day of the month to execute the cron
+ special_time: cron "nickname" time (@reboot, @daily, etc), overrides others
+ interpolate_variables: whether to interpolate variables in ``command``
Cron commands:
Unless ``name`` is specified the command is used to identify crontab entries.
This means commands must be unique within a given users crontab. If you require
multiple identical commands, provide a different name argument for each.
Special times:
When provided, ``special_time`` will be used instead of any values passed in
for ``minute``/``hour``/``month``/``day_of_week``/``day_of_month``.
**Example:**
.. code:: python
# simple example for a crontab
server.crontab(
name="Backup /etc weekly",
command="/bin/tar cf /tmp/etc_bup.tar /etc",
name="backup_etc",
day_of_week=0,
hour=1,
minute=0,
)
"""
def comma_sep(value):
if isinstance(value, (list, tuple)):
return ",".join("{0}".format(v) for v in value)
return value
minute = comma_sep(minute)
hour = comma_sep(hour)
month = comma_sep(month)
day_of_week = comma_sep(day_of_week)
day_of_month = comma_sep(day_of_month)
crontab = host.get_fact(Crontab, user=user)
name_comment = "# pyinfra-name={0}".format(cron_name)
existing_crontab = crontab.get(command)
existing_crontab_command = command
existing_crontab_match = command
if not existing_crontab and cron_name: # find the crontab by name if provided
for cmd, details in crontab.items():
if name_comment in details["comments"]:
existing_crontab = details
existing_crontab_match = cmd
existing_crontab_command = cmd
exists = existing_crontab is not None
edit_commands = []
temp_filename = state.get_temp_filename()
if special_time:
new_crontab_line = "{0} {1}".format(special_time, command)
else:
new_crontab_line = "{minute} {hour} {day_of_month} {month} {day_of_week} {command}".format(
minute=minute,
hour=hour,
day_of_month=day_of_month,
month=month,
day_of_week=day_of_week,
command=command,
)
existing_crontab_match = ".*{0}.*".format(existing_crontab_match)
# Don't want the cron and it does exist? Remove the line
if not present and exists:
edit_commands.append(
sed_replace(
temp_filename,
existing_crontab_match,
"",
interpolate_variables=interpolate_variables,
), )
# Want the cron but it doesn't exist? Append the line
elif present and not exists:
if cron_name:
if crontab: # append a blank line if cron entries already exist
edit_commands.append("echo '' >> {0}".format(temp_filename))
edit_commands.append(
"echo {0} >> {1}".format(
shlex.quote(name_comment),
temp_filename,
), )
edit_commands.append(
"echo {0} >> {1}".format(
shlex.quote(new_crontab_line),
temp_filename,
), )
# We have the cron and it exists, do it's details? If not, replace the line
elif present and exists:
if any((
special_time != existing_crontab.get("special_time"),
minute != existing_crontab.get("minute"),
hour != existing_crontab.get("hour"),
month != existing_crontab.get("month"),
day_of_week != existing_crontab.get("day_of_week"),
day_of_month != existing_crontab.get("day_of_month"),
existing_crontab_command != command,
), ):
edit_commands.append(
sed_replace(
temp_filename,
existing_crontab_match,
new_crontab_line,
interpolate_variables=interpolate_variables,
), )
if edit_commands:
crontab_args = []
if user:
crontab_args.append("-u {0}".format(user))
# List the crontab into a temporary file if it exists
if crontab:
yield "crontab -l {0} > {1}".format(" ".join(crontab_args),
temp_filename)
# Now yield any edits
for edit_command in edit_commands:
yield edit_command
# Finally, use the tempfile to write a new crontab
yield "crontab {0} {1}".format(" ".join(crontab_args), temp_filename)
# Update the crontab fact
if present:
crontab[command] = {
"special_time": special_time,
"minute": minute,
"hour": hour,
"month": month,
"day_of_week": day_of_week,
"day_of_month": day_of_month,
"comments": [cron_name] if cron_name else [],
}
else:
crontab.pop(command)
else:
host.noop(
"crontab {0} {1}".format(
command,
"exists" if present else "does not exist",
), )
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
):
"""
Upload a local file, or file-like object, to the remote system.
+ src: filename or IO-like object to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files, use ``True`` to copy the local file
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``dest``:
If this is a directory that already exists on the remote side, the local
file will be uploaded to that directory with the same filename.
``mode``:
When set to ``True`` the permissions of the local file are applied to the
remote file after the upload is complete.
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
**Examples:**
.. code:: python
files.put(
name="Update the message of the day file",
src="files/motd",
dest="/etc/motd",
mode="644",
)
files.put(
name="Upload a StringIO object",
src=StringIO("file contents"),
dest="/etc/motd",
)
"""
# Upload IO objects as-is
if hasattr(src, "read"):
local_file = src
local_sum = get_file_sha1(src)
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.cwd:
src = os.path.join(state.cwd, src)
local_file = src
if os.path.isfile(local_file):
local_sum = get_file_sha1(local_file)
elif assume_exists:
local_sum = None
else:
raise IOError("No such file: {0}".format(local_file))
if mode is True:
if os.path.isfile(local_file):
mode = get_path_permissions_mode(local_file)
else:
logger.warning((
"No local file exists to get permissions from with `mode=True` ({0})"
).format(get_call_location(), ), )
else:
mode = ensure_mode_int(mode)
remote_file = host.get_fact(File, path=dest)
if not remote_file and host.get_fact(Directory, path=dest):
dest = unix_path_join(dest, os.path.basename(src))
remote_file = host.get_fact(File, path=dest)
if create_remote_dir:
yield from _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
remote_sum = host.get_fact(Sha1File, path=dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
else:
changed = False
# Check mode
if mode and remote_file["mode"] != mode:
yield file_utils.chmod(dest, mode)
changed = True
# Check user/group
if (user and remote_file["user"] != user) or (
group and remote_file["group"] != group):
yield file_utils.chown(dest, user, group)
changed = True
if not changed:
host.noop("file {0} is already uploaded".format(dest))
# Now we've uploaded the file and ensured user/group/mode, update the relevant fact data
host.create_fact(Sha1File, kwargs={"path": dest}, data=local_sum)
host.create_fact(
File,
kwargs={"path": dest},
data={
"user": user,
"group": group,
"mode": mode
},
)
def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
headers=None,
insecure=False,
):
"""
Download files from remote locations using ``curl`` or ``wget``.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
+ headers: optional dictionary of headers to set for the HTTP request
+ insecure: disable SSL verification for the HTTP request
**Example:**
.. code:: python
files.download(
name="Download the Docker repo file",
src="https://download.docker.com/linux/centos/docker-ce.repo",
dest="/etc/yum.repos.d/docker-ce.repo",
)
"""
info = host.get_fact(File, path=dest)
host_datetime = host.get_fact(Date).replace(tzinfo=None)
# Destination is a directory?
if info is False:
raise OperationError(
"Destination {0} already exists and is not a file".format(dest), )
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from the Date fact before comparison.
cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(
seconds=cache_time)
if info["mtime"] and info["mtime"] < cache_time:
download = True
if sha1sum:
if sha1sum != host.get_fact(Sha1File, path=dest):
download = True
if sha256sum:
if sha256sum != host.get_fact(Sha256File, path=dest):
download = True
if md5sum:
if md5sum != host.get_fact(Md5File, path=dest):
download = True
# If we download, always do user/group/mode as SSH user may be different
if download:
temp_file = state.get_temp_filename(dest)
curl_args = ["-sSLf"]
wget_args = ["-q"]
if insecure:
curl_args.append("--insecure")
wget_args.append("--no-check-certificate")
if headers:
for key, value in headers.items():
header_arg = StringCommand("--header",
QuoteString(f"{key}: {value}"))
curl_args.append(header_arg)
wget_args.append(header_arg)
curl_command = make_formatted_string_command(
"curl {0} {1} -o {2}",
StringCommand(*curl_args),
QuoteString(src),
QuoteString(temp_file),
)
wget_command = make_formatted_string_command(
"wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )",
StringCommand(*wget_args),
QuoteString(src),
QuoteString(temp_file),
)
if host.get_fact(Which, command="curl"):
yield curl_command
elif host.get_fact(Which, command="wget"):
yield wget_command
else:
yield "( {0} ) || ( {1} )".format(curl_command, wget_command)
yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest))
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
if sha1sum:
yield make_formatted_string_command(
("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
sha1sum,
QuoteString("SHA1 did not match!"),
)
if sha256sum:
yield make_formatted_string_command(
("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) "
"| grep {1}) || ( echo {2} && exit 1 )"),
QuoteString(dest),
sha256sum,
QuoteString("SHA256 did not match!"),
)
if md5sum:
yield make_formatted_string_command(
("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) "
"|| ( echo {2} && exit 1 )"),
QuoteString(dest),
md5sum,
QuoteString("MD5 did not match!"),
)
host.create_fact(
File,
kwargs={"path": dest},
data={
"mode": mode,
"group": group,
"user": user,
"mtime": host_datetime
},
)
# Remove any checksum facts as we don't know the correct values
for value, fact_cls in (
(sha1sum, Sha1File),
(sha256sum, Sha256File),
(md5sum, Md5File),
):
fact_kwargs = {"path": dest}
if value:
host.create_fact(fact_cls, kwargs=fact_kwargs, data=value)
else:
host.delete_fact(fact_cls, kwargs=fact_kwargs)
else:
host.noop("file {0} has already been downloaded".format(dest))
)
mysql.database(
name="Create the pyinfra_stuff database",
database="pyinfra_stuff",
user="******",
user_privileges=["SELECT", "INSERT"],
charset="utf8",
)
# Upload & import a SQL file into the pyinfra_stuff database
#
filename = "files/a_db.sql"
temp_filename = state.get_temp_filename(filename)
files.put(
name="Upload the a_db.sql file",
src=filename,
dest=temp_filename,
)
mysql.load(
name="Import the a_db.sql file",
src=temp_filename,
database="pyinfra_stuff",
)
# Now duplicate the pyinfra_stuff database -> pyinfra_stuff_copy
def upload(
hostname,
filename,
remote_filename=None,
port=22,
user=None,
use_remote_sudo=False,
ssh_keyscan=False,
):
"""
Upload files to other servers using ``scp``.
+ hostname: hostname to upload to
+ filename: file to upload
+ remote_filename: where to upload the file to (defaults to ``filename``)
+ port: connect to this port
+ user: connect with this user
+ use_remote_sudo: upload to a temporary location and move using sudo
+ ssh_keyscan: execute ``ssh.keyscan`` before uploading the file
"""
remote_filename = remote_filename or filename
# Figure out where we're connecting (host or user@host)
connection_target = hostname
if user:
connection_target = "@".join((user, hostname))
if ssh_keyscan:
yield from keyscan(hostname)
# If we're not using sudo on the remote side, just scp the file over
if not use_remote_sudo:
yield "scp -P {0} {1} {2}:{3}".format(
port,
filename,
connection_target,
remote_filename,
)
else:
# Otherwise - we need a temporary location for the file
temp_remote_filename = state.get_temp_filename()
# scp it to the temporary location
upload_cmd = "scp -P {0} {1} {2}:{3}".format(
port,
filename,
connection_target,
temp_remote_filename,
)
yield upload_cmd
# And sudo sudo to move it
yield from command(
hostname=hostname,
command="sudo mv {0} {1}".format(temp_remote_filename,
remote_filename),
port=port,
user=user,
)
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
):
"""
Upload a local file to the remote system.
+ src: local filename to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
**Examples:**
.. code:: python
# Note: This requires a 'files/motd' file on the local filesystem
files.put(
name="Update the message of the day file",
src="data/content.json",
dest="C:\\data\\content.json",
)
"""
# Upload IO objects as-is
if hasattr(src, "read"):
local_file = src
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.cwd:
src = os.path.join(state.cwd, src)
local_file = src
if not assume_exists and not os.path.isfile(local_file):
raise IOError("No such file: {0}".format(local_file))
mode = ensure_mode_int(mode)
remote_file = host.get_fact(File, path=dest)
if create_remote_dir:
yield from _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
# if user or group:
# yield chown(dest, user, group)
# if mode:
# yield chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
local_sum = get_file_sha1(src)
remote_sum = host.get_fact(Sha1File, path=dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
# if user or group:
# yield chown(dest, user, group)
# if mode:
# yield chmod(dest, mode)
else:
changed = False
# Check mode
# if mode and remote_file['mode'] != mode:
# yield chmod(dest, mode)
# changed = True
# Check user/group
# if (
# (user and remote_file['user'] != user)
# or (group and remote_file['group'] != group)
# ):
# yield chown(dest, user, group)
# changed = True
if not changed:
host.noop("file {0} is already uploaded".format(dest))
def deb(src, present=True, force=False):
"""
Add/remove ``.deb`` file packages.
+ src: filename or URL of the ``.deb`` file
+ present: whether or not the package should exist on the system
+ force: whether to force the package install by passing `--force-yes` to apt
Note:
When installing, ``apt-get install -f`` will be run to install any unmet
dependencies.
URL sources with ``present=False``:
If the ``.deb`` file isn't downloaded, pyinfra can't remove any existing
package as the file won't exist until mid-deploy.
**Example:**
.. code:: python
# Note: Assumes wget is installed.
apt.deb(
name="Install Chrome via deb",
src="https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb",
)
"""
original_src = src
# If source is a url
if urlparse(src).scheme:
# Generate a temp filename
temp_filename = state.get_temp_filename(src)
# Ensure it's downloaded
yield from files.download(src, temp_filename)
# Override the source with the downloaded file
src = temp_filename
# Check for file .deb information (if file is present)
info = host.get_fact(DebPackage, name=src)
current_packages = host.get_fact(DebPackages)
exists = False
# We have deb info! Check against installed packages
if info:
if (info["name"] in current_packages
and info.get("version") in current_packages[info["name"]]):
exists = True
# Package does not exist and we want?
if present:
if not exists:
# Install .deb file - ignoring failure (on unmet dependencies)
yield "dpkg --force-confdef --force-confold -i {0} 2> /dev/null || true".format(
src)
# Attempt to install any missing dependencies
yield "{0} -f".format(noninteractive_apt("install", force=force))
# Now reinstall, and critically configure, the package - if there are still
# missing deps, now we error
yield "dpkg --force-confdef --force-confold -i {0}".format(src)
if info:
current_packages[info["name"]] = [info.get("version")]
else:
host.noop("deb {0} is installed".format(original_src))
# Package exists but we don't want?
if not present:
if exists:
yield "{0} {1}".format(
noninteractive_apt("remove", force=force),
info["name"],
)
current_packages.pop(info["name"])
else:
host.noop("deb {0} is not installed".format(original_src))
