def get( src, dest, add_deploy_dir=True, create_local_dir=False, force=False, ): """ Download a file from the remote system. + src: the remote filename to download + dest: the local filename to download the file to + add_deploy_dir: dest is relative to the deploy directory + create_local_dir: create the local directory if it doesn't exist + force: always download the file, even if the local copy matches Note: This operation is not suitable for large files as it may involve copying the remote file before downloading it. **Example:** .. code:: python files.get( name="Download a file from a remote", src="/etc/centos-release", dest="/tmp/whocares", ) """ if add_deploy_dir and state.cwd: dest = os.path.join(state.cwd, dest) if create_local_dir: local_pathname = os.path.dirname(dest) if not os.path.exists(local_pathname): os.makedirs(local_pathname) remote_file = host.get_fact(File, path=src) # No remote file, so assume exists and download it "blind" if not remote_file or force: yield FileDownloadCommand( src, dest, remote_temp_filename=state.get_temp_filename(dest)) # No local file, so always download elif not os.path.exists(dest): yield FileDownloadCommand( src, dest, remote_temp_filename=state.get_temp_filename(dest)) # Remote file exists - check if it matches our local else: local_sum = get_file_sha1(dest) remote_sum = host.get_fact(Sha1File, path=src) # Check sha1sum, upload if needed if local_sum != remote_sum: yield FileDownloadCommand( src, dest, remote_temp_filename=state.get_temp_filename(dest))
def script_template(src, **data): """ Generate, upload and execute a local script template on the remote host. + src: local script template filename **Example:** .. code:: python # Example showing how to pass python variable to a script template file. # The .j2 file can use `{{ some_var }}` to be interpolated. # To see output need to run pyinfra with '-v' # Note: This assumes there is a file in templates/hello2.bash.j2 locally. some_var = 'blah blah blah ' server.script_template( name="Hello from script", src="templates/hello2.bash.j2", some_var=some_var, ) """ temp_file = state.get_temp_filename("{0}{1}".format(src, data)) yield from files.template(src, temp_file, **data) yield chmod(temp_file, "+x") yield temp_file
def script(src): """ Upload and execute a local script on the remote host. + src: local script filename to upload & execute **Example:** .. code:: python # Note: This assumes there is a file in files/hello.bash locally. server.script( name="Hello", src="files/hello.bash", ) """ temp_file = state.get_temp_filename(src) yield from files.put(src, temp_file) yield chmod(temp_file, "+x") yield temp_file
password='******', ) mysql.database( {'Create the pyinfra_stuff database'}, 'pyinfra_stuff', user='******', user_privileges=['SELECT', 'INSERT'], charset='utf8', ) # Upload & import a SQL file into the pyinfra_stuff database # filename = 'files/a_db.sql' temp_filename = state.get_temp_filename(filename) files.put( {'Upload the a_db.sql file'}, filename, temp_filename, ) mysql.load( {'Import the a_db.sql file'}, temp_filename, database='pyinfra_stuff', ) # Now duplicate the pyinfra_stuff database -> pyinfra_stuff_copy #
def crontab( command, present=True, user=None, cron_name=None, minute="*", hour="*", month="*", day_of_week="*", day_of_month="*", special_time=None, interpolate_variables=False, ): """ Add/remove/update crontab entries. + command: the command for the cron + present: whether this cron command should exist + user: the user whose crontab to manage + cron_name: name the cronjob so future changes to the command will overwrite + minute: which minutes to execute the cron + hour: which hours to execute the cron + month: which months to execute the cron + day_of_week: which day of the week to execute the cron + day_of_month: which day of the month to execute the cron + special_time: cron "nickname" time (@reboot, @daily, etc), overrides others + interpolate_variables: whether to interpolate variables in ``command`` Cron commands: Unless ``name`` is specified the command is used to identify crontab entries. This means commands must be unique within a given users crontab. If you require multiple identical commands, provide a different name argument for each. Special times: When provided, ``special_time`` will be used instead of any values passed in for ``minute``/``hour``/``month``/``day_of_week``/``day_of_month``. **Example:** .. code:: python # simple example for a crontab server.crontab( name="Backup /etc weekly", command="/bin/tar cf /tmp/etc_bup.tar /etc", name="backup_etc", day_of_week=0, hour=1, minute=0, ) """ def comma_sep(value): if isinstance(value, (list, tuple)): return ",".join("{0}".format(v) for v in value) return value minute = comma_sep(minute) hour = comma_sep(hour) month = comma_sep(month) day_of_week = comma_sep(day_of_week) day_of_month = comma_sep(day_of_month) crontab = host.get_fact(Crontab, user=user) name_comment = "# pyinfra-name={0}".format(cron_name) existing_crontab = crontab.get(command) existing_crontab_command = command existing_crontab_match = command if not existing_crontab and cron_name: # find the crontab by name if provided for cmd, details in crontab.items(): if name_comment in details["comments"]: existing_crontab = details existing_crontab_match = cmd existing_crontab_command = cmd exists = existing_crontab is not None edit_commands = [] temp_filename = state.get_temp_filename() if special_time: new_crontab_line = "{0} {1}".format(special_time, command) else: new_crontab_line = "{minute} {hour} {day_of_month} {month} {day_of_week} {command}".format( minute=minute, hour=hour, day_of_month=day_of_month, month=month, day_of_week=day_of_week, command=command, ) existing_crontab_match = ".*{0}.*".format(existing_crontab_match) # Don't want the cron and it does exist? Remove the line if not present and exists: edit_commands.append( sed_replace( temp_filename, existing_crontab_match, "", interpolate_variables=interpolate_variables, ), ) # Want the cron but it doesn't exist? Append the line elif present and not exists: if cron_name: if crontab: # append a blank line if cron entries already exist edit_commands.append("echo '' >> {0}".format(temp_filename)) edit_commands.append( "echo {0} >> {1}".format( shlex.quote(name_comment), temp_filename, ), ) edit_commands.append( "echo {0} >> {1}".format( shlex.quote(new_crontab_line), temp_filename, ), ) # We have the cron and it exists, do it's details? If not, replace the line elif present and exists: if any(( special_time != existing_crontab.get("special_time"), minute != existing_crontab.get("minute"), hour != existing_crontab.get("hour"), month != existing_crontab.get("month"), day_of_week != existing_crontab.get("day_of_week"), day_of_month != existing_crontab.get("day_of_month"), existing_crontab_command != command, ), ): edit_commands.append( sed_replace( temp_filename, existing_crontab_match, new_crontab_line, interpolate_variables=interpolate_variables, ), ) if edit_commands: crontab_args = [] if user: crontab_args.append("-u {0}".format(user)) # List the crontab into a temporary file if it exists if crontab: yield "crontab -l {0} > {1}".format(" ".join(crontab_args), temp_filename) # Now yield any edits for edit_command in edit_commands: yield edit_command # Finally, use the tempfile to write a new crontab yield "crontab {0} {1}".format(" ".join(crontab_args), temp_filename) # Update the crontab fact if present: crontab[command] = { "special_time": special_time, "minute": minute, "hour": hour, "month": month, "day_of_week": day_of_week, "day_of_month": day_of_month, "comments": [cron_name] if cron_name else [], } else: crontab.pop(command) else: host.noop( "crontab {0} {1}".format( command, "exists" if present else "does not exist", ), )
def put( src, dest, user=None, group=None, mode=None, add_deploy_dir=True, create_remote_dir=True, force=False, assume_exists=False, ): """ Upload a local file, or file-like object, to the remote system. + src: filename or IO-like object to upload + dest: remote filename to upload to + user: user to own the files + group: group to own the files + mode: permissions of the files, use ``True`` to copy the local file + add_deploy_dir: src is relative to the deploy directory + create_remote_dir: create the remote directory if it doesn't exist + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists ``dest``: If this is a directory that already exists on the remote side, the local file will be uploaded to that directory with the same filename. ``mode``: When set to ``True`` the permissions of the local file are applied to the remote file after the upload is complete. ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Note: This operation is not suitable for large files as it may involve copying the file before uploading it. **Examples:** .. code:: python files.put( name="Update the message of the day file", src="files/motd", dest="/etc/motd", mode="644", ) files.put( name="Upload a StringIO object", src=StringIO("file contents"), dest="/etc/motd", ) """ # Upload IO objects as-is if hasattr(src, "read"): local_file = src local_sum = get_file_sha1(src) # Assume string filename else: # Add deploy directory? if add_deploy_dir and state.cwd: src = os.path.join(state.cwd, src) local_file = src if os.path.isfile(local_file): local_sum = get_file_sha1(local_file) elif assume_exists: local_sum = None else: raise IOError("No such file: {0}".format(local_file)) if mode is True: if os.path.isfile(local_file): mode = get_path_permissions_mode(local_file) else: logger.warning(( "No local file exists to get permissions from with `mode=True` ({0})" ).format(get_call_location(), ), ) else: mode = ensure_mode_int(mode) remote_file = host.get_fact(File, path=dest) if not remote_file and host.get_fact(Directory, path=dest): dest = unix_path_join(dest, os.path.basename(src)) remote_file = host.get_fact(File, path=dest) if create_remote_dir: yield from _create_remote_dir(state, host, dest, user, group) # No remote file, always upload and user/group/mode if supplied if not remote_file or force: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) # File exists, check sum and check user/group/mode if supplied else: remote_sum = host.get_fact(Sha1File, path=dest) # Check sha1sum, upload if needed if local_sum != remote_sum: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) else: changed = False # Check mode if mode and remote_file["mode"] != mode: yield file_utils.chmod(dest, mode) changed = True # Check user/group if (user and remote_file["user"] != user) or ( group and remote_file["group"] != group): yield file_utils.chown(dest, user, group) changed = True if not changed: host.noop("file {0} is already uploaded".format(dest)) # Now we've uploaded the file and ensured user/group/mode, update the relevant fact data host.create_fact(Sha1File, kwargs={"path": dest}, data=local_sum) host.create_fact( File, kwargs={"path": dest}, data={ "user": user, "group": group, "mode": mode }, )
def download( src, dest, user=None, group=None, mode=None, cache_time=None, force=False, sha256sum=None, sha1sum=None, md5sum=None, headers=None, insecure=False, ): """ Download files from remote locations using ``curl`` or ``wget``. + src: source URL of the file + dest: where to save the file + user: user to own the files + group: group to own the files + mode: permissions of the files + cache_time: if the file exists already, re-download after this time (in seconds) + force: always download the file, even if it already exists + sha256sum: sha256 hash to checksum the downloaded file against + sha1sum: sha1 hash to checksum the downloaded file against + md5sum: md5 hash to checksum the downloaded file against + headers: optional dictionary of headers to set for the HTTP request + insecure: disable SSL verification for the HTTP request **Example:** .. code:: python files.download( name="Download the Docker repo file", src="https://download.docker.com/linux/centos/docker-ce.repo", dest="/etc/yum.repos.d/docker-ce.repo", ) """ info = host.get_fact(File, path=dest) host_datetime = host.get_fact(Date).replace(tzinfo=None) # Destination is a directory? if info is False: raise OperationError( "Destination {0} already exists and is not a file".format(dest), ) # Do we download the file? Force by default download = force # Doesn't exist, lets download it if info is None: download = True # Destination file exists & cache_time: check when the file was last modified, # download if old else: if cache_time: # Time on files is not tz-aware, and will be the same tz as the server's time, # so we can safely remove the tzinfo from the Date fact before comparison. cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta( seconds=cache_time) if info["mtime"] and info["mtime"] < cache_time: download = True if sha1sum: if sha1sum != host.get_fact(Sha1File, path=dest): download = True if sha256sum: if sha256sum != host.get_fact(Sha256File, path=dest): download = True if md5sum: if md5sum != host.get_fact(Md5File, path=dest): download = True # If we download, always do user/group/mode as SSH user may be different if download: temp_file = state.get_temp_filename(dest) curl_args = ["-sSLf"] wget_args = ["-q"] if insecure: curl_args.append("--insecure") wget_args.append("--no-check-certificate") if headers: for key, value in headers.items(): header_arg = StringCommand("--header", QuoteString(f"{key}: {value}")) curl_args.append(header_arg) wget_args.append(header_arg) curl_command = make_formatted_string_command( "curl {0} {1} -o {2}", StringCommand(*curl_args), QuoteString(src), QuoteString(temp_file), ) wget_command = make_formatted_string_command( "wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )", StringCommand(*wget_args), QuoteString(src), QuoteString(temp_file), ) if host.get_fact(Which, command="curl"): yield curl_command elif host.get_fact(Which, command="wget"): yield wget_command else: yield "( {0} ) || ( {1} )".format(curl_command, wget_command) yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest)) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) if sha1sum: yield make_formatted_string_command( ("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) " "|| ( echo {2} && exit 1 )"), QuoteString(dest), sha1sum, QuoteString("SHA1 did not match!"), ) if sha256sum: yield make_formatted_string_command( ("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) " "| grep {1}) || ( echo {2} && exit 1 )"), QuoteString(dest), sha256sum, QuoteString("SHA256 did not match!"), ) if md5sum: yield make_formatted_string_command( ("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) " "|| ( echo {2} && exit 1 )"), QuoteString(dest), md5sum, QuoteString("MD5 did not match!"), ) host.create_fact( File, kwargs={"path": dest}, data={ "mode": mode, "group": group, "user": user, "mtime": host_datetime }, ) # Remove any checksum facts as we don't know the correct values for value, fact_cls in ( (sha1sum, Sha1File), (sha256sum, Sha256File), (md5sum, Md5File), ): fact_kwargs = {"path": dest} if value: host.create_fact(fact_cls, kwargs=fact_kwargs, data=value) else: host.delete_fact(fact_cls, kwargs=fact_kwargs) else: host.noop("file {0} has already been downloaded".format(dest))
) mysql.database( name="Create the pyinfra_stuff database", database="pyinfra_stuff", user="******", user_privileges=["SELECT", "INSERT"], charset="utf8", ) # Upload & import a SQL file into the pyinfra_stuff database # filename = "files/a_db.sql" temp_filename = state.get_temp_filename(filename) files.put( name="Upload the a_db.sql file", src=filename, dest=temp_filename, ) mysql.load( name="Import the a_db.sql file", src=temp_filename, database="pyinfra_stuff", ) # Now duplicate the pyinfra_stuff database -> pyinfra_stuff_copy
def upload( hostname, filename, remote_filename=None, port=22, user=None, use_remote_sudo=False, ssh_keyscan=False, ): """ Upload files to other servers using ``scp``. + hostname: hostname to upload to + filename: file to upload + remote_filename: where to upload the file to (defaults to ``filename``) + port: connect to this port + user: connect with this user + use_remote_sudo: upload to a temporary location and move using sudo + ssh_keyscan: execute ``ssh.keyscan`` before uploading the file """ remote_filename = remote_filename or filename # Figure out where we're connecting (host or user@host) connection_target = hostname if user: connection_target = "@".join((user, hostname)) if ssh_keyscan: yield from keyscan(hostname) # If we're not using sudo on the remote side, just scp the file over if not use_remote_sudo: yield "scp -P {0} {1} {2}:{3}".format( port, filename, connection_target, remote_filename, ) else: # Otherwise - we need a temporary location for the file temp_remote_filename = state.get_temp_filename() # scp it to the temporary location upload_cmd = "scp -P {0} {1} {2}:{3}".format( port, filename, connection_target, temp_remote_filename, ) yield upload_cmd # And sudo sudo to move it yield from command( hostname=hostname, command="sudo mv {0} {1}".format(temp_remote_filename, remote_filename), port=port, user=user, )
def put( src, dest, user=None, group=None, mode=None, add_deploy_dir=True, create_remote_dir=True, force=False, assume_exists=False, ): """ Upload a local file to the remote system. + src: local filename to upload + dest: remote filename to upload to + user: user to own the files + group: group to own the files + mode: permissions of the files + add_deploy_dir: src is relative to the deploy directory + create_remote_dir: create the remote directory if it doesn't exist + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Note: This operation is not suitable for large files as it may involve copying the file before uploading it. **Examples:** .. code:: python # Note: This requires a 'files/motd' file on the local filesystem files.put( name="Update the message of the day file", src="data/content.json", dest="C:\\data\\content.json", ) """ # Upload IO objects as-is if hasattr(src, "read"): local_file = src # Assume string filename else: # Add deploy directory? if add_deploy_dir and state.cwd: src = os.path.join(state.cwd, src) local_file = src if not assume_exists and not os.path.isfile(local_file): raise IOError("No such file: {0}".format(local_file)) mode = ensure_mode_int(mode) remote_file = host.get_fact(File, path=dest) if create_remote_dir: yield from _create_remote_dir(state, host, dest, user, group) # No remote file, always upload and user/group/mode if supplied if not remote_file or force: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) # if user or group: # yield chown(dest, user, group) # if mode: # yield chmod(dest, mode) # File exists, check sum and check user/group/mode if supplied else: local_sum = get_file_sha1(src) remote_sum = host.get_fact(Sha1File, path=dest) # Check sha1sum, upload if needed if local_sum != remote_sum: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) # if user or group: # yield chown(dest, user, group) # if mode: # yield chmod(dest, mode) else: changed = False # Check mode # if mode and remote_file['mode'] != mode: # yield chmod(dest, mode) # changed = True # Check user/group # if ( # (user and remote_file['user'] != user) # or (group and remote_file['group'] != group) # ): # yield chown(dest, user, group) # changed = True if not changed: host.noop("file {0} is already uploaded".format(dest))
def deb(src, present=True, force=False): """ Add/remove ``.deb`` file packages. + src: filename or URL of the ``.deb`` file + present: whether or not the package should exist on the system + force: whether to force the package install by passing `--force-yes` to apt Note: When installing, ``apt-get install -f`` will be run to install any unmet dependencies. URL sources with ``present=False``: If the ``.deb`` file isn't downloaded, pyinfra can't remove any existing package as the file won't exist until mid-deploy. **Example:** .. code:: python # Note: Assumes wget is installed. apt.deb( name="Install Chrome via deb", src="https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb", ) """ original_src = src # If source is a url if urlparse(src).scheme: # Generate a temp filename temp_filename = state.get_temp_filename(src) # Ensure it's downloaded yield from files.download(src, temp_filename) # Override the source with the downloaded file src = temp_filename # Check for file .deb information (if file is present) info = host.get_fact(DebPackage, name=src) current_packages = host.get_fact(DebPackages) exists = False # We have deb info! Check against installed packages if info: if (info["name"] in current_packages and info.get("version") in current_packages[info["name"]]): exists = True # Package does not exist and we want? if present: if not exists: # Install .deb file - ignoring failure (on unmet dependencies) yield "dpkg --force-confdef --force-confold -i {0} 2> /dev/null || true".format( src) # Attempt to install any missing dependencies yield "{0} -f".format(noninteractive_apt("install", force=force)) # Now reinstall, and critically configure, the package - if there are still # missing deps, now we error yield "dpkg --force-confdef --force-confold -i {0}".format(src) if info: current_packages[info["name"]] = [info.get("version")] else: host.noop("deb {0} is installed".format(original_src)) # Package exists but we don't want? if not present: if exists: yield "{0} {1}".format( noninteractive_apt("remove", force=force), info["name"], ) current_packages.pop(info["name"]) else: host.noop("deb {0} is not installed".format(original_src))