def repo(src, present=True, filename=None): """ Add/remove apt repositories. + src: apt source string eg ``deb http://X hardy main`` + present: whether the repo should exist on the system + filename: optional filename to use ``/etc/apt/sources.list.d/<filename>.list``. By default uses ``/etc/apt/sources.list``. **Example:** .. code:: python apt.repo( name="Install VirtualBox repo", src="deb https://download.virtualbox.org/virtualbox/debian bionic contrib", ) """ # Get the target .list file to manage if filename: filename = "/etc/apt/sources.list.d/{0}.list".format(filename) else: filename = "/etc/apt/sources.list" # Work out if the repo exists already apt_sources = host.get_fact(AptSources) is_present = False repo = parse_apt_repo(src) if repo and repo in apt_sources: is_present = True # Doesn't exist and we want it if not is_present and present: yield from files.line( filename, src, escape_regex_characters=True, ) apt_sources.append(repo) # Exists and we don't want it elif is_present and not present: yield from files.line( filename, src, present=False, assume_present=True, escape_regex_characters=True, ) apt_sources.remove(repo) else: host.noop( 'apt repo "{0}" {1}'.format( src, "exists" if present else "does not exist", ), )
def keyscan(hostname, force=False, port=22): """ Check/add hosts to the ``~/.ssh/known_hosts`` file. + hostname: hostname that should have a key in ``known_hosts`` + force: if the key already exists, remove and rescan **Example:** .. code:: python ssh.keyscan( name="Set add server two to known_hosts on one", hostname="two.example.com", ) """ homedir = host.get_fact(Home) yield from files.directory( "{0}/.ssh".format(homedir), mode=700, ) hostname_present = host.get_fact( FindInFile, path="{0}/.ssh/known_hosts".format(homedir), pattern=hostname, ) did_keyscan = False keyscan_command = "ssh-keyscan -p {0} {1} >> {2}/.ssh/known_hosts".format( port, hostname, homedir, ) if not hostname_present: yield keyscan_command did_keyscan = True elif force: yield "ssh-keygen -R {0}".format(hostname) yield keyscan_command did_keyscan = True else: host.noop("host key for {0} already exists".format(hostname)) if did_keyscan: host.create_fact( FindInFile, kwargs={ "path": "{0}/.ssh/known_hosts".format(homedir), "pattern": hostname }, data=["{0} unknown unknown".format(hostname)], )
def mount( path, mounted=True, options=None, # TODO: do we want to manage fstab here? # update_fstab=False, device=None, fs_type=None, ): """ Manage mounted filesystems. + path: the path of the mounted filesystem + mounted: whether the filesystem should be mounted + options: the mount options Options: If the currently mounted filesystem does not have all of the provided options it will be remounted with the options provided. ``/etc/fstab``: This operation does not attempt to modify the on disk fstab file - for that you should use the `files.line operation <./files.html#files-line>`_. """ options = options or [] options_string = ",".join(options) mounts = host.get_fact(Mounts) is_mounted = path in mounts # Want mount but don't have? if mounted and not is_mounted: yield "mount{0} {1}".format( " -o {0}".format(options_string) if options_string else "", path, ) mounts[path] = {"options": options} # Want no mount but mounted? elif mounted is False and is_mounted: yield "umount {0}".format(path) mounts.pop(path) # Want mount and is mounted! Check the options elif is_mounted and mounted and options: mounted_options = mounts[path]["options"] needed_options = set(options) - set(mounted_options) if needed_options: yield "mount -o remount,{0} {1}".format(options_string, path) mounts[path]["options"] = options else: host.noop( "filesystem {0} is {1}".format( path, "mounted" if mounted else "not mounted", ), )
def hostname(hostname, hostname_file=None): """ Set the system hostname using ``hostnamectl`` or ``hostname`` on older systems. + hostname: the hostname that should be set + hostname_file: the file that permanently sets the hostname Hostname file: The hostname file only matters no systems that do not have ``hostnamectl``, which is part of ``systemd``. By default pyinfra will auto detect this by targeting ``/etc/hostname`` on Linux and ``/etc/myname`` on OpenBSD. To completely disable writing the hostname file, set ``hostname_file=False``. **Example:** .. code:: python server.hostname( name="Set the hostname", hostname="server1.example.com", ) """ current_hostname = host.get_fact(Hostname) if host.get_fact(Which, command="hostnamectl"): if current_hostname != hostname: yield "hostnamectl set-hostname {0}".format(hostname) host.create_fact(Hostname, data=hostname) else: host.noop("hostname is set") return if hostname_file is None: os = host.get_fact(Os) if os == "Linux": hostname_file = "/etc/hostname" elif os == "OpenBSD": hostname_file = "/etc/myname" if current_hostname != hostname: yield "hostname {0}".format(hostname) host.create_fact(Hostname, data=hostname) else: host.noop("hostname is set") if hostname_file: # Create a whole new hostname file file = StringIO("{0}\n".format(hostname)) # And ensure it exists yield from files.put(file, hostname_file)
def container( id, present=True, image="ubuntu:16.04", ): """ Add/remove LXD containers. Note: does not check if an existing container is based on the specified image. + id: name/identifier for the container + image: image to base the container on + present: whether the container should be present or absent **Example:** .. code:: python lxd.container( name="Add an ubuntu container", id="ubuntu19", image="ubuntu:19.10", ) """ current_containers = host.get_fact(LxdContainers) container = get_container_named(id, current_containers) # Container exists and we don't want it if not present: if container: if container["status"] == "Running": yield "lxc stop {0}".format(id) # Command to remove the container: yield "lxc delete {0}".format(id) current_containers.remove(container) else: host.noop("container {0} does not exist".format(id)) # Container doesn't exist and we want it if present: if not container: # Command to create the container: yield "lxc launch {image} {id} < /dev/null".format(id=id, image=image) current_containers.append({ "name": id, "image": image, }, ) else: host.noop("container {0} exists".format(id))
def modprobe(module, present=True, force=False): """ Load/unload kernel modules. + module: name of the module to manage + present: whether the module should be loaded or not + force: whether to force any add/remove modules **Example:** .. code:: python server.modprobe( name="Silly example for modprobe", module="floppy", ) """ list_value = [module] if isinstance(module, str) else module # NOTE: https://docs.python.org/3/library/itertools.html#itertools-recipes def partition(predicate, iterable): t1, t2 = tee(iterable) return list(filter(predicate, t2)), list(filterfalse(predicate, t1)) modules = host.get_fact(KernelModules) present_mods, missing_mods = partition(lambda mod: mod in modules, list_value) args = "" if force: args = " -f" # Module is loaded and we don't want it? if not present and present_mods: yield "modprobe{0} -r -a {1}".format(args, " ".join(present_mods)) for mod in present_mods: modules.pop(mod) # Module isn't loaded and we want it? elif present and missing_mods: yield "modprobe{0} -a {1}".format(args, " ".join(missing_mods)) for mod in missing_mods: modules[mod] = {} else: host.noop( "{0} {1} {2} {3}".format( "modules" if len(list_value) > 1 else "module", "/".join(list_value), "are" if len(list_value) > 1 else "is", "loaded" if present else "not loaded", ), )
def sysctl( key, value, persist=False, persist_file="/etc/sysctl.conf", ): """ Edit sysctl configuration. + key: name of the sysctl setting to ensure + value: the value or list of values the sysctl should be + persist: whether to write this sysctl to the config + persist_file: file to write the sysctl to persist on reboot **Example:** .. code:: python server.sysctl( name="Change the fs.file-max value", key="fs.file-max", value=100000, persist=True, ) """ string_value = " ".join(["{0}".format(v) for v in value]) if isinstance( value, list) else value value = [try_int(v) for v in value] if isinstance(value, list) else try_int(value) existing_sysctls = host.get_fact(Sysctl) existing_value = existing_sysctls.get(key) if not existing_value or existing_value != value: yield "sysctl {0}='{1}'".format(key, string_value) existing_sysctls[key] = value else: host.noop("sysctl {0} is set to {1}".format(key, string_value)) if persist: yield from files.line( path=persist_file, line="{0}[[:space:]]*=[[:space:]]*{1}".format(key, string_value), replace="{0} = {1}".format(key, string_value), )
def chain( chain, present=True, table="filter", policy=None, version=4, ): """ Add/remove/update iptables chains. + chain: the name of the chain + present: whether the chain should exist + table: the iptables table this chain should belong to + policy: the policy this table should have + version: whether to target iptables or ip6tables Policy: These can only be applied to system chains (FORWARD, INPUT, OUTPUT, etc). """ chains = (host.get_fact(IptablesChains, table=table) if version == 4 else host.get_fact(Ip6tablesChains, table=table)) command = "iptables" if version == 4 else "ip6tables" command = "{0} -t {1}".format(command, table) if not present: if chain in chains: yield "{0} -X {1}".format(command, chain) chains.pop(chain) else: host.noop("iptables chain {0} does not exist".format(chain)) return if present: if chain not in chains: yield "{0} -N {1}".format(command, chain) chains[chain] = None # policy will be set below else: host.noop("iptables chain {0} exists".format(chain)) if policy: if chain not in chains or chains[chain] != policy: yield "{0} -P {1} {2}".format(command, chain, policy) chains[chain] = policy
def update(cache_time=None): """ Updates apt repositories. + cache_time: cache updates for this many seconds **Example:** .. code:: python apt.update( name="Update apt repositories", cache_time=3600, ) """ # If cache_time check when apt was last updated, prevent updates if within time if cache_time: # Ubuntu provides this handy file cache_info = host.get_fact(File, path=APT_UPDATE_FILENAME) # Time on files is not tz-aware, and will be the same tz as the server's time, # so we can safely remove the tzinfo from the Date fact before comparison. host_cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta( seconds=cache_time) if cache_info and cache_info[ "mtime"] and cache_info["mtime"] > host_cache_time: host.noop("apt is already up to date") return yield "apt-get update" # Some apt systems (Debian) have the /var/lib/apt/periodic directory, but # don't bother touching anything in there - so pyinfra does it, enabling # cache_time to work. if cache_time: yield "touch {0}".format(APT_UPDATE_FILENAME) if cache_info is None: host.create_fact( File, kwargs={"path": APT_UPDATE_FILENAME}, data={"mtime": datetime.utcnow()}, ) else: cache_info["mtime"] = datetime.utcnow()
def config(key, value, repo=None): """ Manage git config for a repository or globally. + key: the key of the config to ensure + value: the value this key should have + repo: specify the git repo path to edit local config (defaults to global) **Example:** .. code:: python git.config( name="Ensure user name is set for a repo", key="user.name", value="Anon E. Mouse", repo="/usr/local/src/pyinfra", ) """ existing_config = {} if not repo: existing_config = host.get_fact(GitConfig) # Only get the config if the repo exists at this stage elif host.get_fact(Directory, path=unix_path_join(repo, ".git")): existing_config = host.get_fact(GitConfig, repo=repo) if existing_config.get(key) != value: if repo is None: yield 'git config --global {0} "{1}"'.format(key, value) else: yield 'cd {0} && git config --local {1} "{2}"'.format( repo, key, value) existing_config[key] = value else: host.noop("git config {0} is set to {1}".format(key, value))
def tap(src, present=True): """ Add/remove brew taps. + src: the name of the tap + present: whether this tap should be present or not **Examples:** .. code:: python brew.tap( name="Add a brew tap", src="includeos/includeos", ) # Multiple taps for tap in ["includeos/includeos", "ktr0731/evans"]: brew.tap( name={f"Add brew tap {tap}"}, src=tap, ) """ taps = host.get_fact(BrewTaps) is_tapped = src in taps if present: if is_tapped: host.noop("tap {0} already exists".format(src)) else: yield "brew tap {0}".format(src) taps.append(src) elif not present: if is_tapped: yield "brew untap {0}".format(src) taps.remove(src) else: host.noop("tap {0} does not exist".format(src))
def role( role, present=True, password=None, login=True, superuser=False, inherit=False, createdb=False, createrole=False, replication=False, connection_limit=None, # Details for speaking to PostgreSQL via `psql` CLI psql_user=None, psql_password=None, psql_host=None, psql_port=None, ): """ Add/remove PostgreSQL roles. + role: name of the role + present: whether the role should be present or absent + password: the password for the role + login: whether the role can login + superuser: whether role will be a superuser + inherit: whether the role inherits from other roles + createdb: whether the role is allowed to create databases + createrole: whether the role is allowed to create new roles + replication: whether this role is allowed to replicate + connection_limit: the connection limit for the role + psql_*: global module arguments, see above Updates: pyinfra will not attempt to change existing roles - it will either create or drop roles, but not alter them (if the role exists this operation will make no changes). **Example:** .. code:: python postgresql.role( name="Create the pyinfra PostgreSQL role", role="pyinfra", password="******", superuser=True, login=True, sudo_user="******", ) """ roles = host.get_fact( PostgresqlRoles, psql_user=psql_user, psql_password=psql_password, psql_host=psql_host, psql_port=psql_port, ) is_present = role in roles # User not wanted? if not present: if is_present: yield make_execute_psql_command( 'DROP ROLE "{0}"'.format(role), user=psql_user, password=psql_password, host=psql_host, port=psql_port, ) roles.pop(role) else: host.noop("postgresql role {0} does not exist".format(role)) return # If we want the user and they don't exist if not is_present: sql_bits = ['CREATE ROLE "{0}"'.format(role)] for key, value in ( ("LOGIN", login), ("SUPERUSER", superuser), ("INHERIT", inherit), ("CREATEDB", createdb), ("CREATEROLE", createrole), ("REPLICATION", replication), ): if value: sql_bits.append(key) if connection_limit: sql_bits.append("CONNECTION LIMIT {0}".format(connection_limit)) if password: sql_bits.append(MaskString("PASSWORD '{0}'".format(password))) yield make_execute_psql_command( StringCommand(*sql_bits), user=psql_user, password=psql_password, host=psql_host, port=psql_port, ) roles[role] = { "super": superuser, "cretedb": createdb, "createrole": createrole, } else: host.noop("postgresql role {0} exists".format(role))
def database( database, present=True, owner=None, template=None, encoding=None, lc_collate=None, lc_ctype=None, tablespace=None, connection_limit=None, # Details for speaking to PostgreSQL via `psql` CLI psql_user=None, psql_password=None, psql_host=None, psql_port=None, ): """ Add/remove PostgreSQL databases. + name: name of the database + present: whether the database should exist or not + owner: the PostgreSQL role that owns the database + template: name of the PostgreSQL template to use + encoding: encoding of the database + lc_collate: lc_collate of the database + lc_ctype: lc_ctype of the database + tablespace: the tablespace to use for the template + connection_limit: the connection limit to apply to the database + psql_*: global module arguments, see above Updates: pyinfra will not attempt to change existing databases - it will either create or drop databases, but not alter them (if the db exists this operation will make no changes). **Example:** .. code:: python postgresql.database( name="Create the pyinfra_stuff database", database="pyinfra_stuff", owner="pyinfra", encoding="UTF8", sudo_user="******", ) """ current_databases = host.get_fact( PostgresqlDatabases, psql_user=psql_user, psql_password=psql_password, psql_host=psql_host, psql_port=psql_port, ) is_present = database in current_databases if not present: if is_present: yield make_execute_psql_command( 'DROP DATABASE "{0}"'.format(database), user=psql_user, password=psql_password, host=psql_host, port=psql_port, ) current_databases.pop(database) else: host.noop( "postgresql database {0} does not exist".format(database)) return # We want the database but it doesn't exist if present and not is_present: sql_bits = ['CREATE DATABASE "{0}"'.format(database)] for key, value in ( ("OWNER", '"{0}"'.format(owner) if owner else owner), ("TEMPLATE", template), ("ENCODING", encoding), ("LC_COLLATE", lc_collate), ("LC_CTYPE", lc_ctype), ("TABLESPACE", tablespace), ("CONNECTION LIMIT", connection_limit), ): if value: sql_bits.append("{0} {1}".format(key, value)) yield make_execute_psql_command( StringCommand(*sql_bits), user=psql_user, password=psql_password, host=psql_host, port=psql_port, ) current_databases[database] = { "template": template, "encoding": encoding, "lc_collate": lc_collate, "lc_ctype": lc_ctype, "tablespace": tablespace, "connection_limit": connection_limit, } else: host.noop("postgresql database {0} exists".format(database))
def user( user, present=True, user_hostname="localhost", password=None, privileges=None, # MySQL REQUIRE SSL/TLS options require=None, # SSL or X509 require_cipher=False, require_issuer=False, require_subject=False, # MySQL WITH resource limit options max_connections=None, max_queries_per_hour=None, max_updates_per_hour=None, max_connections_per_hour=None, # Details for speaking to MySQL via `mysql` CLI via `mysql` CLI mysql_user=None, mysql_password=None, mysql_host=None, mysql_port=None, ): """ Add/remove/update MySQL users. + user: the name of the user + present: whether the user should exist or not + user_hostname: the hostname of the user + password: the password of the user (if created) + privileges: the global privileges for this user + mysql_*: global module arguments, see above Hostname: this + ``name`` makes the username - so changing this will create a new user, rather than update users with the same ``name``. Password: will only be applied if the user does not exist - ie pyinfra cannot detect if the current password doesn't match the one provided, so won't attempt to change it. **Example:** .. code:: python mysql.user( name="Create the pyinfra@localhost MySQL user", user="******", password="******", ) # Create a user with resource limits mysql.user( name="Create the pyinfra@localhost MySQL user", user="******", max_connections=50, max_updates_per_hour=10, ) # Create a user that requires SSL for connections mysql.user( name="Create the pyinfra@localhost MySQL user", user="******", password="******", require="SSL", ) # Create a user that requires a specific certificate mysql.user( name="Create the pyinfra@localhost MySQL user", user="******", password="******", require="X509", require_issuer="/C=SE/ST=Stockholm...", require_cipher="EDH-RSA-DES-CBC3-SHA", ) """ if require and require not in ("SSL", "X509"): raise OperationError( 'Invalid `require` value, must be: "SSL" or "X509"') if require != "X509": if require_cipher: raise OperationError( 'Cannot set `require_cipher` if `require` is not "X509"') if require_issuer: raise OperationError( 'Cannot set `require_issuer` if `require` is not "X509"') if require_subject: raise OperationError( 'Cannot set `require_subject` if `require` is not "X509"') current_users = host.get_fact( MysqlUsers, mysql_user=mysql_user, mysql_password=mysql_password, mysql_host=mysql_host, mysql_port=mysql_port, ) user_host = "{0}@{1}".format(user, user_hostname) is_present = user_host in current_users if not present: if is_present: yield make_execute_mysql_command( 'DROP USER "{0}"@"{1}"'.format(user, user_hostname), user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) current_users.pop(user_host) else: host.noop("mysql user {0}@{1} does not exist".format( user, user_hostname)) return new_or_updated_user_fact = { "ssl_type": "ANY" if require == "SSL" else require, "ssl_cipher": require_cipher, "x509_issuer": require_issuer, "x509_subject": require_subject, "max_user_connections": max_connections, "max_questions": max_queries_per_hour, "max_updates": max_updates_per_hour, "max_connections": max_connections_per_hour, } if present and not is_present: sql_bits = ['CREATE USER "{0}"@"{1}"'.format(user, user_hostname)] if password: sql_bits.append(MaskString('IDENTIFIED BY "{0}"'.format(password))) if require == "SSL": sql_bits.append("REQUIRE SSL") if require == "X509": sql_bits.append("REQUIRE") require_bits = [] if require_cipher: require_bits.append('CIPHER "{0}"'.format(require_cipher)) if require_issuer: require_bits.append('ISSUER "{0}"'.format(require_issuer)) if require_subject: require_bits.append('SUBJECT "{0}"'.format(require_subject)) if not require_bits: require_bits.append("X509") sql_bits.extend(require_bits) resource_bits = [] if max_connections: resource_bits.append( "MAX_USER_CONNECTIONS {0}".format(max_connections)) if max_queries_per_hour: resource_bits.append( "MAX_QUERIES_PER_HOUR {0}".format(max_queries_per_hour)) if max_updates_per_hour: resource_bits.append( "MAX_UPDATES_PER_HOUR {0}".format(max_updates_per_hour)) if max_connections_per_hour: resource_bits.append("MAX_CONNECTIONS_PER_HOUR {0}".format( max_connections_per_hour)) if resource_bits: sql_bits.append("WITH") sql_bits.append(" ".join(resource_bits)) yield make_execute_mysql_command( StringCommand(*sql_bits), user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) current_users[user_host] = new_or_updated_user_fact if present and is_present: current_user = current_users.get(user_host) alter_bits = [] if require == "SSL": if current_user["ssl_type"] != "ANY": alter_bits.append("REQUIRE SSL") if require == "X509": require_bits = [] if require_cipher and current_user["ssl_cipher"] != require_cipher: require_bits.append('CIPHER "{0}"'.format(require_cipher)) if require_issuer and current_user["x509_issuer"] != require_issuer: require_bits.append('ISSUER "{0}"'.format(require_issuer)) if require_subject and current_user[ "x509_subject"] != require_subject: require_bits.append('SUBJECT "{0}"'.format(require_subject)) if not require_bits: if current_user["ssl_type"] != "X509": require_bits.append("X509") if require_bits: alter_bits.append("REQUIRE") alter_bits.extend(require_bits) resource_bits = [] if max_connections and current_user[ "max_user_connections"] != max_connections: resource_bits.append( "MAX_USER_CONNECTIONS {0}".format(max_connections)) if max_queries_per_hour and current_user[ "max_questions"] != max_queries_per_hour: resource_bits.append( "MAX_QUERIES_PER_HOUR {0}".format(max_queries_per_hour)) if max_updates_per_hour and current_user[ "max_updates"] != max_updates_per_hour: resource_bits.append( "MAX_UPDATES_PER_HOUR {0}".format(max_updates_per_hour)) if max_connections_per_hour and current_user[ "max_connections"] != max_connections_per_hour: resource_bits.append("MAX_CONNECTIONS_PER_HOUR {0}".format( max_connections_per_hour)) if resource_bits: alter_bits.append("WITH") alter_bits.append(" ".join(resource_bits)) if alter_bits: sql_bits = ['ALTER USER "{0}"@"{1}"'.format(user, user_hostname)] sql_bits.extend(alter_bits) yield make_execute_mysql_command( StringCommand(*sql_bits), user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) current_user.update(new_or_updated_user_fact) else: host.noop("mysql user {0}@{1} exists".format(user, user_hostname)) # If we're here either the user exists or we just created them; either way # now we can check any privileges are set. if privileges: yield from _privileges( user, privileges, user_hostname=user_hostname, mysql_user=mysql_user, mysql_password=mysql_password, mysql_host=mysql_host, mysql_port=mysql_port, )
def privileges( user, privileges, user_hostname="localhost", database="*", table="*", flush=True, with_grant_option=False, # Details for speaking to MySQL via `mysql` CLI mysql_user=None, mysql_password=None, mysql_host=None, mysql_port=None, ): """ Add/remove MySQL privileges for a user, either global, database or table specific. + user: name of the user to manage privileges for + privileges: list of privileges the user should have (see also: ``with_grant_option`` argument) + user_hostname: the hostname of the user + database: name of the database to grant privileges to (defaults to all) + table: name of the table to grant privileges to (defaults to all) + flush: whether to flush (and update) the privileges table after any changes + with_grant_option: whether the grant option privilege should be set + mysql_*: global module arguments, see above """ # Ensure we have a list if isinstance(privileges, str): privileges = {privileges} if isinstance(privileges, list): privileges = set(privileges) if with_grant_option: privileges.add("GRANT OPTION") if database != "*": database = "`{0}`".format(database) if table != "*": table = "`{0}`".format(table) # We can't set privileges on *.tablename as MySQL won't allow it if database == "*": raise OperationError(( "Cannot apply MySQL privileges on {0}.{1}, no database provided" ).format( database, table, ), ) database_table = "{0}.{1}".format(database, table) user_grants = host.get_fact( MysqlUserGrants, user=user, hostname=user_hostname, mysql_user=mysql_user, mysql_password=mysql_password, mysql_host=mysql_host, mysql_port=mysql_port, ) existing_privileges = set() if database_table in user_grants: existing_privileges = { "ALL" if privilege == "ALL PRIVILEGES" else privilege for privilege in user_grants[database_table] } else: user_grants[database_table] = set() def handle_privileges(action, target, privileges_to_apply): command = ("{action} {privileges} " "ON {database}.{table} " '{target} "{user}"@"{user_hostname}"').format( privileges=", ".join(sorted(privileges_to_apply)), action=action, target=target, database=database, table=table, user=user, user_hostname=user_hostname, ) yield make_execute_mysql_command( command, user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) # Find / revoke any privileges that exist that do not match the desired state privileges_to_revoke = existing_privileges.difference(privileges) # Find / grant any privileges that we want but do not exist privileges_to_grant = privileges - existing_privileges # No privilege and we want it if privileges_to_grant: yield from handle_privileges("GRANT", "TO", privileges_to_grant) user_grants[database_table].update(privileges_to_grant) if privileges_to_revoke: yield from handle_privileges("REVOKE", "FROM", privileges_to_revoke) user_grants[database_table] -= privileges_to_revoke if privileges_to_grant or privileges_to_revoke: if flush: yield make_execute_mysql_command( "FLUSH PRIVILEGES", user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) else: host.noop("mysql privileges are already correct")
def database( database, # Desired database settings present=True, collate=None, charset=None, user=None, user_hostname="localhost", user_privileges="ALL", # Details for speaking to MySQL via `mysql` CLI mysql_user=None, mysql_password=None, mysql_host=None, mysql_port=None, ): """ Add/remove MySQL databases. + name: the name of the database + present: whether the database should exist or not + collate: the collate to use when creating the database + charset: the charset to use when creating the database + user: MySQL user to grant privileges on this database to + user_hostname: the hostname of the MySQL user to grant + user_privileges: privileges to grant to any specified user + mysql_*: global module arguments, see above Collate/charset: these will only be applied if the database does not exist - ie pyinfra will not attempt to alter the existing databases collate/character sets. **Example:** .. code:: python mysql.database( name="Create the pyinfra_stuff database", database="pyinfra_stuff", user="******", user_privileges=["SELECT", "INSERT"], charset="utf8", ) """ current_databases = host.get_fact( MysqlDatabases, mysql_user=mysql_user, mysql_password=mysql_password, mysql_host=mysql_host, mysql_port=mysql_port, ) is_present = database in current_databases if not present: if is_present: yield make_execute_mysql_command( "DROP DATABASE {0}".format(database), user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) current_databases.pop(database) else: host.noop("mysql database {0} does not exist".format(database)) return # We want the database but it doesn't exist if present and not is_present: sql_bits = ["CREATE DATABASE {0}".format(database)] if collate: sql_bits.append("COLLATE {0}".format(collate)) if charset: sql_bits.append("CHARSET {0}".format(charset)) yield make_execute_mysql_command( " ".join(sql_bits), user=mysql_user, password=mysql_password, host=mysql_host, port=mysql_port, ) current_databases[database] = { "collate": collate, "charset": charset, } else: host.noop("mysql database {0} exists".format(database)) # Ensure any user privileges for this database if user and user_privileges: yield from privileges( user, user_hostname=user_hostname, privileges=user_privileges, database=database, mysql_user=mysql_user, mysql_password=mysql_password, mysql_host=mysql_host, mysql_port=mysql_port, )
def virtualenv( path, python=None, venv=False, site_packages=False, always_copy=False, present=True, ): """ Add/remove Python virtualenvs. + python: python interpreter to use + venv: use standard venv module instead of virtualenv + site_packages: give access to the global site-packages + always_copy: always copy files rather than symlinking + present: whether the virtualenv should exist **Example:** .. code:: python pip.virtualenv( name="Create a virtualenv", path="/usr/local/bin/venv", ) """ # Check for *contents* of a virtualenv, ie don't accept an empty directory # as a valid virtualenv but ensure the activate script exists. activate_script_path = "{0}/bin/activate".format(path) if present is False: if host.get_fact(File, path=activate_script_path): yield from files.directory(path, present=False) else: host.noop("virtualenv {0} does not exist".format(path)) if present: if not host.get_fact(File, path=activate_script_path): # Create missing virtualenv command = ["virtualenv"] if venv: command = [python or "python", "-m", "venv"] if python and not venv: command.append("-p {0}".format(python)) if site_packages: command.append("--system-site-packages") if always_copy and not venv: command.append("--always-copy") elif always_copy and venv: command.append("--copies") command.append(path) yield " ".join(command) host.create_fact( File, kwargs={"path": activate_script_path}, data={ "user": None, "group": None }, ) else: host.noop("virtualenv {0} exists".format(path))
def put( src, dest, user=None, group=None, mode=None, add_deploy_dir=True, create_remote_dir=True, force=False, assume_exists=False, ): """ Upload a local file to the remote system. + src: local filename to upload + dest: remote filename to upload to + user: user to own the files + group: group to own the files + mode: permissions of the files + add_deploy_dir: src is relative to the deploy directory + create_remote_dir: create the remote directory if it doesn't exist + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Note: This operation is not suitable for large files as it may involve copying the file before uploading it. **Examples:** .. code:: python # Note: This requires a 'files/motd' file on the local filesystem files.put( name="Update the message of the day file", src="data/content.json", dest="C:\\data\\content.json", ) """ # Upload IO objects as-is if hasattr(src, "read"): local_file = src # Assume string filename else: # Add deploy directory? if add_deploy_dir and state.cwd: src = os.path.join(state.cwd, src) local_file = src if not assume_exists and not os.path.isfile(local_file): raise IOError("No such file: {0}".format(local_file)) mode = ensure_mode_int(mode) remote_file = host.get_fact(File, path=dest) if create_remote_dir: yield from _create_remote_dir(state, host, dest, user, group) # No remote file, always upload and user/group/mode if supplied if not remote_file or force: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) # if user or group: # yield chown(dest, user, group) # if mode: # yield chmod(dest, mode) # File exists, check sum and check user/group/mode if supplied else: local_sum = get_file_sha1(src) remote_sum = host.get_fact(Sha1File, path=dest) # Check sha1sum, upload if needed if local_sum != remote_sum: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) # if user or group: # yield chown(dest, user, group) # if mode: # yield chmod(dest, mode) else: changed = False # Check mode # if mode and remote_file['mode'] != mode: # yield chmod(dest, mode) # changed = True # Check user/group # if ( # (user and remote_file['user'] != user) # or (group and remote_file['group'] != group) # ): # yield chown(dest, user, group) # changed = True if not changed: host.noop("file {0} is already uploaded".format(dest))
def crontab( command, present=True, user=None, cron_name=None, minute="*", hour="*", month="*", day_of_week="*", day_of_month="*", special_time=None, interpolate_variables=False, ): """ Add/remove/update crontab entries. + command: the command for the cron + present: whether this cron command should exist + user: the user whose crontab to manage + cron_name: name the cronjob so future changes to the command will overwrite + minute: which minutes to execute the cron + hour: which hours to execute the cron + month: which months to execute the cron + day_of_week: which day of the week to execute the cron + day_of_month: which day of the month to execute the cron + special_time: cron "nickname" time (@reboot, @daily, etc), overrides others + interpolate_variables: whether to interpolate variables in ``command`` Cron commands: Unless ``name`` is specified the command is used to identify crontab entries. This means commands must be unique within a given users crontab. If you require multiple identical commands, provide a different name argument for each. Special times: When provided, ``special_time`` will be used instead of any values passed in for ``minute``/``hour``/``month``/``day_of_week``/``day_of_month``. **Example:** .. code:: python # simple example for a crontab server.crontab( name="Backup /etc weekly", command="/bin/tar cf /tmp/etc_bup.tar /etc", name="backup_etc", day_of_week=0, hour=1, minute=0, ) """ def comma_sep(value): if isinstance(value, (list, tuple)): return ",".join("{0}".format(v) for v in value) return value minute = comma_sep(minute) hour = comma_sep(hour) month = comma_sep(month) day_of_week = comma_sep(day_of_week) day_of_month = comma_sep(day_of_month) crontab = host.get_fact(Crontab, user=user) name_comment = "# pyinfra-name={0}".format(cron_name) existing_crontab = crontab.get(command) existing_crontab_command = command existing_crontab_match = command if not existing_crontab and cron_name: # find the crontab by name if provided for cmd, details in crontab.items(): if name_comment in details["comments"]: existing_crontab = details existing_crontab_match = cmd existing_crontab_command = cmd exists = existing_crontab is not None edit_commands = [] temp_filename = state.get_temp_filename() if special_time: new_crontab_line = "{0} {1}".format(special_time, command) else: new_crontab_line = "{minute} {hour} {day_of_month} {month} {day_of_week} {command}".format( minute=minute, hour=hour, day_of_month=day_of_month, month=month, day_of_week=day_of_week, command=command, ) existing_crontab_match = ".*{0}.*".format(existing_crontab_match) # Don't want the cron and it does exist? Remove the line if not present and exists: edit_commands.append( sed_replace( temp_filename, existing_crontab_match, "", interpolate_variables=interpolate_variables, ), ) # Want the cron but it doesn't exist? Append the line elif present and not exists: if cron_name: if crontab: # append a blank line if cron entries already exist edit_commands.append("echo '' >> {0}".format(temp_filename)) edit_commands.append( "echo {0} >> {1}".format( shlex.quote(name_comment), temp_filename, ), ) edit_commands.append( "echo {0} >> {1}".format( shlex.quote(new_crontab_line), temp_filename, ), ) # We have the cron and it exists, do it's details? If not, replace the line elif present and exists: if any(( special_time != existing_crontab.get("special_time"), minute != existing_crontab.get("minute"), hour != existing_crontab.get("hour"), month != existing_crontab.get("month"), day_of_week != existing_crontab.get("day_of_week"), day_of_month != existing_crontab.get("day_of_month"), existing_crontab_command != command, ), ): edit_commands.append( sed_replace( temp_filename, existing_crontab_match, new_crontab_line, interpolate_variables=interpolate_variables, ), ) if edit_commands: crontab_args = [] if user: crontab_args.append("-u {0}".format(user)) # List the crontab into a temporary file if it exists if crontab: yield "crontab -l {0} > {1}".format(" ".join(crontab_args), temp_filename) # Now yield any edits for edit_command in edit_commands: yield edit_command # Finally, use the tempfile to write a new crontab yield "crontab {0} {1}".format(" ".join(crontab_args), temp_filename) # Update the crontab fact if present: crontab[command] = { "special_time": special_time, "minute": minute, "hour": hour, "month": month, "day_of_week": day_of_week, "day_of_month": day_of_month, "comments": [cron_name] if cron_name else [], } else: crontab.pop(command) else: host.noop( "crontab {0} {1}".format( command, "exists" if present else "does not exist", ), )
def directory( path, present=True, assume_present=False, user=None, group=None, mode=None, recursive=False, force=False, force_backup=True, force_backup_dir=None, _no_check_owner_mode=False, _no_fail_on_link=False, ): """ Add/remove/update directories. + path: path of the remote folder + present: whether the folder should exist + assume_present: whether to assume the directory exists + user: user to own the folder + group: group to own the folder + mode: permissions of the folder + recursive: recursively apply user/group/mode + force: if the target exists and is not a file, move or remove it and continue + force_backup: set to ``False`` to remove any existing non-file when ``force=True`` + force_backup_dir: directory to move any backup to when ``force=True`` **Examples:** .. code:: python files.directory( name="Ensure the /tmp/dir_that_we_want_removed is removed", path="/tmp/dir_that_we_want_removed", present=False, ) files.directory( name="Ensure /web exists", path="/web", user="******", group="myweb", ) # Multiple directories for dir in ["/netboot/tftp", "/netboot/nfs"]: files.directory( name="Ensure the directory `{}` exists".format(dir), path=dir, ) """ path = _validate_path(path) mode = ensure_mode_int(mode) info = host.get_fact(Directory, path=path) # Not a directory?! if info is False: if _no_fail_on_link and host.get_fact(Link, path=path): host.noop("directory {0} already exists (as a link)".format(path)) return yield from _raise_or_remove_invalid_path( "directory", path, force, force_backup, force_backup_dir, ) info = None # Doesn't exist & we want it if not assume_present and info is None and present: yield StringCommand("mkdir", "-p", QuoteString(path)) if mode: yield file_utils.chmod(path, mode, recursive=recursive) if user or group: yield file_utils.chown(path, user, group, recursive=recursive) host.create_fact( Directory, kwargs={"path": path}, data={ "mode": mode, "group": group, "user": user }, ) # It exists and we don't want it elif (assume_present or info) and not present: yield StringCommand("rm", "-rf", QuoteString(path)) host.delete_fact(Directory, kwargs={"path": path}) # It exists & we want to ensure its state elif (assume_present or info) and present: if assume_present and not info: info = {"mode": None, "group": None, "user": None} host.create_fact(Directory, kwargs={"path": path}, data=info) if _no_check_owner_mode: return changed = False if mode and (not info or info["mode"] != mode): yield file_utils.chmod(path, mode, recursive=recursive) info["mode"] = mode changed = True if ((not info and (user or group)) or (user and info["user"] != user) or (group and info["group"] != group)): yield file_utils.chown(path, user, group, recursive=recursive) changed = True if user: info["user"] = user if group: info["group"] = group if not changed: host.noop("directory {0} already exists".format(path))
def file( path, present=True, assume_present=False, user=None, group=None, mode=None, touch=False, create_remote_dir=True, force=False, force_backup=True, force_backup_dir=None, ): """ Add/remove/update files. + path: name/path of the remote file + present: whether the file should exist + assume_present: whether to assume the file exists + user: user to own the files + group: group to own the files + mode: permissions of the files as an integer, eg: 755 + touch: whether to touch the file + create_remote_dir: create the remote directory if it doesn't exist + force: if the target exists and is not a file, move or remove it and continue + force_backup: set to ``False`` to remove any existing non-file when ``force=True`` + force_backup_dir: directory to move any backup to when ``force=True`` ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. **Example:** .. code:: python # Note: The directory /tmp/secret will get created with the default umask. files.file( name="Create /tmp/secret/file", path="/tmp/secret/file", mode="600", user="******", group="root", touch=True, create_remote_dir=True, ) """ path = _validate_path(path) mode = ensure_mode_int(mode) info = host.get_fact(File, path=path) # Not a file?! if info is False: yield from _raise_or_remove_invalid_path( "file", path, force, force_backup, force_backup_dir, ) info = None # Doesn't exist & we want it if not assume_present and info is None and present: if create_remote_dir: yield from _create_remote_dir(state, host, path, user, group) yield StringCommand("touch", QuoteString(path)) if mode: yield file_utils.chmod(path, mode) if user or group: yield file_utils.chown(path, user, group) host.create_fact( File, kwargs={"path": path}, data={ "mode": mode, "group": group, "user": user }, ) # It exists and we don't want it elif (assume_present or info) and not present: yield StringCommand("rm", "-f", QuoteString(path)) host.delete_fact(File, kwargs={"path": path}) # It exists & we want to ensure its state elif (assume_present or info) and present: if assume_present and not info: info = {"mode": None, "group": None, "user": None} host.create_fact(File, kwargs={"path": path}, data=info) changed = False if touch: changed = True yield StringCommand("touch", QuoteString(path)) # Check mode if mode and (not info or info["mode"] != mode): yield file_utils.chmod(path, mode) info["mode"] = mode changed = True # Check user/group if ((not info and (user or group)) or (user and info["user"] != user) or (group and info["group"] != group)): yield file_utils.chown(path, user, group) changed = True if user: info["user"] = user if group: info["group"] = group if not changed: host.noop("file {0} already exists".format(path))
def link( path, target=None, present=True, assume_present=False, user=None, group=None, symbolic=True, create_remote_dir=True, force=False, force_backup=True, force_backup_dir=None, ): """ Add/remove/update links. + path: the name of the link + target: the file/directory the link points to + present: whether the link should exist + assume_present: whether to assume the link exists + user: user to own the link + group: group to own the link + symbolic: whether to make a symbolic link (vs hard link) + create_remote_dir: create the remote directory if it doesn't exist + force: if the target exists and is not a file, move or remove it and continue + force_backup: set to ``False`` to remove any existing non-file when ``force=True`` + force_backup_dir: directory to move any backup to when ``force=True`` ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Source changes: If the link exists and points to a different target, pyinfra will remove it and recreate a new one pointing to then new target. **Examples:** .. code:: python files.link( name="Create link /etc/issue2 that points to /etc/issue", path="/etc/issue2", target="/etc/issue", ) # Complex example demonstrating the assume_present option from pyinfra.operations import apt, files install_nginx = apt.packages( name="Install nginx", packages=["nginx"], ) files.link( name="Remove default nginx site", path="/etc/nginx/sites-enabled/default", present=False, assume_present=install_nginx.changed, ) """ path = _validate_path(path) if present and not target: raise OperationError("If present is True target must be provided") info = host.get_fact(Link, path=path) # Not a link? if info is False: yield from _raise_or_remove_invalid_path( "link", path, force, force_backup, force_backup_dir, ) info = None add_args = ["ln"] if symbolic: add_args.append("-s") add_cmd = StringCommand(" ".join(add_args), QuoteString(target), QuoteString(path)) remove_cmd = StringCommand("rm", "-f", QuoteString(path)) # No link and we want it if not assume_present and info is None and present: if create_remote_dir: yield from _create_remote_dir(state, host, path, user, group) yield add_cmd if user or group: yield file_utils.chown(path, user, group, dereference=False) host.create_fact( Link, kwargs={"path": path}, data={ "link_target": target, "group": group, "user": user }, ) # It exists and we don't want it elif (assume_present or info) and not present: yield remove_cmd host.delete_fact(Link, kwargs={"path": path}) # Exists and want to ensure it's state elif (assume_present or info) and present: if assume_present and not info: info = {"link_target": None, "group": None, "user": None} host.create_fact(Link, kwargs={"path": path}, data=info) # If we have an absolute path - prepend to any non-absolute values from the fact # and/or the source. if os.path.isabs(path): link_dirname = os.path.dirname(path) if not os.path.isabs(target): target = os.path.normpath(unix_path_join(link_dirname, target)) if info and not os.path.isabs(info["link_target"]): info["link_target"] = os.path.normpath( unix_path_join(link_dirname, info["link_target"]), ) changed = False # If the target is wrong, remove & recreate the link if not info or info["link_target"] != target: changed = True yield remove_cmd yield add_cmd info["link_target"] = target # Check user/group if ((not info and (user or group)) or (user and info["user"] != user) or (group and info["group"] != group)): yield file_utils.chown(path, user, group, dereference=False) changed = True if user: info["user"] = user if group: info["group"] = group if not changed: host.noop("link {0} already exists".format(path))
def link( path, target=None, present=True, assume_present=False, user=None, group=None, symbolic=True, force=True, create_remote_dir=True, ): """ Add/remove/update links. + path: the name of the link + target: the file/directory the link points to + present: whether the link should exist + assume_present: whether to assume the link exists + user: user to own the link + group: group to own the link + symbolic: whether to make a symbolic link (vs hard link) + create_remote_dir: create the remote directory if it doesn't exist ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Source changes: If the link exists and points to a different target, pyinfra will remove it and recreate a new one pointing to then new target. **Examples:** .. code:: python # simple example showing how to link to a file files.link( name=r"Create link C:\\issue2 that points to C:\\issue", path=r"C:\\issue2", target=r"C\\issue", ) """ _validate_path(path) if present and not target: raise OperationError("If present is True target must be provided") info = host.get_fact(Link, path=path) # Not a link? if info is not None and not info: raise OperationError("{0} exists and is not a link".format(path)) add_cmd = "New-Item -ItemType {0} -Path {1} -Target {2} {3}".format( "SymbolicLink" if symbolic else "HardLink", path, target, "-Force" if force else "", ) remove_cmd = "(Get-Item {0}).Delete()".format(path) # We will attempt to link regardless of current existence # since we know by now the path is either a link already # or does not exist if (info is None or force) and present: if create_remote_dir: yield from _create_remote_dir(state, host, path, user, group) yield add_cmd # if user or group: # yield chown(path, user, group, dereference=False) # host.create_fact( # WindowsLink, # kwargs={'name': path}, # data={'link_target': target, 'group': group, 'user': user}, # ) # It exists and we don't want it elif (assume_present or info) and not present: yield remove_cmd # host.delete_fact(WindowsLink, kwargs={'name': path}) else: host.noop("link {0} already exists and force=False".format(path))
def rule( chain, jump, present=True, table="filter", append=True, version=4, # Core iptables filter arguments protocol=None, not_protocol=None, source=None, not_source=None, destination=None, not_destination=None, in_interface=None, not_in_interface=None, out_interface=None, not_out_interface=None, # After-rule arguments to_destination=None, to_source=None, to_ports=None, log_prefix=None, # Extras and extra shortcuts destination_port=None, source_port=None, extras="", ): """ Add/remove iptables rules. + chain: the chain this rule should live in + jump: the target of the rule + table: the iptables table this rule should belong to + append: whether to append or insert the rule (if not present) + version: whether to target iptables or ip6tables Iptables args: + protocol/not_protocol: filter by protocol (tcp or udp) + source/not_source: filter by source IPs + destination/not_destination: filter by destination IPs + in_interface/not_in_interface: filter by incoming interface + out_interface/not_out_interface: filter by outgoing interface + to_destination: where to route to when jump=DNAT + to_source: where to route to when jump=SNAT + to_ports: where to route to when jump=REDIRECT + log_prefix: prefix for the log of this rule when jump=LOG Extras: + extras: a place to define iptables extension arguments (eg --limit, --physdev) + destination_port: destination port (requires protocol) + source_port: source port (requires protocol) **Examples:** .. code:: python iptables.rule( name="Block SSH traffic", chain="INPUT", jump="DROP", destination_port=22, ) iptables.rule( name="NAT traffic on from 8.8.8.8:53 to 8.8.4.4:8080", chain="PREROUTING", jump="DNAT", table="nat", source="8.8.8.8", destination_port=53, to_destination="8.8.4.4:8080", ) """ if isinstance(to_ports, int): to_ports = "{0}".format(to_ports) # These are only shortcuts for extras if destination_port: extras = "{0} --dport {1}".format(extras, destination_port) if source_port: extras = "{0} --sport {1}".format(extras, source_port) # Convert the extras string into a set to enable comparison with the fact extras_set = set(extras.split()) # When protocol is set, the extension is automagically added by iptables (which shows # in iptables-save): http://ipset.netfilter.org/iptables-extensions.man.html if protocol: extras_set.add("-m") extras_set.add(protocol) # --dport and --sport do not work without a protocol (because they need -m [tcp|udp] elif destination_port or source_port: raise OperationError( "iptables cannot filter by destination_port/source_port without a protocol", ) # Verify NAT arguments, --to-destination only w/table=nat, jump=DNAT if to_destination and (table != "nat" or jump != "DNAT"): raise OperationError( "iptables only supports to_destination on the nat table and the DNAT jump " "(table={0}, jump={1})".format(table, jump), ) # As above, --to-source only w/table=nat, jump=SNAT if to_source and (table != "nat" or jump != "SNAT"): raise OperationError( "iptables only supports to_source on the nat table and the SNAT jump " "(table={0}, jump={1})".format(table, jump), ) # As above, --to-ports only w/table=nat, jump=REDIRECT if to_ports and (table != "nat" or jump != "REDIRECT"): raise OperationError( "iptables only supports to_ports on the nat table and the REDIRECT jump " "(table={0}, jump={1})".format(table, jump), ) # --log-prefix is only supported with jump=LOG if log_prefix and jump != "LOG": raise OperationError( "iptables only supports log_prefix with the LOG jump " "(jump={0})".format(jump), ) definition = { "chain": chain, "jump": jump, "protocol": protocol, "source": source, "destination": destination, "in_interface": in_interface, "out_interface": out_interface, "not_protocol": not_protocol, "not_source": not_source, "not_destination": not_destination, "not_in_interface": not_in_interface, "not_out_interface": not_out_interface, # These go *after* the jump argument "log_prefix": log_prefix, "to_destination": to_destination, "to_source": to_source, "to_ports": to_ports, "extras": extras_set, } definition = { key: ("{0}/32".format(value) if (key in ("source", "not_source", "destination", "not_destination") and "/" not in value) else value) for key, value in definition.items() if value } rules = (host.get_fact(IptablesRules, table=table) if version == 4 else host.get_fact(Ip6tablesRules, table=table)) action = None # Definition doesn't exist and we want it if present: if definition not in rules: action = "-A" if append else "-I" else: host.noop("iptables {0} rule exists".format(chain)) return # Definition exists and we don't want it if not present: if definition in rules: action = "-D" else: host.noop("iptables {0} rule does not exists".format(chain)) return # Are we adding/removing a rule? Lets build it if action: args = [ "iptables" if version == 4 else "ip6tables", # Add the table "-t", table, # Add the action and target chain action, chain, ] if protocol: args.extend(("-p", protocol)) if source: args.extend(("-s", source)) if in_interface: args.extend(("-i", in_interface)) if out_interface: args.extend(("-o", out_interface)) if not_protocol: args.extend(("!", "-p", not_protocol)) if not_source: args.extend(("!", "-s", not_source)) if not_in_interface: args.extend(("!", "-i", not_in_interface)) if not_out_interface: args.extend(("!", "-o", not_out_interface)) if extras: args.append(extras.strip()) # Add the jump args.extend(("-j", jump)) if log_prefix: args.extend(("--log-prefix", log_prefix)) if to_destination: args.extend(("--to-destination", to_destination)) if to_source: args.extend(("--to-source", to_source)) if to_ports: args.extend(("--to-ports", to_ports)) # Build the final iptables command yield " ".join(args) if action == "-D": rules.remove(definition) else: rules.append(definition)
def download( src, dest, user=None, group=None, mode=None, cache_time=None, force=False, sha256sum=None, sha1sum=None, md5sum=None, ): """ Download files from remote locations using curl or wget. + src: source URL of the file + dest: where to save the file + user: user to own the files + group: group to own the files + mode: permissions of the files + cache_time: if the file exists already, re-download after this time (in seconds) + force: always download the file, even if it already exists + sha256sum: sha256 hash to checksum the downloaded file against + sha1sum: sha1 hash to checksum the downloaded file against + md5sum: md5 hash to checksum the downloaded file against **Example:** .. code:: python winows_files.download( name="Download the Docker repo file", src="https://download.docker.com/linux/centos/docker-ce.repo", dest="C:\\docker", ) """ info = host.get_fact(File, path=dest) # Destination is a directory? if info is False: raise OperationError( "Destination {0} already exists and is not a file".format(dest), ) # Do we download the file? Force by default download = force # Doesn't exist, lets download it if info is None: download = True # Destination file exists & cache_time: check when the file was last modified, # download if old else: if cache_time: # Time on files is not tz-aware, and will be the same tz as the server's time, # so we can safely remove the tzinfo from Date before comparison. cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(seconds=cache_time) if info["mtime"] and info["mtime"] > cache_time: download = True if sha1sum: if sha1sum != host.get_fact(Sha1File, path=dest): download = True if sha256sum: if sha256sum != host.get_fact(Sha256File, path=dest): download = True if md5sum: if md5sum != host.get_fact(Md5File, path=dest): download = True # If we download, always do user/group/mode as SSH user may be different if download: yield ( '$ProgressPreference = "SilentlyContinue"; ' "Invoke-WebRequest -Uri {0} -OutFile {1}" ).format(src, dest) # if user or group: # yield chown(dest, user, group) # if mode: # yield chmod(dest, mode) if sha1sum: yield ( 'if ((Get-FileHash -Algorithm SHA1 "{0}").hash -ne {1}) {{ ' 'Write-Error "SHA1 did not match!" ' "}}" ).format(dest, sha1sum) if sha256sum: yield ( 'if ((Get-FileHash -Algorithm SHA256 "{0}").hash -ne {1}) {{ ' 'Write-Error "SHA256 did not match!" ' "}}" ).format(dest, sha256sum) if md5sum: yield ( 'if ((Get-FileHash -Algorithm MD5 "{0}").hash -ne {1}) {{ ' 'Write-Error "MD5 did not match!" ' "}}" ).format(dest, md5sum) else: host.noop("file {0} has already been downloaded".format(dest))
def put( src, dest, user=None, group=None, mode=None, add_deploy_dir=True, create_remote_dir=True, force=False, assume_exists=False, ): """ Upload a local file, or file-like object, to the remote system. + src: filename or IO-like object to upload + dest: remote filename to upload to + user: user to own the files + group: group to own the files + mode: permissions of the files, use ``True`` to copy the local file + add_deploy_dir: src is relative to the deploy directory + create_remote_dir: create the remote directory if it doesn't exist + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists ``dest``: If this is a directory that already exists on the remote side, the local file will be uploaded to that directory with the same filename. ``mode``: When set to ``True`` the permissions of the local file are applied to the remote file after the upload is complete. ``create_remote_dir``: If the remote directory does not exist it will be created using the same user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. Note: This operation is not suitable for large files as it may involve copying the file before uploading it. **Examples:** .. code:: python files.put( name="Update the message of the day file", src="files/motd", dest="/etc/motd", mode="644", ) files.put( name="Upload a StringIO object", src=StringIO("file contents"), dest="/etc/motd", ) """ # Upload IO objects as-is if hasattr(src, "read"): local_file = src local_sum = get_file_sha1(src) # Assume string filename else: # Add deploy directory? if add_deploy_dir and state.cwd: src = os.path.join(state.cwd, src) local_file = src if os.path.isfile(local_file): local_sum = get_file_sha1(local_file) elif assume_exists: local_sum = None else: raise IOError("No such file: {0}".format(local_file)) if mode is True: if os.path.isfile(local_file): mode = get_path_permissions_mode(local_file) else: logger.warning(( "No local file exists to get permissions from with `mode=True` ({0})" ).format(get_call_location(), ), ) else: mode = ensure_mode_int(mode) remote_file = host.get_fact(File, path=dest) if not remote_file and host.get_fact(Directory, path=dest): dest = unix_path_join(dest, os.path.basename(src)) remote_file = host.get_fact(File, path=dest) if create_remote_dir: yield from _create_remote_dir(state, host, dest, user, group) # No remote file, always upload and user/group/mode if supplied if not remote_file or force: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) # File exists, check sum and check user/group/mode if supplied else: remote_sum = host.get_fact(Sha1File, path=dest) # Check sha1sum, upload if needed if local_sum != remote_sum: yield FileUploadCommand( local_file, dest, remote_temp_filename=state.get_temp_filename(dest), ) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) else: changed = False # Check mode if mode and remote_file["mode"] != mode: yield file_utils.chmod(dest, mode) changed = True # Check user/group if (user and remote_file["user"] != user) or ( group and remote_file["group"] != group): yield file_utils.chown(dest, user, group) changed = True if not changed: host.noop("file {0} is already uploaded".format(dest)) # Now we've uploaded the file and ensured user/group/mode, update the relevant fact data host.create_fact(Sha1File, kwargs={"path": dest}, data=local_sum) host.create_fact( File, kwargs={"path": dest}, data={ "user": user, "group": group, "mode": mode }, )
def replace( path, text=None, replace=None, flags=None, backup=False, interpolate_variables=False, match=None, # deprecated ): """ Replace contents of a file using ``sed``. + path: target remote file to edit + text: text/regex to match against + replace: text to replace with + flags: list of flags to pass to sed + backup: whether to backup the file (see below) + interpolate_variables: whether to interpolate variables in ``replace`` Backup: If set to ``True``, any editing of the file will place an old copy with the ISO date (taken from the machine running ``pyinfra``) appended as the extension. If you pass a string value this will be used as the extension of the backed up file. **Example:** .. code:: python files.replace( name="Change part of a line in a file", path="/etc/motd", text="verboten", replace="forbidden", ) """ if text is None and match: text = match logger.warning( ("The `match` argument has been replaced by " "`text` in the `files.replace` operation ({0})").format( get_call_location()), ) if text is None: raise TypeError( "Missing argument `text` required in `files.replace` operation") if replace is None: raise TypeError( "Missing argument `replace` required in `files.replace` operation") existing_lines = host.get_fact( FindInFile, path=path, pattern=text, interpolate_variables=interpolate_variables, ) # Only do the replacement if the file does not exist (it may be created earlier) # or we have matching lines. if existing_lines is None or existing_lines: yield sed_replace( path, text, replace, flags=flags, backup=backup, interpolate_variables=interpolate_variables, ) host.create_fact( FindInFile, kwargs={ "path": path, "pattern": text, "interpolate_variables": interpolate_variables, }, data=[], ) else: host.noop('string "{0}" does not exist in {1}'.format(text, path))
def download( hostname, filename, local_filename=None, force=False, port=22, user=None, ssh_keyscan=False, ): """ Download files from other servers using ``scp``. + hostname: hostname to upload to + filename: file to download + local_filename: where to download the file to (defaults to ``filename``) + force: always download the file, even if present locally + port: connect to this port + user: connect with this user + ssh_keyscan: execute ``ssh.keyscan`` before uploading the file """ local_filename = local_filename or filename # Get local file info local_file_info = host.get_fact(File, path=local_filename) # Local file exists but isn't a file? if local_file_info is False: raise OperationError( "Local destination {0} already exists and is not a file".format( local_filename, ), ) # If the local file exists and we're not forcing a re-download, no-op if local_file_info and not force: host.noop("file {0} is already downloaded".format(filename)) return # Figure out where we're connecting (host or user@host) connection_target = hostname if user: connection_target = "@".join((user, hostname)) if ssh_keyscan: yield from keyscan(hostname) # Download the file with scp yield "scp -P {0} {1}:{2} {3}".format( port, connection_target, filename, local_filename, ) host.create_fact( File, kwargs={"path": local_filename}, data={ "mode": None, "group": None, "user": user, "mtime": None }, )
def line( path, line, present=True, replace=None, flags=None, backup=False, interpolate_variables=False, escape_regex_characters=False, assume_present=False, ): """ Ensure lines in files using grep to locate and sed to replace. + path: target remote file to edit + line: string or regex matching the target line + present: whether the line should be in the file + replace: text to replace entire matching lines when ``present=True`` + flags: list of flags to pass to sed when replacing/deleting + backup: whether to backup the file (see below) + interpolate_variables: whether to interpolate variables in ``replace`` + assume_present: whether to assume a matching line already exists in the file + escape_regex_characters: whether to escape regex characters from the matching line Regex line matching: Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To change bits of lines, see ``files.replace``. Regex line escaping: If matching special characters (eg a crontab line containing *), remember to escape it first using Python's ``re.escape``. Backup: If set to ``True``, any editing of the file will place an old copy with the ISO date (taken from the machine running ``pyinfra``) appended as the extension. If you pass a string value this will be used as the extension of the backed up file. Append: If ``line`` is not in the file but we want it (``present`` set to ``True``), then it will be append to the end of the file. **Examples:** .. code:: python # prepare to do some maintenance maintenance_line = "SYSTEM IS DOWN FOR MAINTENANCE" files.line( name="Add the down-for-maintence line in /etc/motd", path="/etc/motd", line=maintenance_line, ) # Then, after the mantenance is done, remove the maintenance line files.line( name="Remove the down-for-maintenance line in /etc/motd", path="/etc/motd", line=maintenance_line, replace="", present=False, ) # example where there is '*' in the line files.line( name="Ensure /netboot/nfs is in /etc/exports", path="/etc/exports", line=r"/netboot/nfs .*", replace="/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash," "insecure,no_subtree_check)", ) files.line( name="Ensure myweb can run /usr/bin/python3 without password", path="/etc/sudoers", line=r"myweb .*", replace="myweb ALL=(ALL) NOPASSWD: /usr/bin/python3", ) # example when there are double quotes (") line = 'QUOTAUSER=""' files.line( name="Example with double quotes (")", path="/etc/adduser.conf", line="^{}$".format(line), replace=line, ) """ match_line = line if escape_regex_characters: match_line = re.sub(r"([\.\\\+\*\?\[\^\]\$\(\)\{\}\-])", r"\\\1", match_line) # Ensure we're matching a whole line, note: match may be a partial line so we # put any matches on either side. if not match_line.startswith("^"): match_line = "^.*{0}".format(match_line) if not match_line.endswith("$"): match_line = "{0}.*$".format(match_line) # Is there a matching line in this file? if assume_present: present_lines = [line] else: present_lines = host.get_fact( FindInFile, path=path, pattern=match_line, interpolate_variables=interpolate_variables, ) # If replace present, use that over the matching line if replace: line = replace # We must provide some kind of replace to sed_replace_command below else: replace = "" # Save commands for re-use in dynamic script when file not present at fact stage echo_command = make_formatted_string_command( "echo {0} >> {1}", '"{0}"'.format(line) if interpolate_variables else QuoteString(line), QuoteString(path), ) if backup: backup_filename = "{0}.{1}".format(path, get_timestamp()) echo_command = StringCommand( make_formatted_string_command( "cp {0} {1} && ", QuoteString(path), QuoteString(backup_filename), ), echo_command, ) sed_replace_command = sed_replace( path, match_line, replace, flags=flags, backup=backup, interpolate_variables=interpolate_variables, ) # No line and we want it, append it if not present_lines and present: # If the file does not exist - it *might* be created, so we handle it # dynamically with a little script. if present_lines is None: yield make_formatted_string_command( """ if [ -f '{target}' ]; then ( grep {match_line} '{target}' && \ {sed_replace_command}) 2> /dev/null || \ {echo_command} ; else {echo_command} ; fi """, target=QuoteString(path), match_line=QuoteString(match_line), echo_command=echo_command, sed_replace_command=sed_replace_command, ) # File exists but has no matching lines - append it. else: # If we're doing replacement, only append if the *replacement* line # does not exist (as we are appending the replacement). if replace: # Ensure replace explicitly matches a whole line replace_line = replace if not replace_line.startswith("^"): replace_line = f"^{replace_line}" if not replace_line.endswith("$"): replace_line = f"{replace_line}$" present_lines = host.get_fact( FindInFile, path=path, pattern=replace_line, interpolate_variables=interpolate_variables, ) if not present_lines: yield echo_command else: host.noop('line "{0}" exists in {1}'.format( replace or line, path)) host.create_fact( FindInFile, kwargs={ "path": path, "pattern": match_line, "interpolate_variables": interpolate_variables, }, data=[replace or line], ) # Line(s) exists and we want to remove them, replace with nothing elif present_lines and not present: yield sed_replace( path, match_line, "", flags=flags, backup=backup, interpolate_variables=interpolate_variables, ) host.delete_fact( FindInFile, kwargs={ "path": path, "pattern": match_line, "interpolate_variables": interpolate_variables, }, ) # Line(s) exists and we have want to ensure they're correct elif present_lines and present: # If any of lines are different, sed replace them if replace and any(line != replace for line in present_lines): yield sed_replace_command del present_lines[:] # TODO: use .clear() when py3+ present_lines.append(replace) else: host.noop('line "{0}" exists in {1}'.format(replace or line, path))
def download( src, dest, user=None, group=None, mode=None, cache_time=None, force=False, sha256sum=None, sha1sum=None, md5sum=None, headers=None, insecure=False, ): """ Download files from remote locations using ``curl`` or ``wget``. + src: source URL of the file + dest: where to save the file + user: user to own the files + group: group to own the files + mode: permissions of the files + cache_time: if the file exists already, re-download after this time (in seconds) + force: always download the file, even if it already exists + sha256sum: sha256 hash to checksum the downloaded file against + sha1sum: sha1 hash to checksum the downloaded file against + md5sum: md5 hash to checksum the downloaded file against + headers: optional dictionary of headers to set for the HTTP request + insecure: disable SSL verification for the HTTP request **Example:** .. code:: python files.download( name="Download the Docker repo file", src="https://download.docker.com/linux/centos/docker-ce.repo", dest="/etc/yum.repos.d/docker-ce.repo", ) """ info = host.get_fact(File, path=dest) host_datetime = host.get_fact(Date).replace(tzinfo=None) # Destination is a directory? if info is False: raise OperationError( "Destination {0} already exists and is not a file".format(dest), ) # Do we download the file? Force by default download = force # Doesn't exist, lets download it if info is None: download = True # Destination file exists & cache_time: check when the file was last modified, # download if old else: if cache_time: # Time on files is not tz-aware, and will be the same tz as the server's time, # so we can safely remove the tzinfo from the Date fact before comparison. cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta( seconds=cache_time) if info["mtime"] and info["mtime"] < cache_time: download = True if sha1sum: if sha1sum != host.get_fact(Sha1File, path=dest): download = True if sha256sum: if sha256sum != host.get_fact(Sha256File, path=dest): download = True if md5sum: if md5sum != host.get_fact(Md5File, path=dest): download = True # If we download, always do user/group/mode as SSH user may be different if download: temp_file = state.get_temp_filename(dest) curl_args = ["-sSLf"] wget_args = ["-q"] if insecure: curl_args.append("--insecure") wget_args.append("--no-check-certificate") if headers: for key, value in headers.items(): header_arg = StringCommand("--header", QuoteString(f"{key}: {value}")) curl_args.append(header_arg) wget_args.append(header_arg) curl_command = make_formatted_string_command( "curl {0} {1} -o {2}", StringCommand(*curl_args), QuoteString(src), QuoteString(temp_file), ) wget_command = make_formatted_string_command( "wget {0} {1} -O {2} || ( rm -f {2} ; exit 1 )", StringCommand(*wget_args), QuoteString(src), QuoteString(temp_file), ) if host.get_fact(Which, command="curl"): yield curl_command elif host.get_fact(Which, command="wget"): yield wget_command else: yield "( {0} ) || ( {1} )".format(curl_command, wget_command) yield StringCommand("mv", QuoteString(temp_file), QuoteString(dest)) if user or group: yield file_utils.chown(dest, user, group) if mode: yield file_utils.chmod(dest, mode) if sha1sum: yield make_formatted_string_command( ("(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) " "|| ( echo {2} && exit 1 )"), QuoteString(dest), sha1sum, QuoteString("SHA1 did not match!"), ) if sha256sum: yield make_formatted_string_command( ("(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) " "| grep {1}) || ( echo {2} && exit 1 )"), QuoteString(dest), sha256sum, QuoteString("SHA256 did not match!"), ) if md5sum: yield make_formatted_string_command( ("(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) " "|| ( echo {2} && exit 1 )"), QuoteString(dest), md5sum, QuoteString("MD5 did not match!"), ) host.create_fact( File, kwargs={"path": dest}, data={ "mode": mode, "group": group, "user": user, "mtime": host_datetime }, ) # Remove any checksum facts as we don't know the correct values for value, fact_cls in ( (sha1sum, Sha1File), (sha256sum, Sha256File), (md5sum, Md5File), ): fact_kwargs = {"path": dest} if value: host.create_fact(fact_cls, kwargs=fact_kwargs, data=value) else: host.delete_fact(fact_cls, kwargs=fact_kwargs) else: host.noop("file {0} has already been downloaded".format(dest))