def command(
self,
# Details for speaking to MySQL via `mysql` CLI via `mysql` CLI
mysql_user=None,
mysql_password=None,
mysql_host=None,
mysql_port=None,
):
mysql_command = make_execute_mysql_command(
self.mysql_command,
user=mysql_user,
password=mysql_password,
host=mysql_host,
port=mysql_port,
)
return StringCommand(
'which',
'mysql',
'>',
'/dev/null',
'&&',
StringCommand('(', mysql_command, ')', separator=''),
'||',
'true',
)
def test_nested_deploy(self):
inventory = make_inventory()
somehost = inventory.get_host("somehost")
state = State(inventory, Config())
# Enable printing on this test to catch any exceptions in the formatting
state.print_output = True
state.print_input = True
state.print_fact_info = True
state.print_noop_info = True
connect_all(state)
@deploy
def test_nested_deploy():
server.shell(commands=["echo nested command"])
@deploy
def test_deploy():
server.shell(commands=["echo first command"])
test_nested_deploy()
server.shell(commands=["echo second command"])
add_deploy(state, test_deploy)
op_order = state.get_op_order()
# Ensure we have an op
assert len(op_order) == 3
first_op_hash = op_order[0]
assert state.op_meta[first_op_hash]["names"] == {
"test_deploy | Server/Shell"
}
assert state.ops[somehost][first_op_hash]["commands"] == [
StringCommand("echo first command"),
]
second_op_hash = op_order[1]
assert state.op_meta[second_op_hash]["names"] == {
"test_deploy | test_nested_deploy | Server/Shell",
}
assert state.ops[somehost][second_op_hash]["commands"] == [
StringCommand("echo nested command"),
]
third_op_hash = op_order[2]
assert state.op_meta[third_op_hash]["names"] == {
"test_deploy | Server/Shell"
}
assert state.ops[somehost][third_op_hash]["commands"] == [
StringCommand("echo second command"),
]
def _raise_or_remove_invalid_path(fs_type, path, force, force_backup,
force_backup_dir):
if force:
if force_backup:
backup_path = "{0}.{1}".format(path, get_timestamp())
if force_backup_dir:
backup_path = os.path.basename(backup_path)
backup_path = "{0}/{1}".format(force_backup_dir, backup_path)
yield StringCommand("mv", QuoteString(path),
QuoteString(backup_path))
else:
yield StringCommand("rm", "-rf", QuoteString(path))
else:
raise OperationError("{0} exists and is not a {1}".format(
path, fs_type))
def load(
state, host,
remote_filename, database=None,
# Details for speaking to PostgreSQL via `psql` CLI
postgresql_user=None, postgresql_password=None,
postgresql_host=None, postgresql_port=None,
):
'''
Load ``.sql`` file into a database.
+ database: name of the database to import into
+ remote_filename: the filename to read from
+ postgresql_*: global module arguments, see above
Example:
.. code:: python
postgresql.load(
{'Import the pyinfra_stuff dump into pyinfra_stuff_copy'},
'/tmp/pyinfra_stuff.dump',
database='pyinfra_stuff_copy',
sudo_user='******',
)
'''
yield StringCommand(make_psql_command(
database=database,
user=postgresql_user,
password=postgresql_password,
host=postgresql_host,
port=postgresql_port,
), '<', remote_filename)
def make_mysql_command(
database=None,
user=None,
password=None,
host=None,
port=None,
executable="mysql",
):
target_bits = [executable]
if database:
target_bits.append(database)
if user:
# Quote the username as in may contain special characters
target_bits.append('-u"{0}"'.format(user))
if password:
# Quote the password as it may contain special characters
target_bits.append(MaskString('-p"{0}"'.format(password)))
if host:
target_bits.append("-h{0}".format(host))
if port:
target_bits.append("-P{0}".format(port))
return StringCommand(*target_bits)
def dump(
state, host,
remote_filename, database=None,
# Details for speaking to PostgreSQL via `psql` CLI
postgresql_user=None, postgresql_password=None,
postgresql_host=None, postgresql_port=None,
):
'''
Dump a PostgreSQL database into a ``.sql`` file. Requires ``pg_dump``.
+ database: name of the database to dump
+ remote_filename: name of the file to dump the SQL to
+ postgresql_*: global module arguments, see above
Example:
.. code:: python
postgresql.dump(
{'Dump the pyinfra_stuff database'},
'/tmp/pyinfra_stuff.dump',
database='pyinfra_stuff',
sudo_user='******',
)
'''
yield StringCommand(make_psql_command(
executable='pg_dump',
database=database,
user=postgresql_user,
password=postgresql_password,
host=postgresql_host,
port=postgresql_port,
), '>', remote_filename)
def test_run_shell_command_masked(self, fake_click):
inventory = make_inventory(hosts=('@local', ))
state = State(inventory, Config())
host = inventory.get_host('@local')
command = StringCommand('echo', MaskString('top-secret-stuff'))
self.fake_popen_mock().returncode = 0
out = host.run_shell_command(state,
command,
print_output=True,
print_input=True)
assert len(out) == 3
status, stdout, stderr = out
assert status is True
self.fake_popen_mock.assert_called_with(
"sh -c 'echo top-secret-stuff'",
shell=True,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
)
fake_click.echo.assert_called_with(
"{0}>>> sh -c 'echo ***'".format(host.print_prefix), )
def make_psql_command(
database=None,
user=None,
password=None,
host=None,
port=None,
executable='psql',
):
target_bits = []
if password:
target_bits.append(MaskString('PGPASSWORD="******"'.format(password)))
target_bits.append(executable)
if database:
target_bits.append('-d {0}'.format(database))
if user:
target_bits.append('-U {0}'.format(user))
if host:
target_bits.append('-h {0}'.format(host))
if port:
target_bits.append('-p {0}'.format(port))
return StringCommand(*target_bits)
def make_execute_psql_command(command, **postgresql_kwargs):
return StringCommand(
make_psql_command(**postgresql_kwargs),
'-Ac',
QuoteString(
command), # quote this whole item as a single shell argument
)
def test_run_shell_command_masked(self, fake_ssh_client, fake_click):
fake_ssh = MagicMock()
fake_stdout = MagicMock()
fake_ssh.exec_command.return_value = MagicMock(
), fake_stdout, MagicMock()
fake_ssh_client.return_value = fake_ssh
inventory = make_inventory(hosts=("somehost", ))
State(inventory, Config())
host = inventory.get_host("somehost")
host.connect()
command = StringCommand("echo", MaskString("top-secret-stuff"))
fake_stdout.channel.recv_exit_status.return_value = 0
out = host.run_shell_command(command,
print_output=True,
print_input=True)
assert len(out) == 3
status, stdout, stderr = out
assert status is True
fake_ssh.exec_command.assert_called_with(
"sh -c 'echo top-secret-stuff'",
get_pty=False,
)
fake_click.echo.assert_called_with(
"{0}>>> sh -c 'echo ***'".format(host.print_prefix),
err=True,
)
def update(
state: Optional[State] = None,
host: Optional[Host] = None
) -> Generator[StringCommand, None, None]: # noqa
"""Upgrade system via PackageKit console client."""
yield StringCommand("pkcon update --plain --noninteractive",
success_exit_codes=[0, 5])
def chmod(target, mode, recursive=False):
args = ["chmod"]
if recursive:
args.append("-R")
args.append("{0}".format(mode))
return StringCommand(" ".join(args), QuoteString(target))
def parse_commands(commands):
json_commands = []
for command in commands:
if isinstance(command,
six.string_types): # matches pyinfra/api/operation.py
command = StringCommand(command.strip())
if isinstance(command, StringCommand):
json_command = get_command_string(command)
elif isinstance(command, dict):
command['command'] = get_command_string(command['command']).strip()
json_command = command
elif isinstance(command, FunctionCommand):
func_name = (command.function if command.function == '__func__'
else command.function.__name__)
json_command = [
func_name,
list(command.args),
command.kwargs,
]
elif isinstance(command, FileUploadCommand):
if hasattr(command.src, 'read'):
command.src.seek(0)
data = command.src.read()
else:
data = command.src
json_command = ['upload', data, command.dest]
elif isinstance(command, FileDownloadCommand):
json_command = ['download', command.src, command.dest]
else:
raise Exception('{0} is not a valid command!'.format(command))
if command.executor_kwargs:
command.executor_kwargs['command'] = json_command
json_command = command.executor_kwargs
json_commands.append(json_command)
return json_commands
def parse_commands(commands):
json_commands = []
for command in commands:
if isinstance(command, str): # matches pyinfra/api/operation.py
command = StringCommand(command.strip())
if isinstance(command, StringCommand):
json_command = get_command_string(command)
elif isinstance(command, dict):
command["command"] = get_command_string(command["command"]).strip()
json_command = command
elif isinstance(command, FunctionCommand):
func_name = (command.function if command.function == "__func__"
else command.function.__name__)
json_command = [
func_name,
list(command.args),
command.kwargs,
]
elif isinstance(command, FileUploadCommand):
if hasattr(command.src, "read"):
command.src.seek(0)
data = command.src.read()
else:
data = str(command.src)
json_command = ["upload", data, str(command.dest)]
elif isinstance(command, FileDownloadCommand):
json_command = ["download", str(command.src), str(command.dest)]
else:
raise Exception("{0} is not a valid command!".format(command))
if command.executor_kwargs:
command.executor_kwargs["command"] = json_command
json_command = command.executor_kwargs
json_commands.append(json_command)
return json_commands
def make_execute_mysql_command(command, ignore_errors=False, **mysql_kwargs):
commands_bits = [
make_mysql_command(**mysql_kwargs),
"-Be",
QuoteString(command), # quote this whole item as a single shell argument
]
if ignore_errors:
commands_bits.extend(["||", "true"])
return StringCommand(*commands_bits)
def jsontest_function(self, test_name, test_data, fact=fact):
short_fact = None
if isinstance(fact, ShortFactBase):
short_fact = fact
fact = fact.fact()
test_args = test_data.get("arg", [])
command = _make_command(fact.command, test_args)
if "command" in test_data:
assert get_command_string(
StringCommand(command)) == test_data["command"]
else:
warnings.warn(
'No command set for test: {0} (got "{1}")'.format(
test_name,
command,
), )
requires_command = _make_command(fact.requires_command, test_args)
if requires_command:
if "requires_command" in test_data:
assert requires_command == test_data["requires_command"]
else:
warnings.warn(
'No requires command set for test: {0} (got "{1}")'.
format(
test_name,
requires_command,
), )
data = fact.process(test_data["output"])
if short_fact:
data = short_fact.process_data(data)
# Encode/decode data to ensure datetimes/etc become JSON
data = json.loads(json.dumps(data, default=json_encode))
try:
assert data == test_data["fact"]
except AssertionError as e:
print()
print("--> GOT:\n",
json.dumps(data, indent=4, default=json_encode))
print(
"--> WANT:",
json.dumps(
test_data["fact"],
indent=4,
default=json_encode,
),
)
raise e
def sed_replace(
filename,
line,
replace,
flags=None,
backup=False,
interpolate_variables=False,
):
flags = "".join(flags) if flags else ""
line = line.replace("/", r"\/")
replace = str(replace)
replace = replace.replace("/", r"\/")
backup_extension = get_timestamp()
if interpolate_variables:
line = line.replace('"', '\\"')
replace = replace.replace('"', '\\"')
sed_script_formatter = '"s/{0}/{1}/{2}"'
else:
# Single quotes cannot contain other single quotes, even when escaped , so turn
# each ' into '"'"' (end string, double quote the single quote, (re)start string)
line = line.replace("'", "'\"'\"'")
replace = replace.replace("'", "'\"'\"'")
sed_script_formatter = "'s/{0}/{1}/{2}'"
sed_script = sed_script_formatter.format(line, replace, flags)
sed_command = StringCommand(
"sed",
"-i.{0}".format(backup_extension),
sed_script,
QuoteString(filename),
)
if not backup: # if we're not backing up, remove the file *if* sed succeeds
backup_filename = "{0}.{1}".format(filename, backup_extension)
sed_command = StringCommand(sed_command, "&&", "rm", "-f",
QuoteString(backup_filename))
return sed_command
def run_shell_command(state,
host,
command,
get_pty=False,
timeout=None,
stdin=None,
success_exit_codes=None,
print_output=False,
print_input=False,
return_combined_output=False,
use_sudo_password=False,
**command_kwargs):
if use_sudo_password:
command_kwargs['use_sudo_password'] = get_sudo_password(
state,
host,
use_sudo_password,
run_shell_command=run_shell_command,
put_file=put_file,
)
chroot_directory = host.host_data['chroot_directory']
command = make_unix_command(command, **command_kwargs)
command = QuoteString(command)
logger.debug(
'--> Running chroot command on ({0}):{1}'.format(
chroot_directory,
command,
), )
chroot_command = StringCommand(
'chroot',
chroot_directory,
'sh',
'-c',
command,
)
return run_local_shell_command(
state,
host,
chroot_command,
timeout=timeout,
stdin=stdin,
success_exit_codes=success_exit_codes,
print_output=print_output,
print_input=print_input,
return_combined_output=return_combined_output,
)
def reboot(delay=10, interval=1, reboot_timeout=300):
"""
Reboot the server and wait for reconnection.
+ delay: number of seconds to wait before attempting reconnect
+ interval: interval (s) between reconnect attempts
+ reboot_timeout: total time before giving up reconnecting
**Example:**
.. code:: python
server.reboot(
name="Reboot the server and wait to reconnect",
delay=60,
reboot_timeout=600,
)
"""
# Remove this now, before we reboot the server - if the reboot fails (expected or
# not) we'll error if we don't clean this up now. Will simply be re-uploaded if
# needed later.
def remove_any_askpass_file(state, host):
remove_any_sudo_askpass_file(host)
yield FunctionCommand(remove_any_askpass_file, (), {})
yield StringCommand("reboot",
success_exit_codes=[0,
-1]) # -1 being error/disconnected
def wait_and_reconnect(state, host): # pragma: no cover
sleep(delay)
max_retries = round(reboot_timeout / interval)
host.connection = None # remove the connection object
retries = 0
while True:
host.connect(show_errors=False)
if host.connection:
break
if retries > max_retries:
raise Exception(
("Server did not reboot in time (reboot_timeout={0}s)"
).format(reboot_timeout), )
sleep(interval)
retries += 1
yield FunctionCommand(wait_and_reconnect, (), {})
def command(self, command):
bits = ["certbot", command]
if self.cert_name:
bits.append("--cert-name {0}".format(self.cert_name))
if self.dns_provider:
# create dns flag like, --dns-google
bits.append("--dns-{0}".format(self.dns_provider))
if self._domains:
bits.append("-d {0}".format(self.domains))
return StringCommand(*bits)
def sed_replace(
filename,
line,
replace,
flags=None,
backup=False,
interpolate_variables=False,
):
flags = ''.join(flags) if flags else ''
line = line.replace('/', r'\/')
replace = str(replace)
replace = replace.replace('/', r'\/')
backup_extension = get_timestamp()
if interpolate_variables:
line = line.replace('"', '\\"')
replace = replace.replace('"', '\\"')
sed_script_formatter = '"s/{0}/{1}/{2}"'
else:
# Single quotes cannot contain other single quotes, even when escaped , so turn
# each ' into '"'"' (end string, double quote the single quote, (re)start string)
line = line.replace("'", "'\"'\"'")
replace = replace.replace("'", "'\"'\"'")
sed_script_formatter = "'s/{0}/{1}/{2}'"
sed_script = sed_script_formatter.format(line, replace, flags)
sed_command = StringCommand('sed', '-i.{0}'.format(backup_extension),
sed_script, filename)
if not backup: # if we're not backing up, remove the file *if* sed succeeds
backup_filename = '{0}.{1}'.format(filename, backup_extension)
sed_command = StringCommand(sed_command, '&&', 'rm', '-f',
backup_filename)
return sed_command
def command(
self,
postgresql_user=None,
postgresql_password=None,
postgresql_host=None,
postgresql_port=None,
):
psql_command = make_execute_psql_command(
self.postgresql_command,
user=postgresql_user,
password=postgresql_password,
host=postgresql_host,
port=postgresql_port,
)
return StringCommand(psql_command, '||', 'true')
def command(
self,
postgresql_user=None,
postgresql_password=None,
postgresql_host=None,
postgresql_port=None,
):
psql_command = make_execute_psql_command(
self.postgresql_command,
user=postgresql_user,
password=postgresql_password,
host=postgresql_host,
port=postgresql_port,
)
return StringCommand(
'which',
'psql',
'>',
'/dev/null',
'&&',
StringCommand('(', psql_command, ')', separator=''),
'||',
'true',
)
def command(
self,
# Details for speaking to MySQL via `mysql` CLI via `mysql` CLI
mysql_user=None,
mysql_password=None,
mysql_host=None,
mysql_port=None,
):
mysql_command = make_execute_mysql_command(
self.mysql_command,
user=mysql_user,
password=mysql_password,
host=mysql_host,
port=mysql_port,
)
return StringCommand(mysql_command, '||', 'true')
def run_shell_command(
state,
host,
command,
get_pty=False,
timeout=None,
stdin=None,
success_exit_codes=None,
print_output=False,
print_input=False,
return_combined_output=False,
**command_kwargs,
):
container_id = host.host_data["docker_container_id"]
# Don't sudo/su in Docker - is this the right thing to do? Makes deploys that
# target SSH systems work w/Docker out of the box (ie most docker commands
# are run as root).
for key in ("sudo", "su_user"):
command_kwargs.pop(key, None)
command = make_unix_command_for_host(state, host, command,
**command_kwargs)
command = QuoteString(command)
docker_flags = "-it" if get_pty else "-i"
docker_command = StringCommand(
"docker",
"exec",
docker_flags,
container_id,
"sh",
"-c",
command,
)
return ssh.run_shell_command(
state,
host,
docker_command,
timeout=timeout,
stdin=stdin,
success_exit_codes=success_exit_codes,
print_output=print_output,
print_input=print_input,
return_combined_output=return_combined_output,
)
def reboot(delay=10, interval=1, reboot_timeout=300, state=None, host=None):
'''
Reboot the server and wait for reconnection.
+ delay: number of seconds to wait before attempting reconnect
+ interval: interval (s) between reconnect attempts
+ reboot_timeout: total time before giving up reconnecting
Note: Probably want sudo enabled.
Example:
.. code:: python
server.reboot(
name='Reboot the server and wait to reconnect',
delay=5,
timeout=30,
)
'''
logger.warning('The server.reboot operation is in beta!')
yield StringCommand('reboot',
success_exit_codes=[-1]) # -1 being error/disconnected
def wait_and_reconnect(state, host): # pragma: no cover
sleep(delay)
max_retries = round(reboot_timeout / interval)
host.connection = None # remove the connection object
retries = 0
while True:
host.connect(state, show_errors=False)
if host.connection:
break
if retries > max_retries:
raise Exception(
('Server did not reboot in time (reboot_timeout={0}s)'
).format(reboot_timeout))
sleep(interval)
retries += 1
yield FunctionCommand(wait_and_reconnect, (), {})
def jsontest_function(self, test_name, test_data, fact=fact):
short_fact = None
if isinstance(fact, ShortFactBase):
short_fact = fact
fact = fact.fact()
command_to_check = None
if callable(fact.command):
args = test_data.get('arg', [])
if not isinstance(args, list):
args = [args]
command = fact.command(*args)
if args or 'command' in test_data:
command_to_check = command
elif 'command' in test_data:
command_to_check = fact.command
if command_to_check:
assert get_command_string(
StringCommand(command_to_check)) == test_data['command']
data = fact.process(test_data['output'])
if short_fact:
data = short_fact.process_data(data)
# Encode/decode data to ensure datetimes/etc become JSON
data = json.loads(json.dumps(data, default=json_encode))
try:
assert data == test_data['fact']
except AssertionError as e:
print()
print('--> GOT:\n',
json.dumps(data, indent=4, default=json_encode))
print(
'--> WANT:',
json.dumps(
test_data['fact'],
indent=4,
default=json_encode,
))
raise e
def run_shell_command(state,
host,
command,
get_pty=False,
timeout=None,
stdin=None,
success_exit_codes=None,
print_output=False,
print_input=False,
return_combined_output=False,
**command_kwargs):
container_id = host.host_data['docker_container_id']
# Don't sudo/su in Docker - is this the right thing to do? Makes deploys that
# target SSH systems work w/Docker out of the box (ie most docker commands
# are run as root).
for key in ('sudo', 'su_user'):
command_kwargs.pop(key, None)
command = make_unix_command(command, **command_kwargs)
command = QuoteString(command)
docker_flags = '-it' if get_pty else '-i'
docker_command = StringCommand(
'docker',
'exec',
docker_flags,
container_id,
'sh',
'-c',
command,
)
return ssh.run_shell_command(
state,
host,
docker_command,
timeout=timeout,
stdin=stdin,
success_exit_codes=success_exit_codes,
print_output=print_output,
print_input=print_input,
return_combined_output=return_combined_output,
)
def run_shell_command(
state,
host,
command,
get_pty=False,
timeout=None,
stdin=None,
success_exit_codes=None,
print_output=False,
print_input=False,
return_combined_output=False,
**command_kwargs,
):
chroot_directory = host.connector_data["chroot_directory"]
command = make_unix_command_for_host(state, host, command,
**command_kwargs)
command = QuoteString(command)
logger.debug("--> Running chroot command on (%s): %s", chroot_directory,
command)
chroot_command = StringCommand(
"chroot",
chroot_directory,
"sh",
"-c",
command,
)
return run_local_shell_command(
state,
host,
chroot_command,
timeout=timeout,
stdin=stdin,
success_exit_codes=success_exit_codes,
print_output=print_output,
print_input=print_input,
return_combined_output=return_combined_output,
)
def dump(
dest,
database=None,
# Details for speaking to PostgreSQL via `psql` CLI
psql_user=None,
psql_password=None,
psql_host=None,
psql_port=None,
):
"""
Dump a PostgreSQL database into a ``.sql`` file. Requires ``pg_dump``.
+ dest: name of the file to dump the SQL to
+ database: name of the database to dump
+ psql_*: global module arguments, see above
**Example:**
.. code:: python
postgresql.dump(
name="Dump the pyinfra_stuff database",
dest="/tmp/pyinfra_stuff.dump",
database="pyinfra_stuff",
sudo_user="******",
)
"""
yield StringCommand(
make_psql_command(
executable="pg_dump",
database=database,
user=psql_user,
password=psql_password,
host=psql_host,
port=psql_port,
),
">",
dest,
)